def test_for_reflections(self, refl): from dials.algorithms.integration.sum import IntegrationAlgorithm from dials.array_family import flex from dials.algorithms.statistics import \ kolmogorov_smirnov_test_standard_normal # Get the calculated background and simulated background B_sim = refl['background.sim.a'].as_double() I_sim = refl['intensity.sim'].as_double() I_exp = refl['intensity.exp'] # Set the background as simulated shoebox = refl['shoebox'] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = IntegrationAlgorithm() integration(refl) I_cal = refl['intensity.sum.value'] I_var = refl['intensity.sum.variance'] # Only select variances greater than zero mask = I_var > 0 I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3 * 3) Z = (I_cal - I_exp) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print "KS: D: %f, p-value: %f" % (D, p) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. assert (abs(Z_mean) <= 3 * Z_var) #from matplotlib import pylab #pylab.hist(Z, 20) #pylab.show() #Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) #edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] #cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print 'OK'
def test_for_reflections(self, refl, filename): from dials.array_family import flex from dials.algorithms.statistics import \ kolmogorov_smirnov_test_standard_normal from os.path import basename print basename(filename) #refl = self.reference # Get the calculated background and simulated background B_sim = refl['background.sim.a'].as_double() I_sim = refl['intensity.sim'].as_double() I_exp = refl['intensity.exp'] # Set the background as simulated shoebox = refl['shoebox'] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = self.experiments[0].profile.fitting_class()( self.experiments[0]) old_size = len(refl) refl.extend(self.reference) integration.model(self.reference) integration.fit(refl) reference = refl[-len(self.reference):] refl = refl[:len(self.reference)] assert (len(refl) == old_size) assert (len(reference) == len(self.reference)) I_cal = refl['intensity.prf.value'] I_var = refl['intensity.prf.variance'] # Check the reference profiles and spots are ok #self.check_profiles(integration.learner) #self.check_reference(reference) #np = reference.locate().size() #for i in range(np): #profile = reference.locate().profile(i) #print "Profile: %d" % i #p = (profile.as_numpy_array() * 1000) #import numpy as np #p = p.astype(np.int) #print p # Only select variances greater than zero mask = refl.get_flags(refl.flags.integrated_prf) assert (mask.count(True) > 0) I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) mask = I_var > 0 I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3 * 3) Z = (I_cal - I_sim) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print "KS: D: %f, p-value: %f" % (D, p) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. #assert(abs(Z_mean) <= 3 * Z_var) #from matplotlib import pylab #pylab.hist(Z, 20) #pylab.show() #Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) #edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] #cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print 'OK'
def test_for_reflections(self, refl): from dials.algorithms.integration.sum import IntegrationAlgorithm from dials.array_family import flex from dials.algorithms.statistics import \ kolmogorov_smirnov_test_standard_normal # Get the calculated background and simulated background B_sim = refl['background.sim.a'].as_double() I_sim = refl['intensity.sim'].as_double() I_exp = refl['intensity.exp'] # Set the background as simulated shoebox = refl['shoebox'] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = IntegrationAlgorithm() integration(refl) I_cal = refl['intensity.sum.value'] I_var = refl['intensity.sum.variance'] # Only select variances greater than zero mask = I_var > 0 I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3*3) Z = (I_cal - I_exp) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print "KS: D: %f, p-value: %f" % (D, p) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. assert(abs(Z_mean) <= 3 * Z_var) #from matplotlib import pylab #pylab.hist(Z, 20) #pylab.show() #Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) #edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] #cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print 'OK'
def test_for_reflections(self, refl, filename): from dials.array_family import flex from dials.algorithms.statistics import \ kolmogorov_smirnov_test_standard_normal from os.path import basename print basename(filename) #refl = self.reference # Get the calculated background and simulated background B_sim = refl['background.sim.a'].as_double() I_sim = refl['intensity.sim'].as_double() I_exp = refl['intensity.exp'] # Set the background as simulated shoebox = refl['shoebox'] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = self.experiments[0].profile.fitting_class()(self.experiments[0]) old_size = len(refl) refl.extend(self.reference) integration.model(self.reference) integration.fit(refl) reference = refl[-len(self.reference):] refl = refl[:len(self.reference)] assert(len(refl) == old_size) assert(len(reference) == len(self.reference)) I_cal = refl['intensity.prf.value'] I_var = refl['intensity.prf.variance'] # Check the reference profiles and spots are ok #self.check_profiles(integration.learner) #self.check_reference(reference) #np = reference.locate().size() #for i in range(np): #profile = reference.locate().profile(i) #print "Profile: %d" % i #p = (profile.as_numpy_array() * 1000) #import numpy as np #p = p.astype(np.int) #print p # Only select variances greater than zero mask = refl.get_flags(refl.flags.integrated_prf) assert(mask.count(True) > 0) I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) mask = I_var > 0 I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3*3) Z = (I_cal - I_sim) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print "KS: D: %f, p-value: %f" % (D, p) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. #assert(abs(Z_mean) <= 3 * Z_var) #from matplotlib import pylab #pylab.hist(Z, 20) #pylab.show() #Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) #edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] #cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print 'OK'
def test_for_reflections(self, refl, filename): from dials.algorithms.integration import ProfileFittingReciprocalSpace from dials.array_family import flex from dials.algorithms.shoebox import MaskCode from dials.algorithms.statistics import kolmogorov_smirnov_test_standard_normal from math import erf, sqrt, pi from copy import deepcopy from dials.algorithms.simulation.reciprocal_space import Simulator from os.path import basename print(basename(filename)) # refl = self.reference # Get the calculated background and simulated background B_sim = refl["background.sim"].as_double() I_sim = refl["intensity.sim"].as_double() I_exp = refl["intensity.exp"] # Set the background as simulated shoebox = refl["shoebox"] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = ProfileFittingReciprocalSpace( grid_size=4, threshold=0.02, frame_interval=0, n_sigma=4, sigma_b=0.024 * pi / 180.0, sigma_m=0.044 * pi / 180.0, ) old_size = len(refl) refl.extend(self.reference) integration(self.experiment, refl) reference = refl[-len(self.reference):] refl = refl[:len(self.reference)] assert len(refl) == old_size assert len(reference) == len(self.reference) I_cal = refl["intensity.sum.value"] I_var = refl["intensity.sum.variance"] # Check the reference profiles and spots are ok self.check_profiles(integration.learner) self.check_reference(reference) # reference = integration.learner # np = reference.locate().size() # for i in range(np): # profile = reference.locate().profile(i) # print "Profile: %d" % i # p = (profile.as_numpy_array() * 1000) # import numpy as np # p = p.astype(np.int) # print p # Only select variances greater than zero mask = refl.get_flags(refl.flags.integrated) I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3 * 3) Z = (I_cal - I_exp) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print("Z: mean: %f, var: %f" % (Z_mean, Z_var)) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print("KS: D: %f, p-value: %f" % (D, p)) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. # assert(abs(Z_mean) <= 3 * Z_var) # from matplotlib import pylab # pylab.hist(Z, 20) # pylab.show() # Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) # edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] # cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print("OK")