def test_for_reflections(self, refl):
from dials.algorithms.integration.sum import IntegrationAlgorithm
from dials.array_family import flex
from dials.algorithms.statistics import \
kolmogorov_smirnov_test_standard_normal
# Get the calculated background and simulated background
B_sim = refl['background.sim.a'].as_double()
I_sim = refl['intensity.sim'].as_double()
I_exp = refl['intensity.exp']
# Set the background as simulated
shoebox = refl['shoebox']
for i in range(len(shoebox)):
bg = shoebox[i].background
ms = shoebox[i].mask
for j in range(len(bg)):
bg[j] = B_sim[i]
# Integrate
integration = IntegrationAlgorithm()
integration(refl)
I_cal = refl['intensity.sum.value']
I_var = refl['intensity.sum.variance']
# Only select variances greater than zero
mask = I_var > 0
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
# Calculate the z score
perc = self.mv3n_tolerance_interval(3 * 3)
Z = (I_cal - I_exp) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print "Z: mean: %f, var: %f" % (Z_mean, Z_var)
# Do the kolmogorov smirnov test
D, p = kolmogorov_smirnov_test_standard_normal(Z)
print "KS: D: %f, p-value: %f" % (D, p)
# FIXME Z score should be a standard normal distribution. When background is
# the main component, we do indeed see that the z score is in a standard
# normal distribution. When the intensity dominates, the variance of the Z
# scores decreases indicating that for increasing intensity of the signal,
# the variance is over estimated.
assert (abs(Z_mean) <= 3 * Z_var)
#from matplotlib import pylab
#pylab.hist(Z, 20)
#pylab.show()
#Z_I = sorted(Z)
##n = int(0.05 * len(Z_I))
##Z_I = Z_I[n:-n]
##mv = flex.mean_and_variance(flex.double(Z_I))
##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation())
#edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))]
#cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I]
print 'OK'
def test_for_reflections(self, refl, filename):
from dials.array_family import flex
from dials.algorithms.statistics import \
kolmogorov_smirnov_test_standard_normal
from os.path import basename
print basename(filename)
#refl = self.reference
# Get the calculated background and simulated background
B_sim = refl['background.sim.a'].as_double()
I_sim = refl['intensity.sim'].as_double()
I_exp = refl['intensity.exp']
# Set the background as simulated
shoebox = refl['shoebox']
for i in range(len(shoebox)):
bg = shoebox[i].background
ms = shoebox[i].mask
for j in range(len(bg)):
bg[j] = B_sim[i]
# Integrate
integration = self.experiments[0].profile.fitting_class()(
self.experiments[0])
old_size = len(refl)
refl.extend(self.reference)
integration.model(self.reference)
integration.fit(refl)
reference = refl[-len(self.reference):]
refl = refl[:len(self.reference)]
assert (len(refl) == old_size)
assert (len(reference) == len(self.reference))
I_cal = refl['intensity.prf.value']
I_var = refl['intensity.prf.variance']
# Check the reference profiles and spots are ok
#self.check_profiles(integration.learner)
#self.check_reference(reference)
#np = reference.locate().size()
#for i in range(np):
#profile = reference.locate().profile(i)
#print "Profile: %d" % i
#p = (profile.as_numpy_array() * 1000)
#import numpy as np
#p = p.astype(np.int)
#print p
# Only select variances greater than zero
mask = refl.get_flags(refl.flags.integrated_prf)
assert (mask.count(True) > 0)
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
mask = I_var > 0
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
# Calculate the z score
perc = self.mv3n_tolerance_interval(3 * 3)
Z = (I_cal - I_sim) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print "Z: mean: %f, var: %f" % (Z_mean, Z_var)
# Do the kolmogorov smirnov test
D, p = kolmogorov_smirnov_test_standard_normal(Z)
print "KS: D: %f, p-value: %f" % (D, p)
# FIXME Z score should be a standard normal distribution. When background is
# the main component, we do indeed see that the z score is in a standard
# normal distribution. When the intensity dominates, the variance of the Z
# scores decreases indicating that for increasing intensity of the signal,
# the variance is over estimated.
#assert(abs(Z_mean) <= 3 * Z_var)
#from matplotlib import pylab
#pylab.hist(Z, 20)
#pylab.show()
#Z_I = sorted(Z)
##n = int(0.05 * len(Z_I))
##Z_I = Z_I[n:-n]
##mv = flex.mean_and_variance(flex.double(Z_I))
##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation())
#edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))]
#cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I]
print 'OK'
def test_for_reflections(self, refl):
from dials.algorithms.integration.sum import IntegrationAlgorithm
from dials.array_family import flex
from dials.algorithms.statistics import \
kolmogorov_smirnov_test_standard_normal
# Get the calculated background and simulated background
B_sim = refl['background.sim.a'].as_double()
I_sim = refl['intensity.sim'].as_double()
I_exp = refl['intensity.exp']
# Set the background as simulated
shoebox = refl['shoebox']
for i in range(len(shoebox)):
bg = shoebox[i].background
ms = shoebox[i].mask
for j in range(len(bg)):
bg[j] = B_sim[i]
# Integrate
integration = IntegrationAlgorithm()
integration(refl)
I_cal = refl['intensity.sum.value']
I_var = refl['intensity.sum.variance']
# Only select variances greater than zero
mask = I_var > 0
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
# Calculate the z score
perc = self.mv3n_tolerance_interval(3*3)
Z = (I_cal - I_exp) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print "Z: mean: %f, var: %f" % (Z_mean, Z_var)
# Do the kolmogorov smirnov test
D, p = kolmogorov_smirnov_test_standard_normal(Z)
print "KS: D: %f, p-value: %f" % (D, p)
# FIXME Z score should be a standard normal distribution. When background is
# the main component, we do indeed see that the z score is in a standard
# normal distribution. When the intensity dominates, the variance of the Z
# scores decreases indicating that for increasing intensity of the signal,
# the variance is over estimated.
assert(abs(Z_mean) <= 3 * Z_var)
#from matplotlib import pylab
#pylab.hist(Z, 20)
#pylab.show()
#Z_I = sorted(Z)
##n = int(0.05 * len(Z_I))
##Z_I = Z_I[n:-n]
##mv = flex.mean_and_variance(flex.double(Z_I))
##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation())
#edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))]
#cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I]
print 'OK'
def test_for_reflections(self, refl, filename):
from dials.array_family import flex
from dials.algorithms.statistics import \
kolmogorov_smirnov_test_standard_normal
from os.path import basename
print basename(filename)
#refl = self.reference
# Get the calculated background and simulated background
B_sim = refl['background.sim.a'].as_double()
I_sim = refl['intensity.sim'].as_double()
I_exp = refl['intensity.exp']
# Set the background as simulated
shoebox = refl['shoebox']
for i in range(len(shoebox)):
bg = shoebox[i].background
ms = shoebox[i].mask
for j in range(len(bg)):
bg[j] = B_sim[i]
# Integrate
integration = self.experiments[0].profile.fitting_class()(self.experiments[0])
old_size = len(refl)
refl.extend(self.reference)
integration.model(self.reference)
integration.fit(refl)
reference = refl[-len(self.reference):]
refl = refl[:len(self.reference)]
assert(len(refl) == old_size)
assert(len(reference) == len(self.reference))
I_cal = refl['intensity.prf.value']
I_var = refl['intensity.prf.variance']
# Check the reference profiles and spots are ok
#self.check_profiles(integration.learner)
#self.check_reference(reference)
#np = reference.locate().size()
#for i in range(np):
#profile = reference.locate().profile(i)
#print "Profile: %d" % i
#p = (profile.as_numpy_array() * 1000)
#import numpy as np
#p = p.astype(np.int)
#print p
# Only select variances greater than zero
mask = refl.get_flags(refl.flags.integrated_prf)
assert(mask.count(True) > 0)
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
mask = I_var > 0
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
# Calculate the z score
perc = self.mv3n_tolerance_interval(3*3)
Z = (I_cal - I_sim) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print "Z: mean: %f, var: %f" % (Z_mean, Z_var)
# Do the kolmogorov smirnov test
D, p = kolmogorov_smirnov_test_standard_normal(Z)
print "KS: D: %f, p-value: %f" % (D, p)
# FIXME Z score should be a standard normal distribution. When background is
# the main component, we do indeed see that the z score is in a standard
# normal distribution. When the intensity dominates, the variance of the Z
# scores decreases indicating that for increasing intensity of the signal,
# the variance is over estimated.
#assert(abs(Z_mean) <= 3 * Z_var)
#from matplotlib import pylab
#pylab.hist(Z, 20)
#pylab.show()
#Z_I = sorted(Z)
##n = int(0.05 * len(Z_I))
##Z_I = Z_I[n:-n]
##mv = flex.mean_and_variance(flex.double(Z_I))
##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation())
#edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))]
#cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I]
print 'OK'
def test_for_reflections(self, refl, filename):
from dials.algorithms.integration import ProfileFittingReciprocalSpace
from dials.array_family import flex
from dials.algorithms.shoebox import MaskCode
from dials.algorithms.statistics import kolmogorov_smirnov_test_standard_normal
from math import erf, sqrt, pi
from copy import deepcopy
from dials.algorithms.simulation.reciprocal_space import Simulator
from os.path import basename
print(basename(filename))
# refl = self.reference
# Get the calculated background and simulated background
B_sim = refl["background.sim"].as_double()
I_sim = refl["intensity.sim"].as_double()
I_exp = refl["intensity.exp"]
# Set the background as simulated
shoebox = refl["shoebox"]
for i in range(len(shoebox)):
bg = shoebox[i].background
ms = shoebox[i].mask
for j in range(len(bg)):
bg[j] = B_sim[i]
# Integrate
integration = ProfileFittingReciprocalSpace(
grid_size=4,
threshold=0.02,
frame_interval=0,
n_sigma=4,
sigma_b=0.024 * pi / 180.0,
sigma_m=0.044 * pi / 180.0,
)
old_size = len(refl)
refl.extend(self.reference)
integration(self.experiment, refl)
reference = refl[-len(self.reference):]
refl = refl[:len(self.reference)]
assert len(refl) == old_size
assert len(reference) == len(self.reference)
I_cal = refl["intensity.sum.value"]
I_var = refl["intensity.sum.variance"]
# Check the reference profiles and spots are ok
self.check_profiles(integration.learner)
self.check_reference(reference)
# reference = integration.learner
# np = reference.locate().size()
# for i in range(np):
# profile = reference.locate().profile(i)
# print "Profile: %d" % i
# p = (profile.as_numpy_array() * 1000)
# import numpy as np
# p = p.astype(np.int)
# print p
# Only select variances greater than zero
mask = refl.get_flags(refl.flags.integrated)
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
# Calculate the z score
perc = self.mv3n_tolerance_interval(3 * 3)
Z = (I_cal - I_exp) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print("Z: mean: %f, var: %f" % (Z_mean, Z_var))
# Do the kolmogorov smirnov test
D, p = kolmogorov_smirnov_test_standard_normal(Z)
print("KS: D: %f, p-value: %f" % (D, p))
# FIXME Z score should be a standard normal distribution. When background is
# the main component, we do indeed see that the z score is in a standard
# normal distribution. When the intensity dominates, the variance of the Z
# scores decreases indicating that for increasing intensity of the signal,
# the variance is over estimated.
# assert(abs(Z_mean) <= 3 * Z_var)
# from matplotlib import pylab
# pylab.hist(Z, 20)
# pylab.show()
# Z_I = sorted(Z)
##n = int(0.05 * len(Z_I))
##Z_I = Z_I[n:-n]
##mv = flex.mean_and_variance(flex.double(Z_I))
##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation())
# edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))]
# cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I]
print("OK")
