def outlier_rejection(reflections):
# http://scripts.iucr.org/cgi-bin/paper?ba0032
if len(reflections) == 1:
return reflections
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
i_max = flex.max_index(intensities)
sel = flex.bool(len(reflections), True)
sel[i_max] = False
i_test = intensities[i_max]
var_test = variances[i_max]
intensities_subset = intensities.select(sel)
var_subset = variances.select(sel)
var_prior = var_test + 1/flex.sum(1/var_subset)
p_prior = 1/math.sqrt(2*math.pi * var_prior) * math.exp(
-(i_test - flex.mean(intensities_subset))**2/(2 * var_prior))
#print p_prior
if p_prior > 1e-10:
return reflections
return outlier_rejection(reflections.select(sel))
def plot_prob_for_zero(c, b, s):
from math import log, exp, factorial
from dials.array_family import flex
L = flex.double(flex.grid(100, 100))
MASK = flex.bool(flex.grid(100, 100))
c = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]
b = [bb / sum(b) for bb in b]
s = [ss / sum(s) for ss in s]
for BB in range(0, 100):
for SS in range(0, 100):
B = 0 + BB / 10000.0
S = 0 + SS / 40.0
LL = 0
for i in range(len(b)):
if B*b[i] + S*s[i] <= 0:
MASK[BB, SS] = True
LL = -999999
break
else:
LL += c[i]*log(B*b[i]+S*s[i]) - log(factorial(c[i])) - B*b[i] - S*s[i]
L[BB, SS] = LL
index = flex.max_index(L)
i = index % 100
j = index // 100
B = 0 + j / 10000.0
S = 0 + i / 40.0
print flex.max(L), B, S
from matplotlib import pylab
import numpy
im = numpy.ma.masked_array(flex.exp(L).as_numpy_array(), mask=MASK.as_numpy_array())
pylab.imshow(im)
pylab.show()
exit(0)
def show_image(c,b,s, BB=None, SS=None):
import numpy.ma
from dials.array_family import flex
N = 100
im = flex.double(flex.grid(N, N))
mask = flex.bool(flex.grid(N, N))
for j in range(N):
for i in range(N):
B = -1.0 + j * 10.0 / N
S = -1.0 + i * 10.0 / N
im[j,i], mask[j,i] = func(c,b,s,B,S)
im[j,i] = -im[j,i]
masked_im = numpy.ma.array(
# im.as_numpy_array(),
flex.exp(im).as_numpy_array(),
mask = mask.as_numpy_array())
mask2 = flex.bool(flex.grid(N, N))
indices = []
for i in range(len(mask)):
if mask[i] == False:
indices.append(i)
indices = flex.size_t(indices)
ind = flex.max_index(im.select(indices))
ind = indices[ind]
maxy = -1.0 + (ind % N) * 10.0 / N
maxx = -1.0 + (ind // N) * 10.0 / N
from matplotlib import pylab
pylab.imshow(masked_im, origin='bottom', extent=[-1.0, 9.0, -1.0, 9.0])
if YY is not None and XX is not None:
pylab.plot(YY, XX)
pylab.scatter([maxy], [maxx])
pylab.show()
def outlier_rejection(reflections):
# http://scripts.iucr.org/cgi-bin/paper?ba0032
if len(reflections) == 1:
return reflections
intensities = reflections["intensity.sum.value"]
variances = reflections["intensity.sum.variance"]
i_max = flex.max_index(intensities)
sel = flex.bool(len(reflections), True)
sel[i_max] = False
i_test = intensities[i_max]
var_test = variances[i_max]
intensities_subset = intensities.select(sel)
var_subset = variances.select(sel)
var_prior = var_test + 1 / flex.sum(1 / var_subset)
p_prior = (1 / math.sqrt(2 * math.pi * var_prior) *
math.exp(-((i_test - flex.mean(intensities_subset))**2) /
(2 * var_prior)))
if p_prior > 1e-10:
return reflections
return outlier_rejection(reflections.select(sel))
def plot_prob_for_zero(c, b, s):
from math import log, exp, factorial
from dials.array_family import flex
L = flex.double(flex.grid(100, 100))
MASK = flex.bool(flex.grid(100, 100))
c = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]
b = [bb / sum(b) for bb in b]
s = [ss / sum(s) for ss in s]
for BB in range(0, 100):
for SS in range(0, 100):
B = 0 + BB / 10000.0
S = 0 + SS / 40.0
LL = 0
for i in range(len(b)):
if B * b[i] + S * s[i] <= 0:
MASK[BB, SS] = True
LL = -999999
break
else:
LL += (
c[i] * log(B * b[i] + S * s[i])
- log(factorial(c[i]))
- B * b[i]
- S * s[i]
)
L[BB, SS] = LL
index = flex.max_index(L)
i = index % 100
j = index // 100
B = 0 + j / 10000.0
S = 0 + i / 40.0
print(flex.max(L), B, S)
from matplotlib import pylab
import numpy
im = numpy.ma.masked_array(flex.exp(L).as_numpy_array(), mask=MASK.as_numpy_array())
pylab.imshow(im)
pylab.show()
exit(0)
def show_image(c, b, s, BB=None, SS=None):
import numpy.ma
from dials.array_family import flex
N = 100
im = flex.double(flex.grid(N, N))
mask = flex.bool(flex.grid(N, N))
for j in range(N):
for i in range(N):
B = -1.0 + j * 10.0 / N
S = -1.0 + i * 10.0 / N
im[j, i], mask[j, i] = func(c, b, s, B, S)
im[j, i] = -im[j, i]
masked_im = numpy.ma.array(
# im.as_numpy_array(),
flex.exp(im).as_numpy_array(),
mask=mask.as_numpy_array(),
)
mask2 = flex.bool(flex.grid(N, N))
indices = []
for i in range(len(mask)):
if mask[i] == False:
indices.append(i)
indices = flex.size_t(indices)
ind = flex.max_index(im.select(indices))
ind = indices[ind]
maxy = -1.0 + (ind % N) * 10.0 / N
maxx = -1.0 + (ind // N) * 10.0 / N
from matplotlib import pylab
pylab.imshow(masked_im, origin="bottom", extent=[-1.0, 9.0, -1.0, 9.0])
if YY is not None and XX is not None:
pylab.plot(YY, XX)
pylab.scatter([maxy], [maxx])
pylab.show()
def estimate_resolution_limit_distl_method1(reflections, plot_filename=None):
# Implementation of Method 1 (section 2.4.4) of:
# Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
# J. Appl. Cryst. (2006). 39, 112-119
# https://doi.org/10.1107/S0021889805040677
variances = reflections["intensity.sum.variance"]
sel = variances > 0
reflections = reflections.select(sel)
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
d_star_cubed = flex.pow(reflections["rlp"].norms(), 3)
step = 2
while len(reflections) / step > 40:
step += 1
order = flex.sort_permutation(d_spacings, reverse=True)
ds3_subset = flex.double()
d_subset = flex.double()
for i in range(len(reflections) // step):
ds3_subset.append(d_star_cubed[order[i * step]])
d_subset.append(d_spacings[order[i * step]])
x = flex.double(range(len(ds3_subset)))
# (i)
# Usually, Pm is the last point, that is, m = n. But m could be smaller than
# n if an unusually high number of spots are detected around a certain
# intermediate resolution. In that case, our search for the image resolution
# does not go outside the spot 'bump;. This is particularly useful when
# ice-rings are present.
slopes = (ds3_subset[1:] - ds3_subset[0]) / (x[1:] - x[0])
skip_first = 3
p_m = flex.max_index(slopes[skip_first:]) + 1 + skip_first
# (ii)
x1 = matrix.col((0, ds3_subset[0]))
x2 = matrix.col((p_m, ds3_subset[p_m]))
gaps = flex.double([0])
v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize()
for i in range(1, p_m):
x0 = matrix.col((i, ds3_subset[i]))
r = x1 - x0
g = abs(v.dot(r))
gaps.append(g)
mv = flex.mean_and_variance(gaps)
s = mv.unweighted_sample_standard_deviation()
# (iii)
p_k = flex.max_index(gaps)
g_k = gaps[p_k]
p_g = p_k
for i in range(p_k + 1, len(gaps)):
g_i = gaps[i]
if g_i > (g_k - 0.5 * s):
p_g = i
d_g = d_subset[p_g]
noisiness = 0
n = len(ds3_subset)
for i in range(n - 1):
for j in range(i + 1, n - 1):
if slopes[i] >= slopes[j]:
noisiness += 1
noisiness /= (n - 1) * (n - 2) / 2
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(range(len(ds3_subset)), ds3_subset)
ax.set_ylabel("D^-3")
xlim = pyplot.xlim()
ylim = pyplot.ylim()
ax.vlines(p_g, ylim[0], ylim[1], colors="red")
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.savefig(plot_filename)
pyplot.close()
return d_g, noisiness
def determine_fraction_new_cutoff(self, fraction_new, cutoff): imax = flex.max_index(fraction_new) isel = (fraction_new < cutoff).iselection() return isel[(isel > imax).iselection()[0]]
def run_stills_pred_param(self, verbose = False):
if verbose:
print 'Testing derivatives for StillsPredictionParameterisation'
print '========================================================'
# Build a prediction parameterisation for the stills experiment
pred_param = StillsPredictionParameterisation(self.stills_experiments,
detector_parameterisations = [self.det_param],
beam_parameterisations = [self.s0_param],
xl_orientation_parameterisations = [self.xlo_param],
xl_unit_cell_parameterisations = [self.xluc_param])
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(self.stills_experiments)
ref_predictor.update()
ref_predictor.predict(self.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(self.reflections)
fd_grads = self.get_fd_gradients(pred_param, ref_predictor)
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
# compare FD with analytical calculations
if verbose: print "\nParameter {0}: {1}". format(i, fd_grad['name'])
for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]):
if verbose: print name
a = fd_grad[name]
b = an_grad[name]
abs_error = a - b
denom = a + b
fns = five_number_summary(abs_error)
if verbose: print (" summary of absolute errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
assert flex.max(flex.abs(abs_error)) < 0.0003
# largest absolute error found to be about 0.00025 for dY/dp of
# Crystal0g_param_3. Reject outlying absolute errors and test again.
iqr = fns[3] - fns[1]
# skip further stats on errors with an iqr of near zero, e.g. dDeltaPsi_dp
# for detector parameters, which are all equal to zero
if iqr < 1.e-10:
continue
sel1 = abs_error < fns[3] + 1.5 * iqr
sel2 = abs_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(abs_error.select(sel)))
tst_val = abs_error.select(sel)[tst]
n_outliers = sel.count(False)
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"absolute error: {1:9.6f}").format(n_outliers, tst_val)
# largest absolute error now 0.000086 for dX/dp of Beam0Mu2
assert abs(tst_val) < 0.00009
# Completely skip parameters with FD gradients all zero (e.g. gradients of
# DeltaPsi for detector parameters)
sel1 = flex.abs(a) < 1.e-10
if sel1.all_eq(True):
continue
# otherwise calculate normalised errors, by dividing absolute errors by
# the IQR (more stable than relative error calculation)
norm_error = abs_error / iqr
fns = five_number_summary(norm_error)
if verbose: print (" summary of normalised errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
# largest normalised error found to be about 25.7 for dY/dp of
# Crystal0g_param_3.
try:
assert flex.max(flex.abs(norm_error)) < 30
except AssertionError as e:
e.args += ("extreme normalised error value: {0}".format(
flex.max(flex.abs(norm_error))),)
raise e
# Reject outlying normalised errors and test again
iqr = fns[3] - fns[1]
if iqr > 0.:
sel1 = norm_error < fns[3] + 1.5 * iqr
sel2 = norm_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(norm_error.select(sel)))
tst_val = norm_error.select(sel)[tst]
n_outliers = sel.count(False)
# most outliers found for for dY/dp of Crystal0g_param_3 (which had
# largest errors, so no surprise there).
try:
assert n_outliers < 250
except AssertionError as e:
e.args += ("too many outliers rejected: {0}".format(n_outliers),)
raise e
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"normalised error: {1:9.6f}").format(n_outliers, tst_val)
# largest normalied error now about -4. for dX/dp of Detector0Tau1
assert abs(tst_val) < 4.5
if verbose: print
return
def test_for_reference(self):
from dials.algorithms.integration import ProfileFittingReciprocalSpace
from dials.array_family import flex
from dials.algorithms.shoebox import MaskCode
from dials.algorithms.statistics import kolmogorov_smirnov_test_standard_normal
from math import erf, sqrt, pi
from copy import deepcopy
from dials.algorithms.simulation.reciprocal_space import Simulator
from os.path import basename
# Integrate
integration = ProfileFittingReciprocalSpace(
grid_size=4,
threshold=0.00,
frame_interval=100,
n_sigma=5,
mask_n_sigma=3,
sigma_b=0.024 * pi / 180.0,
sigma_m=0.044 * pi / 180.0,
)
# Integrate the reference profiles
integration(self.experiment, self.reference)
p = integration.learner.locate().profile(0)
m = integration.learner.locate().mask(0)
locator = integration.learner.locate()
cor = locator.correlations()
for j in range(cor.all()[0]):
print(" ".join([str(cor[j, i]) for i in range(cor.all()[1])]))
# exit(0)
# from matplotlib import pylab
# pylab.imshow(cor.as_numpy_array(), interpolation='none', vmin=-1, vmax=1)
# pylab.show()
# n = locator.size()
# for i in range(n):
# c = locator.coord(i)
# p = locator.profile(i)
# vmax = flex.max(p)
# from matplotlib import pylab
# for j in range(9):
# pylab.subplot(3, 3, j+1)
# pylab.imshow(p.as_numpy_array()[j], vmin=0, vmax=vmax,
# interpolation='none')
# pylab.show()
# print "NRef: ", n
# x = []
# y = []
# for i in range(n):
# c = locator.coord(i)
# x.append(c[0])
# y.append(c[1])
# from matplotlib import pylab
# pylab.scatter(x,y)
# pylab.show()
# exit(0)
import numpy
# pmax = flex.max(p)
# scale = 100 / pmax
# print "Scale: ", 100 / pmax
# p = p.as_numpy_array() *100 / pmax
# p = p.astype(numpy.int)
# print p
# print m.as_numpy_array()
# Check the reference profiles and spots are ok
# self.check_profiles(integration.learner)
# Make sure background is zero
profiles = self.reference["rs_shoebox"]
eps = 1e-7
for p in profiles:
assert abs(flex.sum(p.background) - 0) < eps
print("OK")
# Only select variances greater than zero
mask = self.reference.get_flags(self.reference.flags.integrated)
I_cal = self.reference["intensity.sum.value"]
I_var = self.reference["intensity.sum.variance"]
B_sim = self.reference["background.sim"].as_double()
I_sim = self.reference["intensity.sim"].as_double()
I_exp = self.reference["intensity.exp"]
P_cor = self.reference["profile.correlation"]
X_pos, Y_pos, Z_pos = self.reference["xyzcal.px"].parts()
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
P_cor = P_cor.select(mask)
max_ind = flex.max_index(I_cal)
max_I = I_cal[max_ind]
max_P = self.reference[max_ind]["rs_shoebox"].data
max_C = self.reference[max_ind]["xyzcal.px"]
max_S = self.reference[max_ind]["shoebox"].data
min_ind = flex.min_index(P_cor)
min_I = I_cal[min_ind]
min_P = self.reference[min_ind]["rs_shoebox"].data
min_C = self.reference[min_ind]["xyzcal.px"]
min_S = self.reference[min_ind]["shoebox"].data
##for k in range(max_S.all()[0]):
# if False:
# for j in range(max_S.all()[1]):
# for i in range(max_S.all()[2]):
# max_S[k,j,i] = 0
# if (abs(i - max_S.all()[2] // 2) < 2 and
# abs(j - max_S.all()[1] // 2) < 2 and
# abs(k - max_S.all()[0] // 2) < 2):
# max_S[k,j,i] = 100
# p = max_P.as_numpy_array() * 100 / flex.max(max_P)
# p = p.astype(numpy.int)
# print p
# from dials.scratch.jmp.misc.test_transform import test_transform
# grid_size = 4
# ndiv = 5
# sigma_b = 0.024 * pi / 180.0
# sigma_m = 0.044 * pi / 180.0
# n_sigma = 4.0
# max_P2 = test_transform(
# self.experiment,
# self.reference[max_ind]['shoebox'],
# self.reference[max_ind]['s1'],
# self.reference[max_ind]['xyzcal.mm'][2],
# grid_size,
# sigma_m,
# sigma_b,
# n_sigma,
# ndiv)
# max_P = max_P2
ref_ind = locator.index(max_C)
ref_P = locator.profile(ref_ind)
ref_C = locator.coord(ref_ind)
print("Max Index: ", max_ind, max_I, flex.sum(max_P), flex.sum(max_S))
print("Coord: ", max_C, "Ref Coord: ", ref_C)
print("Min Index: ", min_ind, min_I, flex.sum(min_P), flex.sum(min_S))
print("Coord: ", min_C, "Ref Coord: ", ref_C)
# vmax = flex.max(max_P)
# print sum(max_S)
# print sum(max_P)
# from matplotlib import pylab, cm
# for j in range(9):
# pylab.subplot(3, 3, j+1)
# pylab.imshow(max_P.as_numpy_array()[j], vmin=0, vmax=vmax,
# interpolation='none', cmap=cm.Greys_r)
# pylab.show()
# vmax = flex.max(min_P)
# print sum(min_S)
# print sum(min_P)
# from matplotlib import pylab, cm
# for j in range(9):
# pylab.subplot(3, 3, j+1)
# pylab.imshow(min_P.as_numpy_array()[j], vmin=0, vmax=vmax,
# interpolation='none', cmap=cm.Greys_r)
# pylab.show()
# for k in range(max_S.all()[0]):
# print ''
# print 'Slice %d' % k
# for j in range(max_S.all()[1]):
# print ' '.join(["%-4d" % int(max_S[k,j,i]) for i in range(max_S.all()[2])])
print("Testing")
def f(I):
mask = flex.bool(flex.grid(9, 9, 9), False)
for k in range(9):
for j in range(9):
for i in range(9):
dx = 5 * (i - 4.5) / 4.5
dy = 5 * (j - 4.5) / 4.5
dz = 5 * (k - 4.5) / 4.5
dd = sqrt(dx**2 + dy**2 + dz**2)
if dd <= 3:
mask[k, j, i] = True
mask = mask.as_1d() & (ref_P.as_1d() > 0)
p = ref_P.as_1d().select(mask)
c = max_P.as_1d().select(mask)
return flex.sum((c - I * p)**2 / (I * p))
def df(I):
mask = flex.bool(flex.grid(9, 9, 9), False)
for k in range(9):
for j in range(9):
for i in range(9):
dx = 5 * (i - 4.5) / 4.5
dy = 5 * (j - 4.5) / 4.5
dz = 5 * (k - 4.5) / 4.5
dd = sqrt(dx**2 + dy**2 + dz**2)
if dd <= 3:
mask[k, j, i] = True
mask = mask.as_1d() & (ref_P.as_1d() > 0)
p = ref_P.as_1d().select(mask)
c = max_P.as_1d().select(mask)
b = 0
return flex.sum(p) - flex.sum(c * c / (I * I * p))
# return flex.sum(p - p*c*c / ((b + I*p)**2))
# return flex.sum(3*p*p + (c*c*p*p - 4*b*p*p) / ((b + I*p)**2))
# return flex.sum(p - c*c / (I*I*p))
# return flex.sum(p * (-c+p*I)*(c+p*I)/((p*I)**2))
def d2f(I):
mask = flex.bool(flex.grid(9, 9, 9), False)
for k in range(9):
for j in range(9):
for i in range(9):
dx = 5 * (i - 4.5) / 4.5
dy = 5 * (j - 4.5) / 4.5
dz = 5 * (k - 4.5) / 4.5
dd = sqrt(dx**2 + dy**2 + dz**2)
if dd <= 3:
mask[k, j, i] = True
mask = mask.as_1d() & (ref_P.as_1d() > 0)
p = ref_P.as_1d().select(mask)
c = max_P.as_1d().select(mask)
return flex.sum(2 * c * c * p * p / (p * I)**3)
I = 10703 # flex.sum(max_P)
mask = ref_P.as_1d() > 0
p = ref_P.as_1d().select(mask)
c = max_P.as_1d().select(mask)
for i in range(10):
I = I - df(I) / d2f(I)
# v = I*p
# I = flex.sum(c * p / v) / flex.sum(p*p / v)
print(I)
from math import log
ff = []
for I in range(9500, 11500):
ff.append(f(I))
print(sorted(range(len(ff)), key=lambda x: ff[x])[0] + 9500)
from matplotlib import pylab
pylab.plot(range(9500, 11500), ff)
pylab.show()
# exit(0)
# I = 10000
# print flex.sum((max_P - I * ref_P)**2) / flex.sum(I * ref_P)
# I = 10100
# print flex.sum((max_P - I * ref_P)**2) / flex.sum(I * ref_P)
# exit(0)
print(flex.sum(self.reference[0]["rs_shoebox"].data))
print(I_cal[0])
# Calculate the z score
perc = self.mv3n_tolerance_interval(3 * 3)
Z = (I_cal - I_sim) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print("Z: mean: %f, var: %f, sig: %f" % (Z_mean, Z_var, sqrt(Z_var)))
print(len(I_cal))
from matplotlib import pylab
from mpl_toolkits.mplot3d import Axes3D
# fig = pylab.figure()
# ax = fig.add_subplot(111, projection='3d')
# ax.scatter(X_pos, Y_pos, P_cor)
# pylab.scatter(X_pos, P_cor)
# pylab.scatter(Y_pos, P_cor)
# pylab.scatter(Z_pos, P_cor)
# pylab.hist(P_cor,100)
# pylab.scatter(P_cor, (I_cal - I_exp) / I_exp)
pylab.hist(Z, 100)
# pylab.hist(I_cal,100)
# pylab.hist(I_cal - I_sim, 100)
pylab.show()
def estimate_resolution_limit_distl_method1(
reflections, imageset, ice_sel=None, plot_filename=None):
# Implementation of Method 1 (section 2.4.4) of:
# Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
# J. Appl. Cryst. (2006). 39, 112-119
# http://dx.doi.org/10.1107/S0021889805040677
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = reflections['intensity.sum.value']
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
reflections = reflections.select(sel)
intensities = reflections['intensity.sum.value']
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
d_star_cubed = flex.pow(reflections['rlp'].norms(), 3)
step = 2
while len(reflections)/step > 40:
step += 1
order = flex.sort_permutation(d_spacings, reverse=True)
ds3_subset = flex.double()
d_subset = flex.double()
for i in range(len(reflections)//step):
ds3_subset.append(d_star_cubed[order[i*step]])
d_subset.append(d_spacings[order[i*step]])
x = flex.double(range(len(ds3_subset)))
# (i)
# Usually, Pm is the last point, that is, m = n. But m could be smaller than
# n if an unusually high number of spots are detected around a certain
# intermediate resolution. In that case, our search for the image resolution
# does not go outside the spot 'bump;. This is particularly useful when
# ice-rings are present.
slopes = (ds3_subset[1:] - ds3_subset[0])/(x[1:]-x[0])
skip_first = 3
p_m = flex.max_index(slopes[skip_first:]) + 1 + skip_first
# (ii)
from scitbx import matrix
x1 = matrix.col((0, ds3_subset[0]))
x2 = matrix.col((p_m, ds3_subset[p_m]))
gaps = flex.double([0])
v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize()
for i in range(1, p_m):
x0 = matrix.col((i, ds3_subset[i]))
r = x1 - x0
g = abs(v.dot(r))
gaps.append(g)
mv = flex.mean_and_variance(gaps)
s = mv.unweighted_sample_standard_deviation()
# (iii)
p_k = flex.max_index(gaps)
g_k = gaps[p_k]
p_g = p_k
for i in range(p_k+1, len(gaps)):
g_i = gaps[i]
if g_i > (g_k - 0.5 * s):
p_g = i
ds3_g = ds3_subset[p_g]
d_g = d_subset[p_g]
noisiness = 0
n = len(ds3_subset)
for i in range(n-1):
for j in range(i+1, n-1):
if slopes[i] >= slopes[j]:
noisiness += 1
noisiness /= ((n-1)*(n-2)/2)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(range(len(ds3_subset)), ds3_subset)
#ax.set_xlabel('')
ax.set_ylabel('D^-3')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
ax.vlines(p_g, ylim[0], ylim[1], colors='red')
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.savefig(plot_filename)
pyplot.close()
return d_g, noisiness
def paper_test(B, S):
from numpy.random import poisson
from math import exp
background_shape = [1 for i in range(20)]
signal_shape = [1 if i >= 6 and i < 15 else 0 for i in range(20)]
background = [poisson(bb * B,1)[0] for bb in background_shape]
signal = [poisson(ss * S, 1)[0] for ss in signal_shape]
# background = [bb * B for bb in background_shape]
# signal = [ss * S for ss in signal_shape]
total = [b + s for b, s in zip(background, signal)]
# from matplotlib import pylab
# pylab.plot(total)
# pylab.plot(signal)
# pylab.plot(background)
# pylab.show()
total = [0, 1, 0, 0, 0, 0, 3, 1, 3, 3, 6, 6, 4, 1, 4, 0, 2, 0, 1, 1]
total = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]
# plot_prob_for_zero(total, background_shape, signal_shape)
# signal_shape = [exp(-(x - 10.0)**2 / (2*3.0**2)) for x in range(20)]
# signal_shape = [ss / sum(signal_shape) for ss in signal_shape]
# print signal_shape
#plot_valid(background_shape, signal_shape)
B = 155.0 / 296.0
from dials.array_family import flex
from math import log, factorial
V = flex.double(flex.grid(100, 100))
L = flex.double(flex.grid(100, 100))
DB = flex.double(flex.grid(100, 100))
DS = flex.double(flex.grid(100, 100))
P = flex.double(flex.grid(100, 100))
Fb = sum(background_shape)
Fs = sum(signal_shape)
SV = []
MASK = flex.bool(flex.grid(100, 100), False)
for BB in range(100):
for SS in range(100):
B = -5.0 + (BB) / 10.0
S = -5.0 + (SS) / 10.0
# SV.append(S)
VV = 0
LL = 0
DDB = 0
DDS = 0
for i in range(20):
s = signal_shape[i]
b = background_shape[i]
c = total[i]
if B*b + S*s <= 0:
# MASK[BB, SS] = True
LL = 0
if b == 0:
DDB += 0
else:
DDB += 1e7
if s == 0:
DDS += 0
else:
DDS += 1e7
# break
else:
# VV += (b + s)*c / (B*b + S*s)
# LL += c*log(B*b+S*s) - log(factorial(c)) - B*b - S*s
DDB += c*b/(B*b+S*s) - b
DDS += c*s/(B*b+S*s) - s
VV -= (Fb + Fs)
# print B, S, VV
# V[BB, SS] = abs(VV)
L[BB, SS] = LL
DB[BB,SS] = DDB
DS[BB,SS] = DDS
max_ind = flex.max_index(L)
j = max_ind // 100
i = max_ind % 100
print "Approx: ", (j+1) / 20.0, (i+1) / 20.0
print "Min/Max DB: ", flex.min(DB), flex.max(DB)
print "Min/Max DS: ", flex.min(DS), flex.max(DS)
from matplotlib import pylab
# pylab.imshow(flex.log(V).as_numpy_array(), extent=[0.05, 5.05, 5.05, 0.05])
# pylab.plot(SV, V)
# pylab.plot(SV, [0] * 100)
# pylab.show()
im = flex.exp(L).as_numpy_array()
import numpy
# im = numpy.ma.masked_array(im, mask=MASK.as_numpy_array())
# pylab.imshow(im)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.imshow(DB.as_numpy_array(), vmin=-100, vmax=100)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.contour(DB.as_numpy_array(), levels=[0], colors=['red'])
pylab.contour(DS.as_numpy_array(), levels=[0], colors=['black'])
pylab.show()
# im = numpy.ma.masked_array(DB.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# im = numpy.ma.masked_array(DS.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# exit(0)
S1, B1 = value(total, background_shape, signal_shape)
exit(0)
try:
S1, B1 = value(total, background_shape, signal_shape)
print "Result:"
print S1, B1
exit(0)
except Exception, e:
raise e
import sys
import traceback
traceback.print_exc()
from dials.array_family import flex
Fs = sum(signal_shape)
Fb = sum(background_shape)
Rs = flex.double(flex.grid(100, 100))
print "-----"
print B, S
# from matplotlib import pylab
# pylab.plot(total)
# pylab.show()
print background_shape
print signal_shape
print total
from math import exp, factorial, log
minx = -1
miny = -1
minr = 9999
for BB in range(0, 100):
for SS in range(0, 100):
B = -10 + (BB) / 5.0
S = -10 + (SS) / 5.0
L = 0
Fb2 = 0
Fs2 = 0
for i in range(len(total)):
c = total[i]
b = background_shape[i]
s = signal_shape[i]
# P = exp(-(B*b + S*s)) * (B*b+S*s)**c / factorial(c)
# print P
# if P > 0:
# L += log(P)
den = B*b + S*s
num1 = b*c
num2 = s*c
if den != 0:
Fb2 += num1 / den
Fs2 += num2 / den
R = (Fb2 - Fb)**2 + (Fs2 - Fs)**2
if R > 1000:
R = 0
# Rs[BB,SS] = L#R#Fs2 - Fs
Rs[BB,SS] = R#Fs2 - Fs
from matplotlib import pylab
pylab.imshow(flex.log(Rs).as_numpy_array(), extent=[-5,5,5,-5])
pylab.show()
exit(0)
def run_stills_pred_param(self, verbose=False):
if verbose:
print 'Testing derivatives for StillsPredictionParameterisation'
print '========================================================'
# Build a prediction parameterisation for the stills experiment
pred_param = StillsPredictionParameterisation(
self.stills_experiments,
detector_parameterisations=[self.det_param],
beam_parameterisations=[self.s0_param],
xl_orientation_parameterisations=[self.xlo_param],
xl_unit_cell_parameterisations=[self.xluc_param])
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(self.stills_experiments)
ref_predictor(self.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(self.reflections)
fd_grads = self.get_fd_gradients(pred_param, ref_predictor)
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
# compare FD with analytical calculations
if verbose: print "\nParameter {0}: {1}".format(i, fd_grad['name'])
for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]):
if verbose: print name
a = fd_grad[name]
b = an_grad[name]
abs_error = a - b
denom = a + b
fns = five_number_summary(abs_error)
if verbose: print (" summary of absolute errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
assert flex.max(flex.abs(abs_error)) < 0.0003
# largest absolute error found to be about 0.00025 for dY/dp of
# Crystal0g_param_3. Reject outlying absolute errors and test again.
iqr = fns[3] - fns[1]
# skip further stats on errors with an iqr of near zero, e.g. dDeltaPsi_dp
# for detector parameters, which are all equal to zero
if iqr < 1.e-10:
continue
sel1 = abs_error < fns[3] + 1.5 * iqr
sel2 = abs_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(abs_error.select(sel)))
tst_val = abs_error.select(sel)[tst]
n_outliers = sel.count(False)
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"absolute error: {1:9.6f}").format(n_outliers, tst_val)
# largest absolute error now 0.000086 for dX/dp of Beam0Mu2
assert abs(tst_val) < 0.00009
# Completely skip parameters with FD gradients all zero (e.g. gradients of
# DeltaPsi for detector parameters)
sel1 = flex.abs(a) < 1.e-10
if sel1.all_eq(True):
continue
# otherwise calculate normalised errors, by dividing absolute errors by
# the IQR (more stable than relative error calculation)
norm_error = abs_error / iqr
fns = five_number_summary(norm_error)
if verbose: print (" summary of normalised errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
# largest normalised error found to be about 25.7 for dY/dp of
# Crystal0g_param_3.
try:
assert flex.max(flex.abs(norm_error)) < 30
except AssertionError as e:
e.args += ("extreme normalised error value: {0}".format(
flex.max(flex.abs(norm_error))), )
raise e
# Reject outlying normalised errors and test again
iqr = fns[3] - fns[1]
if iqr > 0.:
sel1 = norm_error < fns[3] + 1.5 * iqr
sel2 = norm_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(norm_error.select(sel)))
tst_val = norm_error.select(sel)[tst]
n_outliers = sel.count(False)
# most outliers found for for dY/dp of Crystal0g_param_3 (which had
# largest errors, so no surprise there).
try:
assert n_outliers < 250
except AssertionError as e:
e.args += ("too many outliers rejected: {0}".format(
n_outliers), )
raise e
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"normalised error: {1:9.6f}").format(n_outliers, tst_val)
# largest normalied error now about -4. for dX/dp of Detector0Tau1
assert abs(
tst_val) < 4.5, 'should be about 4 not %s' % tst_val
if verbose: print
return
def paper_test(B, S):
from numpy.random import poisson
from math import exp
background_shape = [1 for i in range(20)]
signal_shape = [1 if i >= 6 and i < 15 else 0 for i in range(20)]
background = [poisson(bb * B, 1)[0] for bb in background_shape]
signal = [poisson(ss * S, 1)[0] for ss in signal_shape]
# background = [bb * B for bb in background_shape]
# signal = [ss * S for ss in signal_shape]
total = [b + s for b, s in zip(background, signal)]
# from matplotlib import pylab
# pylab.plot(total)
# pylab.plot(signal)
# pylab.plot(background)
# pylab.show()
total = [0, 1, 0, 0, 0, 0, 3, 1, 3, 3, 6, 6, 4, 1, 4, 0, 2, 0, 1, 1]
total = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]
# plot_prob_for_zero(total, background_shape, signal_shape)
# signal_shape = [exp(-(x - 10.0)**2 / (2*3.0**2)) for x in range(20)]
# signal_shape = [ss / sum(signal_shape) for ss in signal_shape]
# print signal_shape
# plot_valid(background_shape, signal_shape)
B = 155.0 / 296.0
from dials.array_family import flex
from math import log, factorial
V = flex.double(flex.grid(100, 100))
L = flex.double(flex.grid(100, 100))
DB = flex.double(flex.grid(100, 100))
DS = flex.double(flex.grid(100, 100))
P = flex.double(flex.grid(100, 100))
Fb = sum(background_shape)
Fs = sum(signal_shape)
SV = []
MASK = flex.bool(flex.grid(100, 100), False)
for BB in range(100):
for SS in range(100):
B = -5.0 + (BB) / 10.0
S = -5.0 + (SS) / 10.0
# SV.append(S)
VV = 0
LL = 0
DDB = 0
DDS = 0
for i in range(20):
s = signal_shape[i]
b = background_shape[i]
c = total[i]
if B * b + S * s <= 0:
# MASK[BB, SS] = True
LL = 0
if b == 0:
DDB += 0
else:
DDB += 1e7
if s == 0:
DDS += 0
else:
DDS += 1e7
# break
else:
# VV += (b + s)*c / (B*b + S*s)
# LL += c*log(B*b+S*s) - log(factorial(c)) - B*b - S*s
DDB += c * b / (B * b + S * s) - b
DDS += c * s / (B * b + S * s) - s
VV -= Fb + Fs
# print B, S, VV
# V[BB, SS] = abs(VV)
L[BB, SS] = LL
DB[BB, SS] = DDB
DS[BB, SS] = DDS
max_ind = flex.max_index(L)
j = max_ind // 100
i = max_ind % 100
print("Approx: ", (j + 1) / 20.0, (i + 1) / 20.0)
print("Min/Max DB: ", flex.min(DB), flex.max(DB))
print("Min/Max DS: ", flex.min(DS), flex.max(DS))
from matplotlib import pylab
# pylab.imshow(flex.log(V).as_numpy_array(), extent=[0.05, 5.05, 5.05, 0.05])
# pylab.plot(SV, V)
# pylab.plot(SV, [0] * 100)
# pylab.show()
im = flex.exp(L).as_numpy_array()
import numpy
# im = numpy.ma.masked_array(im, mask=MASK.as_numpy_array())
# pylab.imshow(im)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.imshow(
DB.as_numpy_array(), vmin=-100, vmax=100
) # , extent=[-5.0, 5.0, 5.0, -5.0], origin='lower')
pylab.contour(DB.as_numpy_array(), levels=[0], colors=["red"])
pylab.contour(DS.as_numpy_array(), levels=[0], colors=["black"])
pylab.show()
# im = numpy.ma.masked_array(DB.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# im = numpy.ma.masked_array(DS.as_numpy_array(), mask=MASK.as_numpy_array())
# pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100)
# pylab.show()
# exit(0)
S1, B1 = value(total, background_shape, signal_shape)
exit(0)
try:
S1, B1 = value(total, background_shape, signal_shape)
print("Result:")
print(S1, B1)
exit(0)
except Exception as e:
raise e
import sys
import traceback
traceback.print_exc()
from dials.array_family import flex
Fs = sum(signal_shape)
Fb = sum(background_shape)
Rs = flex.double(flex.grid(100, 100))
print("-----")
print(B, S)
# from matplotlib import pylab
# pylab.plot(total)
# pylab.show()
print(background_shape)
print(signal_shape)
print(total)
from math import exp, factorial, log
minx = -1
miny = -1
minr = 9999
for BB in range(0, 100):
for SS in range(0, 100):
B = -10 + (BB) / 5.0
S = -10 + (SS) / 5.0
L = 0
Fb2 = 0
Fs2 = 0
for i in range(len(total)):
c = total[i]
b = background_shape[i]
s = signal_shape[i]
# P = exp(-(B*b + S*s)) * (B*b+S*s)**c / factorial(c)
# print P
# if P > 0:
# L += log(P)
den = B * b + S * s
num1 = b * c
num2 = s * c
if den != 0:
Fb2 += num1 / den
Fs2 += num2 / den
R = (Fb2 - Fb) ** 2 + (Fs2 - Fs) ** 2
if R > 1000:
R = 0
# Rs[BB,SS] = L#R#Fs2 - Fs
Rs[BB, SS] = R # Fs2 - Fs
from matplotlib import pylab
pylab.imshow(flex.log(Rs).as_numpy_array(), extent=[-5, 5, 5, -5])
pylab.show()
exit(0)
exit(0)
# print S, B, sum(signal), sum(background), S1, B1
return S, B, sum(signal), sum(background), S1, B1
def test_for_reference(self):
from dials.array_family import flex
from math import sqrt, pi
# Integrate
integration = self.experiments[0].profile.fitting_class()(
self.experiments[0])
# Integrate the reference profiles
integration(self.experiments, self.reference)
locator = integration.learner.locate()
# Check the reference profiles and spots are ok
#self.check_profiles(integration.learner)
# Make sure background is zero
profiles = self.reference['rs_shoebox']
eps = 1e-7
for p in profiles:
assert (abs(flex.sum(p.background) - 0) < eps)
print 'OK'
# Only select variances greater than zero
mask = self.reference.get_flags(self.reference.flags.integrated,
all=False)
assert (mask.count(True) > 0)
I_cal = self.reference['intensity.prf.value']
I_var = self.reference['intensity.prf.variance']
B_sim = self.reference['background.sim.a'].as_double()
I_sim = self.reference['intensity.sim'].as_double()
I_exp = self.reference['intensity.exp']
P_cor = self.reference['profile.correlation']
X_pos, Y_pos, Z_pos = self.reference['xyzcal.px'].parts()
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
P_cor = P_cor.select(mask)
max_ind = flex.max_index(flex.abs(I_cal - I_sim))
max_I = I_cal[max_ind]
max_P = self.reference[max_ind]['rs_shoebox'].data
max_C = self.reference[max_ind]['xyzcal.px']
max_S = self.reference[max_ind]['shoebox'].data
ref_ind = locator.index(max_C)
ref_P = locator.profile(ref_ind)
ref_C = locator.coord(ref_ind)
#def f(I):
#mask = flex.bool(flex.grid(9,9,9), False)
#for k in range(9):
#for j in range(9):
#for i in range(9):
#dx = 5 * (i - 4.5) / 4.5
#dy = 5 * (j - 4.5) / 4.5
#dz = 5 * (k - 4.5) / 4.5
#dd = sqrt(dx**2 + dy**2 + dz**2)
#if dd <= 3:
#mask[k,j,i] = True
#mask = mask.as_1d() & (ref_P.as_1d() > 0)
#p = ref_P.as_1d().select(mask)
#c = max_P.as_1d().select(mask)
#return flex.sum((c - I * p)**2 / (I * p))
#ff = []
#for I in range(9500, 11500):
#ff.append(f(I))
#print 'Old I: ', sorted(range(len(ff)), key=lambda x: ff[x])[0] + 9500
#from matplotlib import pylab
#pylab.plot(range(9500, 11500), ff)
#pylab.show()
#def estI(I):
#mask = flex.bool(flex.grid(9,9,9), False)
#for k in range(9):
#for j in range(9):
#for i in range(9):
#dx = 5 * (i - 4.5) / 4.5
#dy = 5 * (j - 4.5) / 4.5
#dz = 5 * (k - 4.5) / 4.5
#dd = sqrt(dx**2 + dy**2 + dz**2)
#if dd <= 3:
#mask[k,j,i] = True
#mask = mask.as_1d() & (ref_P.as_1d() > 0)
#p = ref_P.as_1d().select(mask)
#c = max_P.as_1d().select(mask)
#v = I * p
#return flex.sum(c * p / v) / flex.sum(p*p/v)
#def iterI(I0):
#I = estI(I0)
#print I
#if abs(I - I0) < 1e-3:
#return I
#return iterI(I)
#newI = iterI(10703)#flex.sum(max_P))
#print "New I: ", newI
# Calculate the z score
perc = self.mv3n_tolerance_interval(3 * 3)
Z = (I_cal - I_sim) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print "Z: mean: %f, var: %f, sig: %f" % (Z_mean, Z_var, sqrt(Z_var))
def test_for_reference(self):
from dials.algorithms.integration import ProfileFittingReciprocalSpace
from dials.array_family import flex
from dials.algorithms.shoebox import MaskCode
from dials.algorithms.statistics import \
kolmogorov_smirnov_test_standard_normal
from math import erf, sqrt, pi
from copy import deepcopy
from dials.algorithms.simulation.reciprocal_space import Simulator
from os.path import basename
# Integrate
integration = ProfileFittingReciprocalSpace(
grid_size=4,
threshold=0.00,
frame_interval=100,
n_sigma=5,
mask_n_sigma=3,
sigma_b=0.024 * pi / 180.0,
sigma_m=0.044 * pi / 180.0
)
# Integrate the reference profiles
integration(self.experiment, self.reference)
p = integration.learner.locate().profile(0)
m = integration.learner.locate().mask(0)
locator = integration.learner.locate()
cor = locator.correlations()
for j in range(cor.all()[0]):
print ' '.join([str(cor[j,i]) for i in range(cor.all()[1])])
#exit(0)
#from matplotlib import pylab
#pylab.imshow(cor.as_numpy_array(), interpolation='none', vmin=-1, vmax=1)
#pylab.show()
#n = locator.size()
#for i in range(n):
#c = locator.coord(i)
#p = locator.profile(i)
#vmax = flex.max(p)
#from matplotlib import pylab
#for j in range(9):
#pylab.subplot(3, 3, j+1)
#pylab.imshow(p.as_numpy_array()[j], vmin=0, vmax=vmax,
#interpolation='none')
#pylab.show()
#print "NRef: ", n
#x = []
#y = []
#for i in range(n):
#c = locator.coord(i)
#x.append(c[0])
#y.append(c[1])
#from matplotlib import pylab
#pylab.scatter(x,y)
#pylab.show()
#exit(0)
import numpy
#pmax = flex.max(p)
#scale = 100 / pmax
#print "Scale: ", 100 / pmax
#p = p.as_numpy_array() *100 / pmax
#p = p.astype(numpy.int)
#print p
#print m.as_numpy_array()
# Check the reference profiles and spots are ok
#self.check_profiles(integration.learner)
# Make sure background is zero
profiles = self.reference['rs_shoebox']
eps = 1e-7
for p in profiles:
assert(abs(flex.sum(p.background) - 0) < eps)
print 'OK'
# Only select variances greater than zero
mask = self.reference.get_flags(self.reference.flags.integrated)
I_cal = self.reference['intensity.sum.value']
I_var = self.reference['intensity.sum.variance']
B_sim = self.reference['background.sim'].as_double()
I_sim = self.reference['intensity.sim'].as_double()
I_exp = self.reference['intensity.exp']
P_cor = self.reference['profile.correlation']
X_pos, Y_pos, Z_pos = self.reference['xyzcal.px'].parts()
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
P_cor = P_cor.select(mask)
max_ind = flex.max_index(I_cal)
max_I = I_cal[max_ind]
max_P = self.reference[max_ind]['rs_shoebox'].data
max_C = self.reference[max_ind]['xyzcal.px']
max_S = self.reference[max_ind]['shoebox'].data
min_ind = flex.min_index(P_cor)
min_I = I_cal[min_ind]
min_P = self.reference[min_ind]['rs_shoebox'].data
min_C = self.reference[min_ind]['xyzcal.px']
min_S = self.reference[min_ind]['shoebox'].data
##for k in range(max_S.all()[0]):
#if False:
#for j in range(max_S.all()[1]):
#for i in range(max_S.all()[2]):
#max_S[k,j,i] = 0
#if (abs(i - max_S.all()[2] // 2) < 2 and
#abs(j - max_S.all()[1] // 2) < 2 and
#abs(k - max_S.all()[0] // 2) < 2):
#max_S[k,j,i] = 100
#p = max_P.as_numpy_array() * 100 / flex.max(max_P)
#p = p.astype(numpy.int)
#print p
#from dials.scratch.jmp.misc.test_transform import test_transform
#grid_size = 4
#ndiv = 5
#sigma_b = 0.024 * pi / 180.0
#sigma_m = 0.044 * pi / 180.0
#n_sigma = 4.0
#max_P2 = test_transform(
#self.experiment,
#self.reference[max_ind]['shoebox'],
#self.reference[max_ind]['s1'],
#self.reference[max_ind]['xyzcal.mm'][2],
#grid_size,
#sigma_m,
#sigma_b,
#n_sigma,
#ndiv)
#max_P = max_P2
ref_ind = locator.index(max_C)
ref_P = locator.profile(ref_ind)
ref_C = locator.coord(ref_ind)
print "Max Index: ", max_ind, max_I, flex.sum(max_P), flex.sum(max_S)
print "Coord: ", max_C, "Ref Coord: ", ref_C
print "Min Index: ", min_ind, min_I, flex.sum(min_P), flex.sum(min_S)
print "Coord: ", min_C, "Ref Coord: ", ref_C
#vmax = flex.max(max_P)
#print sum(max_S)
#print sum(max_P)
#from matplotlib import pylab, cm
#for j in range(9):
#pylab.subplot(3, 3, j+1)
#pylab.imshow(max_P.as_numpy_array()[j], vmin=0, vmax=vmax,
#interpolation='none', cmap=cm.Greys_r)
#pylab.show()
#vmax = flex.max(min_P)
#print sum(min_S)
#print sum(min_P)
#from matplotlib import pylab, cm
#for j in range(9):
#pylab.subplot(3, 3, j+1)
#pylab.imshow(min_P.as_numpy_array()[j], vmin=0, vmax=vmax,
#interpolation='none', cmap=cm.Greys_r)
#pylab.show()
#for k in range(max_S.all()[0]):
#print ''
#print 'Slice %d' % k
#for j in range(max_S.all()[1]):
#print ' '.join(["%-4d" % int(max_S[k,j,i]) for i in range(max_S.all()[2])])
print "Testing"
def f(I):
mask = flex.bool(flex.grid(9,9,9), False)
for k in range(9):
for j in range(9):
for i in range(9):
dx = 5 * (i - 4.5) / 4.5
dy = 5 * (j - 4.5) / 4.5
dz = 5 * (k - 4.5) / 4.5
dd = sqrt(dx**2 + dy**2 + dz**2)
if dd <= 3:
mask[k,j,i] = True
mask = mask.as_1d() & (ref_P.as_1d() > 0)
p = ref_P.as_1d().select(mask)
c = max_P.as_1d().select(mask)
return flex.sum((c - I * p)**2 / (I * p))
def df(I):
mask = flex.bool(flex.grid(9,9,9), False)
for k in range(9):
for j in range(9):
for i in range(9):
dx = 5 * (i - 4.5) / 4.5
dy = 5 * (j - 4.5) / 4.5
dz = 5 * (k - 4.5) / 4.5
dd = sqrt(dx**2 + dy**2 + dz**2)
if dd <= 3:
mask[k,j,i] = True
mask = mask.as_1d() & (ref_P.as_1d() > 0)
p = ref_P.as_1d().select(mask)
c = max_P.as_1d().select(mask)
b = 0
return flex.sum(p) - flex.sum(c*c / (I*I*p))
#return flex.sum(p - p*c*c / ((b + I*p)**2))
#return flex.sum(3*p*p + (c*c*p*p - 4*b*p*p) / ((b + I*p)**2))
#return flex.sum(p - c*c / (I*I*p))
#return flex.sum(p * (-c+p*I)*(c+p*I)/((p*I)**2))
def d2f(I):
mask = flex.bool(flex.grid(9,9,9), False)
for k in range(9):
for j in range(9):
for i in range(9):
dx = 5 * (i - 4.5) / 4.5
dy = 5 * (j - 4.5) / 4.5
dz = 5 * (k - 4.5) / 4.5
dd = sqrt(dx**2 + dy**2 + dz**2)
if dd <= 3:
mask[k,j,i] = True
mask = mask.as_1d() & (ref_P.as_1d() > 0)
p = ref_P.as_1d().select(mask)
c = max_P.as_1d().select(mask)
return flex.sum(2*c*c*p*p / (p*I)**3)
I = 10703#flex.sum(max_P)
mask = ref_P.as_1d() > 0
p = ref_P.as_1d().select(mask)
c = max_P.as_1d().select(mask)
for i in range(10):
I = I - df(I) / d2f(I)
#v = I*p
#I = flex.sum(c * p / v) / flex.sum(p*p / v)
print I
from math import log
ff = []
for I in range(9500, 11500):
ff.append(f(I))
print sorted(range(len(ff)), key=lambda x: ff[x])[0] + 9500
from matplotlib import pylab
pylab.plot(range(9500,11500), ff)
pylab.show()
#exit(0)
#I = 10000
#print flex.sum((max_P - I * ref_P)**2) / flex.sum(I * ref_P)
#I = 10100
#print flex.sum((max_P - I * ref_P)**2) / flex.sum(I * ref_P)
#exit(0)
print flex.sum(self.reference[0]['rs_shoebox'].data)
print I_cal[0]
# Calculate the z score
perc = self.mv3n_tolerance_interval(3*3)
Z = (I_cal - I_sim) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print "Z: mean: %f, var: %f, sig: %f" % (Z_mean, Z_var, sqrt(Z_var))
print len(I_cal)
from matplotlib import pylab
from mpl_toolkits.mplot3d import Axes3D
#fig = pylab.figure()
#ax = fig.add_subplot(111, projection='3d')
#ax.scatter(X_pos, Y_pos, P_cor)
#pylab.scatter(X_pos, P_cor)
#pylab.scatter(Y_pos, P_cor)
#pylab.scatter(Z_pos, P_cor)
#pylab.hist(P_cor,100)
#pylab.scatter(P_cor, (I_cal - I_exp) / I_exp)
pylab.hist(Z, 100)
#pylab.hist(I_cal,100)
#pylab.hist(I_cal - I_sim, 100)
pylab.show()
def determine_fraction_new_cutoff(self, fraction_new, cutoff): imax = flex.max_index(fraction_new) isel = (fraction_new < cutoff).iselection() return isel[(isel > imax).iselection()[0]]
def test_stills_pred_param(tc):
print("Testing derivatives for StillsPredictionParameterisation")
print("========================================================")
# Build a prediction parameterisation for the stills experiment
pred_param = StillsPredictionParameterisation(
tc.stills_experiments,
detector_parameterisations=[tc.det_param],
beam_parameterisations=[tc.s0_param],
xl_orientation_parameterisations=[tc.xlo_param],
xl_unit_cell_parameterisations=[tc.xluc_param],
)
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
ref_predictor = StillsExperimentsPredictor(tc.stills_experiments)
ref_predictor(tc.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(tc.reflections)
fd_grads = tc.get_fd_gradients(pred_param, ref_predictor)
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
# compare FD with analytical calculations
print(f"\nParameter {i}: {fd_grad['name']}")
for name in ["dX_dp", "dY_dp", "dDeltaPsi_dp"]:
print(name)
a = fd_grad[name]
b = an_grad[name]
abs_error = a - b
fns = five_number_summary(abs_error)
print((" summary of absolute errors: %9.6f %9.6f %9.6f " +
"%9.6f %9.6f") % fns)
assert flex.max(flex.abs(abs_error)) < 0.0003
# largest absolute error found to be about 0.00025 for dY/dp of
# Crystal0g_param_3. Reject outlying absolute errors and test again.
iqr = fns[3] - fns[1]
# skip further stats on errors with an iqr of near zero, e.g. dDeltaPsi_dp
# for detector parameters, which are all equal to zero
if iqr < 1.0e-10:
continue
sel1 = abs_error < fns[3] + 1.5 * iqr
sel2 = abs_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(abs_error.select(sel)))
tst_val = abs_error.select(sel)[tst]
n_outliers = sel.count(False)
print((" {0} outliers rejected, leaving greatest " +
"absolute error: {1:9.6f}").format(n_outliers, tst_val))
# largest absolute error now 0.000086 for dX/dp of Beam0Mu2
assert abs(tst_val) < 0.00009
# Completely skip parameters with FD gradients all zero (e.g. gradients of
# DeltaPsi for detector parameters)
sel1 = flex.abs(a) < 1.0e-10
if sel1.all_eq(True):
continue
# otherwise calculate normalised errors, by dividing absolute errors by
# the IQR (more stable than relative error calculation)
norm_error = abs_error / iqr
fns = five_number_summary(norm_error)
print((" summary of normalised errors: %9.6f %9.6f %9.6f " +
"%9.6f %9.6f") % fns)
# largest normalised error found to be about 25.7 for dY/dp of
# Crystal0g_param_3.
try:
assert flex.max(flex.abs(norm_error)) < 30
except AssertionError as e:
e.args += (
f"extreme normalised error value: {flex.max(flex.abs(norm_error))}",
)
raise e
# Reject outlying normalised errors and test again
iqr = fns[3] - fns[1]
if iqr > 0.0:
sel1 = norm_error < fns[3] + 1.5 * iqr
sel2 = norm_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(norm_error.select(sel)))
tst_val = norm_error.select(sel)[tst]
n_outliers = sel.count(False)
# most outliers found for for dY/dp of Crystal0g_param_3 (which had
# largest errors, so no surprise there).
try:
assert n_outliers < 250
except AssertionError as e:
e.args += (f"too many outliers rejected: {n_outliers}", )
raise e
print(
(" {0} outliers rejected, leaving greatest " +
"normalised error: {1:9.6f}").format(n_outliers, tst_val))
# largest normalied error now about -4. for dX/dp of Detector0Tau1
assert abs(tst_val) < 6, f"should be < 6, not {tst_val}"
def test_for_reference(self):
from dials.array_family import flex
from math import sqrt, pi
# Integrate
integration = self.experiments[0].profile.fitting_class()(self.experiments[0])
# Integrate the reference profiles
integration(self.experiments, self.reference)
locator = integration.learner.locate()
# Check the reference profiles and spots are ok
#self.check_profiles(integration.learner)
# Make sure background is zero
profiles = self.reference['rs_shoebox']
eps = 1e-7
for p in profiles:
assert(abs(flex.sum(p.background) - 0) < eps)
print 'OK'
# Only select variances greater than zero
mask = self.reference.get_flags(self.reference.flags.integrated, all=False)
assert(mask.count(True) > 0)
I_cal = self.reference['intensity.prf.value']
I_var = self.reference['intensity.prf.variance']
B_sim = self.reference['background.sim.a'].as_double()
I_sim = self.reference['intensity.sim'].as_double()
I_exp = self.reference['intensity.exp']
P_cor = self.reference['profile.correlation']
X_pos, Y_pos, Z_pos = self.reference['xyzcal.px'].parts()
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
P_cor = P_cor.select(mask)
max_ind = flex.max_index(flex.abs(I_cal-I_sim))
max_I = I_cal[max_ind]
max_P = self.reference[max_ind]['rs_shoebox'].data
max_C = self.reference[max_ind]['xyzcal.px']
max_S = self.reference[max_ind]['shoebox'].data
ref_ind = locator.index(max_C)
ref_P = locator.profile(ref_ind)
ref_C = locator.coord(ref_ind)
#def f(I):
#mask = flex.bool(flex.grid(9,9,9), False)
#for k in range(9):
#for j in range(9):
#for i in range(9):
#dx = 5 * (i - 4.5) / 4.5
#dy = 5 * (j - 4.5) / 4.5
#dz = 5 * (k - 4.5) / 4.5
#dd = sqrt(dx**2 + dy**2 + dz**2)
#if dd <= 3:
#mask[k,j,i] = True
#mask = mask.as_1d() & (ref_P.as_1d() > 0)
#p = ref_P.as_1d().select(mask)
#c = max_P.as_1d().select(mask)
#return flex.sum((c - I * p)**2 / (I * p))
#ff = []
#for I in range(9500, 11500):
#ff.append(f(I))
#print 'Old I: ', sorted(range(len(ff)), key=lambda x: ff[x])[0] + 9500
#from matplotlib import pylab
#pylab.plot(range(9500, 11500), ff)
#pylab.show()
#def estI(I):
#mask = flex.bool(flex.grid(9,9,9), False)
#for k in range(9):
#for j in range(9):
#for i in range(9):
#dx = 5 * (i - 4.5) / 4.5
#dy = 5 * (j - 4.5) / 4.5
#dz = 5 * (k - 4.5) / 4.5
#dd = sqrt(dx**2 + dy**2 + dz**2)
#if dd <= 3:
#mask[k,j,i] = True
#mask = mask.as_1d() & (ref_P.as_1d() > 0)
#p = ref_P.as_1d().select(mask)
#c = max_P.as_1d().select(mask)
#v = I * p
#return flex.sum(c * p / v) / flex.sum(p*p/v)
#def iterI(I0):
#I = estI(I0)
#print I
#if abs(I - I0) < 1e-3:
#return I
#return iterI(I)
#newI = iterI(10703)#flex.sum(max_P))
#print "New I: ", newI
# Calculate the z score
perc = self.mv3n_tolerance_interval(3*3)
Z = (I_cal - I_sim) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print "Z: mean: %f, var: %f, sig: %f" % (Z_mean, Z_var, sqrt(Z_var))