def dotproduct(): a=[3,-4,1] b=[0,5,2] a1=flex.double(a) b1=flex.double(b) angle = acos(a1.dot(b1)/(a1.norm()*b1.norm()))*180/pi return angle
def compare_times(max_n_power=8):
from scitbx.linalg import lapack_dsyev
mt = flex.mersenne_twister(seed=0)
show_tab_header = True
tfmt = "%5.2f"
for n_power in xrange(5,max_n_power+1):
n = 2**n_power
l = mt.random_double(size=n*(n+1)//2) * 2 - 1
a = l.matrix_packed_l_as_symmetric()
aes = a.deep_copy()
ala = [a.deep_copy(), a.deep_copy()]
wla = [flex.double(n, -1e100), flex.double(n, -1e100)]
t0 = time.time()
es = eigensystem.real_symmetric(aes)
tab = [n, tfmt % (time.time() - t0)]
for i,use_fortran in enumerate([False, True]):
t0 = time.time()
info = lapack_dsyev(
jobz="V", uplo="U", a=ala[i], w=wla[i], use_fortran=use_fortran)
if (info == 99):
tab.append(" --- ")
else:
assert info == 0
tab.append(tfmt % (time.time() - t0))
assert approx_equal(list(reversed(es.values())), wla[i])
if (show_tab_header):
print " time [s] eigenvalues"
print " n es fem for min max"
show_tab_header = False
tab.extend([es.values()[-1], es.values()[0]])
print "%3d %s %s %s [%6.2f %6.2f]" % tuple(tab)
def __init__(self, frames, z, zeta, sweep):
from scitbx import simplex
from scitbx.array_family import flex
import numpy
self.L = Likelihood(FractionOfObservedIntensity(frames, z, zeta, sweep.get_scan()))
x = 0.1 + numpy.arange(1000) / 2000.0
l = [self.L(xx) for xx in x]
from matplotlib import pylab
pylab.plot(x, l)
pylab.show()
# print 1/0
startA = 0.3
startB = 0.4
# startA = 0.2*3.14159 / 180
# startB = 0.3*3.14159 / 180
print "Start: ", startA, startB
starting_simplex=[flex.double([startA]), flex.double([startB])]
# for ii in range(2):
# starting_simplex.append(flex.double([start]))#flex.random_double(1))
self.optimizer = simplex.simplex_opt(
1, matrix=starting_simplex, evaluator=self, tolerance=1e-7)
def exercise(lbfgs_impl, n=100, m=5, iprint=[1, 0]):
assert n % 2 == 0
x = flex.double(n)
g = flex.double(n)
diag = flex.double(n)
diagco = 0
eps = 1.0e-5
xtol = 1.0e-16
for j in xrange(0, n, 2):
x[j] = -1.2
x[j+1] = 1.
size_w = n*(2*m+1)+2*m
w = flex.double(size_w)
iflag = 0
icall = 0
while 1:
f = 0.
for j in xrange(0, n, 2):
t1 = 1.e0 - x[j]
t2 = 1.e1 * (x[j+1] - x[j] * x[j])
g[j+1] = 2.e1 * t2
g[j] = -2.e0 * (x[j] * g[j+1] + t1)
f = f + t1 * t1 + t2 * t2
iflag = lbfgs_impl(
n=n, m=m, x=x, f=f, g=g, diagco=diagco, diag=diag,
iprint=iprint, eps=eps, xtol=xtol, w=w, iflag=iflag)
if (iflag <= 0): break
icall += 1
# We allow at most 2000 evaluations of f and g
if (icall > 2000): break
def find_pixel_nearest_neighbour(image, mask, reflections):
from annlib_ext import AnnAdaptor
from scitbx.array_family import flex
from math import sqrt
import numpy
# Get the predicted coords
pred_xyz = []
for r in reflections:
x = r.image_coord_px[0] * 0.172
y = r.image_coord_px[1] * 0.172
z = r.frame_number * 0.2
pred_xyz.append((x, y, z))
# Create the KD Tree
ann = AnnAdaptor(flex.double(pred_xyz).as_1d(), 3)
pixel_xyz = []
ind = numpy.where(mask != 0)
z = ind[0] * 0.2
y = ind[1] * 0.172
x = ind[2] * 0.172
ann.query(flex.double(zip(x, y, z)).as_1d())
# for i in xrange(len(ann.nn)):
# print "Neighbor of {0}, index {1} distance {2}".format(
# obs_xyz[i], ann.nn[i], sqrt(ann.distances[i]))
owner = numpy.zeros(shape=mask.shape, dtype=numpy.int32)
owner[ind] = ann.nn.as_numpy_array()
return owner
def tst_scan(self):
from dxtbx.serialize import scan
from dxtbx.model import Scan
from scitbx.array_family import flex
s1 = Scan((1, 3), (1.0, 0.2), flex.double([0.1, 0.1, 0.1]), flex.double([0.1, 0.2, 0.3]))
d = scan.to_dict(s1)
s2 = scan.from_dict(d)
assert(d['image_range'] == (1, 3))
assert(d['oscillation'] == (1.0, 0.2))
assert(d['exposure_time'] == [0.1, 0.1, 0.1])
assert(d['epochs'] == [0.1, 0.2, 0.3])
assert(s1 == s2)
# Test with a template and partial dictionary
d2 = { 'exposure_time' : [0.2, 0.2, 0.2] }
s3 = scan.from_dict(d2, d)
assert(s3.get_image_range() == (1, 3))
assert(s3.get_oscillation() == (1.0, 0.2))
assert(list(s3.get_exposure_times()) == [0.2, 0.2, 0.2])
assert(list(s3.get_epochs()) == [0.1, 0.2, 0.3])
assert(s2 != s3)
# Test with a partial epoch
d3 = { 'image_range' : (1, 10), 'epochs' : [0.1, 0.2] }
s4 = scan.from_dict(d3, d)
assert(abs(s4.get_epochs()[2] - 0.3) < 1e-7)
assert(abs(s4.get_epochs()[9] - 1.0) < 1e-7)
print 'OK'
def find_nearest_neighbour(obs_xyz, reflections):
from annlib_ext import AnnAdaptor
from scitbx.array_family import flex
from math import sqrt
# Get the predicted coords
pred_xyz = []
for r in reflections:
x = r.image_coord_px[0]# * 0.172
y = r.image_coord_px[1]# * 0.172
z = r.frame_number# * 0.2
pred_xyz.append((x, y, z))
# Create the KD Tree
ann = AnnAdaptor(flex.double(pred_xyz).as_1d(), 3)
# obs_xyz2 = []
# for x, y, z in obs_xyz:
# x = x * 0.172
# y = y * 0.172
# z = z * 0.2
# obs_xyz2.append((x, y, z))
ann.query(flex.double(obs_xyz).as_1d())
# for i in xrange(len(ann.nn)):
# print "Neighbor of {0}, index {1} distance {2}".format(
# obs_xyz[i], ann.nn[i], sqrt(ann.distances[i]))
return ann.nn, ann.distances
def radial_transverse_analysis(self):
# very time consuming; should be recoded (C++?)
SIGN = -1.
PIXEL_SZ = 0.11 # mm/pixel
zunit = col([0.,0.,1.])
self.frame_del_radial = {}
self.frame_del_azimuthal = {}
for x in xrange(len(self.frame_id)):
frame_id = self.frame_id[x]
frame_param_no = self.frame_id_to_param_no[frame_id]
if frame_param_no >= self.n_refined_frames: continue
if not self.frame_del_radial.has_key(frame_id):
self.frame_del_radial[frame_id] = flex.double()
self.frame_del_azimuthal[frame_id] = flex.double()
correction_vector = col([self.correction_vector_x[x],self.correction_vector_y[x],0.])
position_vector = col([self.spotfx[x] -
SIGN * self.FRAMES["detector_origin_x_refined"][frame_param_no]/PIXEL_SZ,
self.spotfy[x] -
SIGN * self.FRAMES["detector_origin_y_refined"][frame_param_no]/PIXEL_SZ,
0.])
position_unit = position_vector.normalize()
transverse_unit = position_unit.cross(zunit)
self.frame_del_radial[frame_id].append(correction_vector.dot(position_unit))
self.frame_del_azimuthal[frame_id].append(correction_vector.dot(transverse_unit))
def get_symop_correlation_coefficients(miller_array, use_binning=False):
from copy import deepcopy
from scitbx.array_family import flex
from cctbx import miller
corr_coeffs = flex.double()
n_refs = flex.int()
space_group = miller_array.space_group()
for smx in space_group.smx():
reindexed_array = miller_array.change_basis(sgtbx.change_of_basis_op(smx))
intensity, intensity_rdx = reindexed_array.common_sets(miller_array)
if use_binning:
intensity.use_binning_of(miller_array)
intensity_rdx.use_binning_of(miller_array)
cc = intensity.correlation(intensity_rdx, use_binning=use_binning)
corr_coeffs.append(
flex.mean_weighted(
flex.double(i for i in cc.data if i is not None),
flex.double(j for i, j in zip(cc.data, cc.binner.counts()) if i is not None),
)
)
else:
corr_coeffs.append(intensity.correlation(intensity_rdx, use_binning=use_binning).coefficient())
n_refs.append(intensity.size())
return corr_coeffs, n_refs
def run(args):
assert len(args) == 0
have_cma_es = libtbx.env.has_module("cma_es")
if not have_cma_es:
print "Skipping some tests: cma_es module not available or not configured."
print
names = ["easom", "rosenbrock", "ackley", "rastrigin"]
for name in names:
print "****", name, "****"
if name == "easom":
start = flex.double([0.0, 0.0])
else:
start = flex.double([4, 4])
for ii in range(1):
test_lbfgs(name)
if have_cma_es:
test_cma_es(name)
test_differential_evolution(name)
test_cross_entropy(name)
test_simplex(name)
test_dssa(name)
print
print
from libtbx.utils import format_cpu_times
print format_cpu_times()
def compute_functional_gradients_and_curvatures(self):
self.prepare_for_step()
# observation terms
blocks = self._target.split_matches_into_blocks(nproc=self._nproc)
if self._nproc > 1:
task_results = easy_mp.parallel_map(
func=self._target.compute_functional_gradients_and_curvatures,
iterable=blocks,
processes=self._nproc,
method="multiprocessing",
# preserve_exception_message=True
)
else:
task_results = [self._target.compute_functional_gradients_and_curvatures(block) for block in blocks]
# reduce blockwise results
flist, glist, clist = zip(*task_results)
glist = zip(*glist)
clist = zip(*clist)
f = sum(flist)
g = [sum(g) for g in glist]
c = [sum(c) for c in clist]
# restraints terms
restraints = self._target.compute_restraints_functional_gradients_and_curvatures()
if restraints:
f += restraints[0]
g = [a + b for a, b in zip(g, restraints[1])]
c = [a + b for a, b in zip(c, restraints[2])]
return f, flex.double(g), flex.double(c)
def plot_combo(self, histogram, gaussians,
window_title=None, title=None,
log_scale=False, normalise=False, save_image=False, interpretation=None):
from matplotlib import pyplot
#from xfel.command_line.view_pixel_histograms import hist_outline
slots = flex.double(histogram.astype(np.float64))
if normalise:
normalisation = (flex.sum(slots) + histogram.n_out_of_slot_range()) / 1e5
print "normalising by factor: ", normalisation
slots /= normalisation
slot_centers = flex.double(xrange(self.work_params.first_slot_value,
self.work_params.first_slot_value + len(histogram)))
bins, data = hist_outline(slot_width=1,slots=slots,slot_centers=slot_centers)
if log_scale:
data.set_selected(data == 0, 0.1) # otherwise lines don't get drawn when we have some empty bins
pyplot.yscale("log")
pyplot.plot(bins, data, '-k', linewidth=2)
pyplot.plot(bins, data/1000., '-k', linewidth=2)
pyplot.suptitle(title)
#data_min = min([slot.low_cutoff for slot in histogram.slot_infos() if slot.n > 0])
#data_max = max([slot.low_cutoff for slot in histogram.slot_infos() if slot.n > 0])
#pyplot.xlim(data_min, data_max+histogram.slot_width())
pyplot.xlim(-50, 150)
pyplot.ylim(-10, 40)
x = slot_centers
for g in gaussians:
print "Height %7.2f mean %4.1f sigma %3.1f"%(g.params)
pyplot.plot(x, g(x), linewidth=2)
if interpretation is not None:
interpretation.plot_multiphoton_fit(pyplot)
interpretation.plot_quality(pyplot)
pyplot.show()
def _find_nearest_neighbours(self, observed, predicted):
'''Find the nearest predicted spot to the observed spot.
Params:
observed The observed reflections
predicted The predicted reflections
Returns:
(nearest neighbours, distance)
'''
from annlib_ext import AnnAdaptor
from scitbx.array_family import flex
from math import sqrt
# Get the predicted coordinates
predicted_xyz = []
for r in predicted:
x, y = r.image_coord_px
z = r.frame_number
predicted_xyz.append((x, y, z))
# Create the KD Tree
ann = AnnAdaptor(flex.double(predicted_xyz).as_1d(), 3)
# Get the observed coordinates
observed_xyz = [r.centroid_position for r in observed]
# Query to find all the nearest neighbours
ann.query(flex.double(observed_xyz).as_1d())
# Return the nearest neighbours and distances
return ann.nn, flex.sqrt(ann.distances)
def callback_after_step(O, minimizer):
xk = O.prev_x
xl = O.x
gk = O.fgh.g(xk)
gl = O.fgh.g(xl)
def check(bk, hk):
hl = bfgs.h_update(hk, xl-xk, gl-gk)
bl = bfgs.b_update(bk, xl-xk, gl-gk)
assert approx_equal(matrix.sqr(hl).inverse(), bl)
es = eigensystem.real_symmetric(bl)
assert es.values().all_gt(0)
assert bfgs.h0_scaling(sk=xl-xk, yk=gl-gk) > 0
#
bk = flex.double([[1,0],[0,1]])
hk = bk
check(bk, hk)
#
bk = O.fgh.h(xk)
es = eigensystem.real_symmetric(bk)
if (es.values().all_gt(0)):
hk = bk.deep_copy()
hk.matrix_inversion_in_place()
check(bk, hk)
#
h0 = flex.double([[0.9,0.1],[-0.2,0.8]])
hg_tlr = bfgs.hg_two_loop_recursion(memory=O.memory, hk0=h0, gk=gl)
h_upd = h0
for m in O.memory:
h_upd = bfgs.h_update(hk=h_upd, sk=m.s, yk=m.y)
hg_upd = h_upd.matrix_multiply(gl)
assert approx_equal(hg_tlr, hg_upd)
#
O.memory.append(bfgs.memory_element(s=xl-xk, y=gl-gk))
O.prev_x = O.x.deep_copy()
def exercise_two_loop_recursion(fgh): x0 = flex.double([3.0, -4.0]) g0 = fgh.g(x0) h0 = flex.double([[1,0],[0,1]]) memory = [] hg0 = bfgs.hg_two_loop_recursion(memory, hk0=h0, gk=g0) assert approx_equal(hg0, h0.matrix_multiply(g0)) # x1 = x0 - 1/3 * hg0 g1 = fgh.g(x1) h1 = bfgs.h_update(hk=h0, sk=x1-x0, yk=g1-g0) memory.append(bfgs.memory_element(s=x1-x0, y=g1-g0)) hg1 = bfgs.hg_two_loop_recursion(memory, hk0=h0, gk=g1) assert approx_equal(hg1, h1.matrix_multiply(g1)) # x2 = x1 - 1/5 * hg1 g2 = fgh.g(x2) h2 = bfgs.h_update(hk=h1, sk=x2-x1, yk=g2-g1) memory.append(bfgs.memory_element(s=x2-x1, y=g2-g1)) hg2 = bfgs.hg_two_loop_recursion(memory, hk0=h0, gk=g2) assert approx_equal(hg2, h2.matrix_multiply(g2)) # x3 = x2 - 3/8 * hg2 g3 = fgh.g(x3) h3 = bfgs.h_update(hk=h2, sk=x3-x2, yk=g3-g2) memory.append(bfgs.memory_element(s=x3-x2, y=g3-g2)) hg3 = bfgs.hg_two_loop_recursion(memory, hk0=h0, gk=g3) assert approx_equal(hg3, h3.matrix_multiply(g3))
def compute_functional_and_gradients(self):
from scitbx.array_family import flex
import math
self.model_mean_x = flex.double(len(self.observed_x))
self.model_mean_y = flex.double(len(self.observed_x))
for x in xrange(6):
selection = (self.master_groups==x)
self.model_mean_x.set_selected(selection, self.x[2*x])
self.model_mean_y.set_selected(selection, self.x[2*x+1])
delx = self.observed_x - self.model_mean_x
dely = self.observed_y - self.model_mean_y
delrsq = delx*delx + dely*dely
f = flex.sum(delrsq)
gradients = flex.double([0.]*12)
for x in xrange(6):
selection = (self.master_groups==x)
gradients[2*x] = -2. * flex.sum( delx.select(selection) )
gradients[2*x+1]=-2. * flex.sum( dely.select(selection) )
if self.verbose:
print "Functional ",math.sqrt(flex.mean(delrsq))
self.count_iterations += 1
return f,gradients
def exercise_linear_normal_equations():
py_eqs = [ ( 1, (-1, 0, 0), 1),
( 2, ( 2, -1, 0), 3),
(-1, ( 0, 2, 1), 2),
(-2, ( 0, 1, 0), -2),
]
eqs_0 = normal_eqns.linear_ls(3)
for b, a, w in py_eqs:
eqs_0.add_equation(right_hand_side=b,
design_matrix_row=flex.double(a),
weight=w)
eqs_1 = normal_eqns.linear_ls(3)
b = flex.double()
w = flex.double()
a = sparse.matrix(len(py_eqs), 3)
for i, (b_, a_, w_) in enumerate(py_eqs):
b.append(b_)
w.append(w_)
for j in xrange(3):
if a_[j]: a[i, j] = a_[j]
eqs_1.add_equations(right_hand_side=b, design_matrix=a, weights=w)
assert approx_equal(
eqs_0.normal_matrix_packed_u(), eqs_1.normal_matrix_packed_u(), eps=1e-15)
assert approx_equal(
eqs_0.right_hand_side(), eqs_1.right_hand_side(), eps=1e-15)
assert approx_equal(
list(eqs_0.normal_matrix_packed_u()), [ 13, -6, 0, 9, 4, 2 ], eps=1e-15)
assert approx_equal(
list(eqs_0.right_hand_side()), [ 11, -6, -2 ], eps=1e-15)
def interpolate(x, y, half_window=10):
perm = flex.sort_permutation(x)
x = x.select(perm)
y = y.select(perm)
x_all = flex.double()
y_all = flex.double()
for i in range(x.size()):
x_all.append(x[i])
y_all.append(y[i])
if i < x.size()-1 and (x[i+1] - x[i]) > 1:
window_left = min(half_window, i)
window_right = min(half_window, x.size() - i)
x_ = x[i-window_left:i+window_right]
y_ = y[i-window_left:i+window_right]
from scitbx.math import curve_fitting
# fit a 2nd order polynomial through the missing points
polynomial = curve_fitting.univariate_polynomial(1, 1)
fit = curve_fitting.lbfgs_minimiser([polynomial], x_,y_).functions[0]
missing_x = flex.double(range(int(x[i]+1), int(x[i+1])))
x_all.extend(missing_x)
y_all.extend(fit(missing_x))
perm = flex.sort_permutation(x_all)
x_all = x_all.select(perm)
y_all = y_all.select(perm)
return x_all, y_all
def __init__ (self, all_results) : self._copy_constructor(all_results[0]) self.res_type = all_results[0].res_type self.rama_type = [ r.rama_type for r in all_results ] self.phi = flex.double([ r.phi for r in all_results ]) self.psi = flex.double([ r.psi for r in all_results ]) self.score = flex.double([ r.score for r in all_results ])
def exercise_01 () :
"""
Sanity check - don't crash when mean intensity for a bin is zero.
"""
xrs = random_structure.xray_structure(
unit_cell=(50,50,50,90,90,90),
space_group_symbol="P1",
n_scatterers=1200,
elements="random")
fc = abs(xrs.structure_factors(d_min=1.5).f_calc())
fc = fc.set_observation_type_xray_amplitude()
cs = fc.complete_set(d_min=1.4)
ls = cs.lone_set(other=fc)
f_zero = ls.array(data=flex.double(ls.size(), 0))
f_zero.set_observation_type_xray_amplitude()
fc = fc.concatenate(other=f_zero)
sigf = flex.double(fc.size(), 0.1) + (fc.data() * 0.03)
fc = fc.customized_copy(sigmas=sigf)
try :
fc_fc = french_wilson.french_wilson_scale(miller_array=fc, log=null_out())
except Sorry :
pass
else :
raise Exception_expected
ic = fc.f_as_f_sq().set_observation_type_xray_intensity()
fc_fc = french_wilson.french_wilson_scale(miller_array=ic, log=null_out())
def populate_bin_to_individual_k_mask_linear_interpolation(self, k_mask_bin):
assert len(k_mask_bin) == len(self.cores_and_selections)
def linear_interpolation(x1,x2,y1,y2):
k=0
if(x1!=x2): k=(y2-y1)/(x2-x1)
b = y1-k*x1
return k,b
result1 = flex.double(self.f_obs.size(), -1)
result2 = flex.double(self.f_obs.size(), -1)
result = flex.double(self.f_obs.size(), -1)
for i, cas in enumerate(self.cores_and_selections):
selection, zzz, zzz, zzz = cas
x1,x2 = self.ss_bin_values[i][0], self.ss_bin_values[i][1]
y1 = k_mask_bin[i]
if(i==len(k_mask_bin)-1):
y2 = k_mask_bin[i-1]
else:
y2 = k_mask_bin[i+1]
k,b = linear_interpolation(x1=x1,x2=x2,y1=y1,y2=y2)
bulk_solvent.set_to_linear_interpolated(self.ss,k,b,selection,result1)
result2.set_selected(selection, y1)
r1 = self.try_scale(k_mask = result1, selection=selection) # XXX inefficient
r2 = self.try_scale(k_mask = result2, selection=selection) # XXX inefficient
if(r1<r2):
bulk_solvent.set_to_linear_interpolated(self.ss,k,b,selection,result)
else:
result.set_selected(selection, y1)
#print list(result)
assert (result < 0).count(True) == 0
return result
def get_map_stats_for_atoms (self, atoms) :
from cctbx import maptbx
from scitbx.array_family import flex
sites_cart = flex.vec3_double()
sites_cart_nonH = flex.vec3_double()
values_2fofc = flex.double()
values_fofc = flex.double()
for atom in atoms :
sites_cart.append(atom.xyz)
if (not atom.element.strip() in ["H","D"]) : #XXX trap: neutrons?
sites_cart_nonH.append(atom.xyz)
site_frac = self.unit_cell.fractionalize(atom.xyz)
values_2fofc.append(self.f_map.eight_point_interpolation(site_frac))
values_fofc.append(self.diff_map.eight_point_interpolation(site_frac))
if (len(sites_cart_nonH) == 0) :
return None
sel = maptbx.grid_indices_around_sites(
unit_cell=self.unit_cell,
fft_n_real=self.f_map.focus(),
fft_m_real=self.f_map.all(),
sites_cart=sites_cart,
site_radii=get_atom_radii(atoms, self.atom_radius))
f_map_sel = self.f_map.select(sel)
model_map_sel = self.model_map.select(sel)
diff_map_sel = self.diff_map.select(sel)
cc = flex.linear_correlation(x=f_map_sel, y=model_map_sel).coefficient()
return group_args(cc=cc,
mean_2fofc=flex.mean(values_2fofc),
mean_fofc=flex.mean(values_fofc))
def test_cma_es_rosebrock_n(M=10):
def funct(x,y):
result = 0
for xx,yy in zip(x,y):
result+=100.0*((yy-xx*xx)**2.0) + (1-xx)**2.0
return result
N=M*2
x = flex.double(N,10.0)
sd = flex.double(N,3.0)
m = cma_es(N,x,sd)
while ( not m.converged() ):
# sample population
p = m.sample_population()
pop_size = p.accessor().all()[0]
# update objective function
v = flex.double(pop_size)
for ii in range(pop_size):
vector = p[(ii*N):(ii*N + N)]
x = vector[0:M]
y = vector[M:]
v[ii] = funct(x,y)
m.update_distribution(v)
print list(m.get_result())
print flex.min(v)
print
x_final = m.get_result()
print list(x_final)
def __init__(self,rawdata,projection_vector,spotfinder_spot,verbose=False):
# projection vector is either the radial or azimuthal unit vector
# at a specific Bragg spot position
model_center = col((spotfinder_spot.ctr_mass_x(),spotfinder_spot.ctr_mass_y()))
px_x,px_y = project_2d_response_onto_line(projection_vector)
point_projections = flex.double()
pixel_values = flex.double()
for point in spotfinder_spot.bodypixels:
point_projection = (col((point.x,point.y)) - model_center).dot( projection_vector )
point_projections.append(point_projection)
pxval = rawdata[(point.x,point.y)]
if verbose:
print "point_projection",point_projection,
print "signal",pxval
pixel_values.append( pxval )
Lmin = flex.min(point_projections)
Lmax = flex.max(point_projections)
#print "Range %6.2f"%(Lmax-Lmin)
Rmin = round(Lmin-2.0,1)
Rmax = round(Lmax+2.0,1)
#print "Range %6.2f"%(Rmax-Rmin)
def histogram_bin (j) : return int(10.*(j-Rmin)) # bin units of 1/10 pixel
histo_x = flex.double((int(10*(Rmax-Rmin))))
histo_y = flex.double(len(histo_x))
for ihis in xrange(len(histo_x)): histo_x[ihis] = Rmin + 0.1*ihis
for ipp, point_projection in enumerate(point_projections):
value = pixel_values[ipp]
for isample in xrange(len(px_x)):
histo_y[int(10*(point_projection + px_x[isample] - Rmin))] += value * px_y[isample]
self.histo_x = histo_x
self.histo_y = histo_y
def calculate_gold2d(self):
from scitbx.array_family import flex
from scitbx import matrix
r_tot = 0.0
c_tot = 0.0
d_tot = 0.0
for (r, c), d in zip(self.points2d, self.pixels2d):
r_tot += d * r
c_tot += d * c
d_tot += d
self.gold2d = matrix.col((r_tot / d_tot, c_tot / d_tot))
_r, _c = self.gold2d
r_tot = 0.0
c_tot = 0.0
for (r, c), d in zip(self.points2d, self.pixels2d):
r_tot += d * (r - _r) ** 2
c_tot += d * (c - _c) ** 2
_sr = r_tot / d_tot
_sc = c_tot / d_tot
self.gold2dvar = matrix.col((_sr, _sc))
pixel_x, pixel_y = zip(*self.points2d)
xc = flex.mean_and_variance(flex.double(pixel_x), self.pixels2d.as_1d())
yc = flex.mean_and_variance(flex.double(pixel_y), self.pixels2d.as_1d())
self.gold2dubvar = matrix.col((xc.gsl_stats_wvariance(),
yc.gsl_stats_wvariance()))
def __init__(self,xarr,yarr,lo_deriv1=None,hi_deriv1=None):
from scitbx.array_family import flex
assert len(xarr)==len(yarr)
tempu = flex.double(len(xarr))
self.deriv2 = flex.double(len(xarr))
if lo_deriv1==None:
self.deriv2[0] = 0.0; tempu[0] = 0.0
else:
self.deriv2[0] = -0.5;
tempu[0] = (3./(xarr[1]-xarr[0]))*((yarr[1]-yarr[0])/(xarr[1]-xarr[0])-lo_deriv1)
for i in xrange(1,len(xarr)-1):
sig = (xarr[i]-xarr[i-1])/(xarr[i+1]-xarr[i-1])
p = sig*self.deriv2[i-1]+2.0
self.deriv2[i] = (sig-1.0)/p
tempu[i] = (yarr[i+1]-yarr[i])/(xarr[i+1]-xarr[i]) - \
(yarr[i]-yarr[i-1])/(xarr[i]-xarr[i-1])
tempu[i] = (6.*tempu[i] / (xarr[i+1]-xarr[i-1]) - sig *tempu[i-1])/p
NX = len(xarr)
if hi_deriv1==None:
qn = 0.0; un = 0.0
else:
qn = 0.5;
un = (3./(xarr[NX-1]-xarr[NX-2]))* \
(hi_deriv1-(yarr[NX-1]-yarr[NX-2])/(xarr[NX-1]-xarr[NX-2]))
self.deriv2[NX-1] = (un-qn*tempu[NX-2])/(qn*self.deriv2[NX-2]+1.0)
for k in xrange(NX-2,-1,-1):
self.deriv2[k]=self.deriv2[k]*self.deriv2[k+1]+tempu[k]
self.xarr = xarr
self.yarr = yarr
def accumulate_dose(imagesets):
from scitbx.array_family import flex
epochs = flex.double()
exposure_times = flex.double()
for imageset in imagesets:
scan = imageset.get_scan()
# workaround for read_all_image_headers=False option
epochs.extend(flex.double(
e if e else float(os.stat(imageset.get_path(j)).st_mtime)
for (j, e) in enumerate(scan.get_epochs())))
exposure_times.extend(scan.get_exposure_times())
perm = flex.sort_permutation(epochs)
epochs = epochs.select(perm)
exposure_times = exposure_times.select(perm)
from libtbx.containers import OrderedDict
integrated_dose = OrderedDict()
total = 0.0
for e, t in zip(epochs, exposure_times):
integrated_dose[e] = total + 0.5 * t
total += t
return integrated_dose
def exercise_c_grid_conversions(verbose=0):
if (verbose): print 'Checking flex->ref_c_grid conversions'
a = flex.double(flex.grid((2, 3)))
assert a.accessor() == rt.use_const_ref_c_grid_2(a)
assert a.all() == rt.use_const_ref_c_grid_2(a).all()
assert a.accessor().all() == rt.use_const_ref_c_grid_padded_2(a).all()
assert a.accessor().all() == rt.use_const_ref_c_grid_padded_2(a).focus()
a = flex.double(flex.grid((2, 3, 4)))
assert a.accessor() == rt.use_const_ref_c_grid_3(a)
assert a.all() == rt.use_const_ref_c_grid_3(a).all()
assert a.accessor().all() == rt.use_const_ref_c_grid_padded_3(a).all()
assert a.accessor().all() == rt.use_const_ref_c_grid_padded_3(a).focus()
a = flex.double(flex.grid((2, 3, 5)).set_focus((2, 3, 4)))
assert a.accessor().all() == rt.use_const_ref_c_grid_padded_3(a).all()
assert a.accessor().focus() == rt.use_const_ref_c_grid_padded_3(a).focus()
assert a.is_0_based()
assert a.is_padded()
b = flex.double(flex.grid((1,2,3), (4,6,8)))
assert not b.is_0_based()
assert not b.is_padded()
for c in [a,b]:
try: rt.use_const_ref_c_grid_3(c)
except KeyboardInterrupt: raise
except Exception, e:
assert str(e).startswith("Python argument types in")
else: raise RuntimeError("Boost.Python.ArgumentError expected.")
def set_ladp(xray_structure, axes_and_atoms_i_seqs, value, depth,
enable_recursion=True):
sc = (math.pi/180)
sites_cart = xray_structure.sites_cart()
scatterers = xray_structure.scatterers()
all_selections = flex.size_t()
u_carts = flex.sym_mat3_double(sites_cart.size(), [0,0,0,0,0,0])
for i_seq, aaa_ in enumerate(axes_and_atoms_i_seqs):
if(enable_recursion): query = i_seq >= depth
else: query = i_seq == depth
if(query): all_selections.extend(aaa_[0][1])
for i_seq, r in enumerate(axes_and_atoms_i_seqs):
if(enable_recursion): query = i_seq >= depth
else: query = i_seq == depth
if(query):
for aaai in r:
G1 = flex.double(sites_cart[aaai[0][0]])
G2 = flex.double(sites_cart[aaai[0][1]])
g = G2-G1
dg = math.sqrt(g[0]**2+g[1]**2+g[2]**2)
lx,ly,lz = g/dg
l = [lx,ly,lz]
L = matrix.sqr((lx**2,lx*ly,lx*lz, lx*ly,ly**2,ly*lz, lx*lz,ly*lz,lz**2))
for i_seq_moving in aaai[1]:
site_cart = sites_cart[i_seq_moving]
delta = flex.double(site_cart) - G1
A = matrix.sqr(
(0,delta[2],-delta[1], -delta[2],0,delta[0], delta[1],-delta[0],0))
u_cart = (value * A * L * A.transpose() * sc).as_sym_mat3()
check_u_cart(axis = l, u_cart = u_cart)
scatterers[i_seq_moving].flags.set_use_u_aniso(True)
u_carts[i_seq_moving] = list(flex.double(u_carts[i_seq_moving]) +
flex.double(u_cart))
xray_structure.set_u_cart(u_cart = u_carts, selection = all_selections)
return xray_structure
def cc_model_map (self, selection=None, radius=1.5) :
"""
Calculate the correlation coefficient for the current model (in terms of
F(calc) from the xray structure) and the target map, calculated at atomic
positions rather than grid points. This will be much
less accurate than the CC calculated in the original crystal environment,
with full F(model) including bulk solvent correction.
"""
from scitbx.array_family import flex
if (selection is None) :
selection = self.selection_in_box
fcalc = self.box.xray_structure_box.structure_factors(d_min=self.d_min).f_calc()
fc_fft_map = fcalc.fft_map(resolution_factor=self.resolution_factor)
fc_map = fc_fft_map.apply_sigma_scaling().real_map_unpadded()
sites_selected = self.get_selected_sites(selection, hydrogens=False)
assert (len(sites_selected) > 0)
fc_values = flex.double()
map_values = flex.double()
unit_cell = self.box.xray_structure_box.unit_cell()
for site in sites_selected :
site_frac = unit_cell.fractionalize(site)
fc_values.append(fc_map.tricubic_interpolation(site_frac))
map_values.append(self.target_map_box.tricubic_interpolation(site_frac))
return flex.linear_correlation(
x=map_values,
y=fc_values).coefficient()
def score_by_volume(self, reverse=False):
# smaller volume = better
volumes = flex.double(s.crystal.get_unit_cell().volume()
for s in self.all_solutions)
score = flex.log(volumes) / math.log(2)
return self.volume_weight * (score - flex.min(score))
def get_items():
from six.moves import cPickle as pickle
with open("dump_coarse_file.pickle","r") as inp:
while 1:
yield pickle.load(inp)
if __name__=="__main__":
pdb_lines = open("/net/dials/raid1/sauter/LS49/1m2a.pdb","r").read()
from LS49.sim.util_fmodel import gen_fmodel
GF = gen_fmodel(resolution=3.0,pdb_text=pdb_lines,algorithm="fft",wavelength=1.7)
CB_OP_C_P = GF.xray_structure.change_of_basis_op_to_primitive_setting() # from C to P
print(str(CB_OP_C_P))
icount=0
from scitbx.array_family import flex
angles=flex.double()
for stuff in get_items():
#print stuff
icount+=1
print("Iteration",icount)
from cctbx import crystal_orientation
# work up the ground truth from header
header_Ori = crystal_orientation.crystal_orientation(stuff["coarse"], crystal_orientation.basis_type.direct)
header_Ori.show(legend="header_Ori")
C2_ground_truth = header_Ori.change_basis(CB_OP_C_P.inverse())
C2_ground_truth.show(legend="coarse_ground_truth")
# work up the ground truth from mosaic ensemble
def __init__(self, raw_pixels_lst):
if not isinstance(raw_pixels_lst[0], flex.double):
raw_pixels_lst = [flex.double(data) for data in raw_pixels_lst]
self.raw_pixels_panels = tuple(raw_pixels_lst)
panel_shape = self.raw_pixels_panels[0].focus()
self.mask = tuple([flex.bool(flex.grid(panel_shape), True)]*len(self.raw_pixels_panels) ) # TODO: use nanoBragg internal mask
def run(args, out=sys.stdout, validated=False):
show_citation(out=out)
if (len(args) == 0):
master_phil.show(out=out)
print('\nUsage: phenix.map_comparison <CCP4> <CCP4>\n',\
' phenix.map_comparison <CCP4> <MTZ> mtz_label_1=<label>\n',\
' phenix.map_comparison <MTZ 1> mtz_label_1=<label 1> <MTZ 2> mtz_label_2=<label 2>\n', file=out)
sys.exit()
# process arguments
params = None
input_attributes = ['map_1', 'mtz_1', 'map_2', 'mtz_2']
try: # automatic parsing
params = phil.process_command_line_with_files(
args=args, master_phil=master_phil).work.extract()
except Exception: # map_file_def only handles one map phil
from libtbx.phil.command_line import argument_interpreter
arg_int = argument_interpreter(master_phil=master_phil)
command_line_args = list()
map_files = list()
for arg in args:
if (os.path.isfile(arg)):
map_files.append(arg)
else:
command_line_args.append(arg_int.process(arg))
params = master_phil.fetch(sources=command_line_args).extract()
# check if more files are necessary
n_defined = 0
for attribute in input_attributes:
if (getattr(params.input, attribute) is not None):
n_defined += 1
# matches files to phil scope, stops once there is sufficient data
for map_file in map_files:
if (n_defined < 2):
current_map = file_reader.any_file(map_file)
if (current_map.file_type == 'ccp4_map'):
n_defined += 1
if (params.input.map_1 is None):
params.input.map_1 = map_file
elif (params.input.map_2 is None):
params.input.map_2 = map_file
elif (current_map.file_type == 'hkl'):
n_defined += 1
if (params.input.mtz_1 is None):
params.input.mtz_1 = map_file
elif (params.input.mtz_2 is None):
params.input.mtz_2 = map_file
else:
print('WARNING: only the first two files are used', file=out)
break
# validate arguments (GUI sets validated to true, no need to run again)
assert (params is not None)
if (not validated):
validate_params(params)
# ---------------------------------------------------------------------------
# check if maps need to be generated from mtz
n_maps = 0
maps = list()
map_names = list()
for attribute in input_attributes:
filename = getattr(params.input, attribute)
if (filename is not None):
map_names.append(filename)
current_map = file_reader.any_file(filename)
maps.append(current_map)
if (current_map.file_type == 'ccp4_map'):
n_maps += 1
# construct maps, if necessary
crystal_gridding = None
m1 = None
m2 = None
# 1 map, 1 mtz file
if (n_maps == 1):
for current_map in maps:
if (current_map.file_type == 'ccp4_map'):
uc = current_map.file_object.unit_cell()
sg_info = space_group_info(
current_map.file_object.space_group_number)
n_real = current_map.file_object.unit_cell_grid
crystal_gridding = maptbx.crystal_gridding(
uc, space_group_info=sg_info, pre_determined_n_real=n_real)
m1 = current_map.file_object.map_data()
if (crystal_gridding is not None):
label = None
for attribute in [('mtz_1', 'mtz_label_1'),
('mtz_2', 'mtz_label_2')]:
filename = getattr(params.input, attribute[0])
label = getattr(params.input, attribute[1])
if ((filename is not None) and (label is not None)):
break
# labels will match currently open mtz file
for current_map in maps:
if (current_map.file_type == 'hkl'):
m2 = miller.fft_map(
crystal_gridding=crystal_gridding,
fourier_coefficients=current_map.file_server.
get_miller_array(
label)).apply_sigma_scaling().real_map_unpadded()
else:
raise Sorry('Gridding is not defined.')
# 2 mtz files
elif (n_maps == 0):
crystal_symmetry = get_crystal_symmetry(maps[0])
d_min = min(get_d_min(maps[0]), get_d_min(maps[1]))
crystal_gridding = maptbx.crystal_gridding(
crystal_symmetry.unit_cell(),
d_min=d_min,
resolution_factor=params.options.resolution_factor,
space_group_info=crystal_symmetry.space_group_info())
m1 = miller.fft_map(
crystal_gridding=crystal_gridding,
fourier_coefficients=maps[0].file_server.get_miller_array(
params.input.mtz_label_1)).apply_sigma_scaling(
).real_map_unpadded()
m2 = miller.fft_map(
crystal_gridding=crystal_gridding,
fourier_coefficients=maps[1].file_server.get_miller_array(
params.input.mtz_label_2)).apply_sigma_scaling(
).real_map_unpadded()
# 2 maps
else:
m1 = maps[0].file_object.map_data()
m2 = maps[1].file_object.map_data()
if params.options.shift_origin:
m1.shift_origin()
m2.shift_origin()
# ---------------------------------------------------------------------------
# analyze maps
assert ((m1 is not None) and (m2 is not None))
results = dict()
results['map_files'] = None
results['map_statistics'] = None
results['cc_input_maps'] = None
results['cc_quantile'] = None
results['cc_peaks'] = None
results['discrepancies'] = None
results['map_histograms'] = None
if params.options.contour_to_match:
match_contour_level(m1=m1,
m2=m2,
contour_to_match=params.options.contour_to_match,
results=results)
print ("Contour level map 1: %.4f (fractional volume of %.3f ) " %(
params.options.contour_to_match,results['v1']),\
"\nmatches enclosed volume of "+\
"contour level map 2 of : %.4f (volume %.3f )" %(
results['matching_contour'],results['v2']),file=out)
return results
# show general statistics
s1 = maptbx.more_statistics(m1)
s2 = maptbx.more_statistics(m2)
show_overall_statistics(out=out, s=s1, header="Map 1 (%s):" % map_names[0])
show_overall_statistics(out=out, s=s2, header="Map 2 (%s):" % map_names[1])
cc_input_maps = flex.linear_correlation(x=m1.as_1d(),
y=m2.as_1d()).coefficient()
print("CC, input maps: %6.4f" % cc_input_maps, file=out)
# compute CCpeak
cc_peaks = list()
m1_he = maptbx.volume_scale(map=m1, n_bins=10000).map_data()
m2_he = maptbx.volume_scale(map=m2, n_bins=10000).map_data()
cc_quantile = flex.linear_correlation(x=m1_he.as_1d(),
y=m2_he.as_1d()).coefficient()
print("CC, quantile rank-scaled (histogram equalized) maps: %6.4f" % \
cc_quantile, file=out)
print("Peak correlation:", file=out)
print(" cutoff CCpeak", file=out)
cutoffs = [i / 100.
for i in range(1, 90)] + [i / 1000 for i in range(900, 1000)]
for cutoff in cutoffs:
cc_peak = maptbx.cc_peak(map_1=m1_he, map_2=m2_he, cutoff=cutoff)
print(" %3.2f %7.4f" % (cutoff, cc_peak), file=out)
cc_peaks.append((cutoff, cc_peak))
# compute discrepancy function (D-function)
discrepancies = list()
cutoffs = flex.double(cutoffs)
df = maptbx.discrepancy_function(map_1=m1_he, map_2=m2_he, cutoffs=cutoffs)
print("Discrepancy function:", file=out)
print(" cutoff D", file=out)
for c, d in zip(cutoffs, df):
print(" %3.2f %7.4f" % (c, d), file=out)
discrepancies.append((c, d))
# compute and output histograms
h1 = maptbx.histogram(map=m1, n_bins=10000)
h2 = maptbx.histogram(map=m2, n_bins=10000)
print("Map histograms:", file=out)
print("Map 1 (%s) Map 2 (%s)"%\
(params.input.map_1,params.input.map_2), file=out)
print("(map_value,cdf,frequency) <> (map_value,cdf,frequency)", file=out)
for a1, c1, v1, a2, c2, v2 in zip(h1.arguments(), h1.c_values(),
h1.values(), h2.arguments(),
h2.c_values(), h2.values()):
print("(%9.5f %9.5f %9.5f) <> (%9.5f %9.5f %9.5f)"%\
(a1,c1,v1, a2,c2,v2), file=out)
# store results
s1_dict = create_statistics_dict(s=s1)
s2_dict = create_statistics_dict(s=s2)
results = dict()
inputs = list()
for attribute in input_attributes:
filename = getattr(params.input, attribute)
if (filename is not None):
inputs.append(filename)
assert (len(inputs) == 2)
results['map_files'] = inputs
results['map_statistics'] = (s1_dict, s2_dict)
results['cc_input_maps'] = cc_input_maps
results['cc_quantile'] = cc_quantile
results['cc_peaks'] = cc_peaks
results['discrepancies'] = discrepancies
# TODO, verify h1,h2 are not dicts, e.g. .values is py2/3 compat. I assume it is here
results['map_histograms'] = ((h1.arguments(), h1.c_values(), h1.values()),
(h2.arguments(), h2.c_values(), h2.values()))
return results
from __future__ import division, print_function
import pickle
import os
os.environ["JSON_GLOB"] = "null"
os.environ["PICKLE_GLOB"] = "null"
os.environ["USE_POSTREFINE"] = "null"
os.environ["MODEL_MODE"] = "null"
from LS49.work2_for_aca_lsq.abc_background import fit_roi_multichannel # implicit import
from scitbx.array_family import flex
abc_glob_dials_refine = "/global/cscratch1/sd/nksauter/proj-paper1/work/abc_coverage_dials_refine/abcX%06d.pickle"
abc_glob_pixel_refine = "/global/cscratch1/sd/nksauter/proj-paper1/work/abc_coverage_pixel_refine/abcX%06d.pickle"
abc_glob_coarse_ground_truth = "/global/cscratch1/sd/nksauter/proj-paper1/work/abc_coverage_coarse_ground_truth/abcX%06d.pickle"
from scitbx.matrix import col
deltafast = flex.double()
deltaslow = flex.double()
for key in range(10000):
print(key)
try:
pixel = pickle.load(open(abc_glob_dials_refine % key, "rb"))
dials = pickle.load(open(abc_glob_coarse_ground_truth % key, "rb"))
except IOError:
continue
if not len(pixel) == len(dials):
# only one image in 10000 fails
continue
for ispot in range(len(pixel)):
dials_roi = dials[ispot].roi
pixel_roi = pixel[ispot].roi
from __future__ import print_function from dials.algorithms.integration import add_2d from scitbx.array_family import flex from dials.scratch.luiso_s import model_2d nrow = ncol = 25 ref2d_01 = model_2d(nrow, ncol, 3, 6, 0.2, 85, 0.5) ref2d_02 = model_2d(nrow + 1, ncol + 1, 3, 6, 0.8, 85, 0.5) descr = flex.double(flex.grid(1, 3)) descr[0, 0] = 12.5 descr[0, 1] = 12.5 descr[0, 2] = 2.5 # tmp_ref01 = ref2d_01[:,:] # tmp_ref02 = ref2d_02[:,:] print(id(ref2d_01)) print(id(ref2d_02)) print() sumation = add_2d(descr, ref2d_01, ref2d_02) # sumation = add_2d(descr, tmp_ref01, tmp_ref02) print() print(id(ref2d_01)) print(id(ref2d_02)) print(id(sumation)) from matplotlib import pyplot as plt plt.imshow(ref2d_01.as_numpy_array(), interpolation="nearest") plt.show() plt.imshow(ref2d_02.as_numpy_array(), interpolation="nearest") plt.show()
def score_by_fraction_indexed(self, reverse=False):
# more indexed reflections = better
fraction_indexed = flex.double(s.fraction_indexed
for s in self.all_solutions)
score = flex.log(fraction_indexed) / math.log(2)
return self.n_indexed_weight * (-score + flex.max(score))
def print_scaling_summary(script):
"""Log summary information after scaling."""
logger.info(print_scaling_model_error_summary(script.experiments))
valid_ranges = get_valid_image_ranges(script.experiments)
image_ranges = get_image_ranges(script.experiments)
msg = []
for (img, valid, refl) in zip(image_ranges, valid_ranges, script.reflections):
if valid:
if len(valid) > 1 or valid[0][0] != img[0] or valid[-1][1] != img[1]:
msg.append(
"Excluded images for experiment id: %s, image range: %s, limited range: %s"
% (
refl.experiment_identifiers().keys()[0],
list(img),
list(valid),
)
)
if msg:
msg = ["Summary of image ranges removed:"] + msg
logger.info("\n".join(msg))
# report on partiality of dataset
partials = flex.double()
for r in script.reflections:
if "partiality" in r:
partials.extend(r["partiality"])
not_full_sel = partials < 0.99
not_zero_sel = partials > 0.01
gt_half = partials > 0.5
lt_half = partials < 0.5
partial_gt_half_sel = not_full_sel & gt_half
partial_lt_half_sel = not_zero_sel & lt_half
logger.info("Summary of dataset partialities")
header = ["Partiality (p)", "n_refl"]
rows = [
["all reflections", str(partials.size())],
["p > 0.99", str(not_full_sel.count(False))],
["0.5 < p < 0.99", str(partial_gt_half_sel.count(True))],
["0.01 < p < 0.5", str(partial_lt_half_sel.count(True))],
["p < 0.01", str(not_zero_sel.count(False))],
]
logger.info(tabulate(rows, header))
logger.info(
"""
Reflections below a partiality_cutoff of %s are not considered for any
part of the scaling analysis or for the reporting of merging statistics.
Additionally, if applicable, only reflections with a min_partiality > %s
were considered for use when refining the scaling model.
""",
script.params.cut_data.partiality_cutoff,
script.params.reflection_selection.min_partiality,
)
stats = script.merging_statistics_result
if stats:
anom_stats, cut_stats, cut_anom_stats = (None, None, None)
if not script.scaled_miller_array.space_group().is_centric():
anom_stats = script.anom_merging_statistics_result
logger.info(make_merging_statistics_summary(stats))
try:
d_min = resolution_cc_half(stats, limit=0.3).d_min
except RuntimeError as e:
logger.debug(f"Resolution fit failed: {e}")
else:
max_current_res = stats.bins[-1].d_min
if d_min and d_min - max_current_res > 0.005:
logger.info(
"Resolution limit suggested from CC"
+ "\u00BD"
+ " fit (limit CC"
+ "\u00BD"
+ "=0.3): %.2f",
d_min,
)
try:
cut_stats, cut_anom_stats = merging_stats_from_scaled_array(
script.scaled_miller_array.resolution_filter(d_min=d_min),
script.params.output.merging.nbins,
script.params.output.use_internal_variance,
)
except DialsMergingStatisticsError:
pass
else:
if script.scaled_miller_array.space_group().is_centric():
cut_anom_stats = None
logger.info(table_1_summary(stats, anom_stats, cut_stats, cut_anom_stats))
def initialize(pfh, initial_estimates): pfh.x_0 = flex.double(initial_estimates) pfh.restart() pfh.counter = 0
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
None,
"--as_pickle",
"-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files"
).option(
None,
"--raw_data",
"-R",
action="store_true",
default=False,
dest="raw_data",
help=
"Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
).option(
None,
"--background_pickle",
"-B",
default=None,
dest="background_pickle",
help=""
).option(
None,
"--config",
"-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file"
).option(
None,
"--experiment",
"-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)"
).option(
None,
"--run",
"-r",
type="int",
default=None,
dest="run",
help="run number"
).option(
None,
"--address",
"-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)"
).option(
None,
"--detz_offset",
"-d",
type="float",
default=None,
dest="detz_offset",
help=
"offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
).option(
None,
"--outputdir",
"-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files"
).option(
None,
"--averagebase",
"-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help=
"Path to output average image without extension. String substitution allowed"
).option(
None,
"--stddevbase",
"-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help=
"Path to output standard deviation image without extension. String substitution allowed"
).option(
None,
"--maxbase",
"-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help=
"Path to output maximum projection image without extension. String substitution allowed"
).option(
None,
"--numevents",
"-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all"
).option(
None,
"--skipevents",
"-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0"
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--pickle-optical-metrology",
"-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help=
"If writing pickle files, use the optical metrology in the experiment's calib directory"
).option(
None,
"--bin_size",
"-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size"
).option(
None,
"--override_beam_x",
"-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate"
).option(
None,
"--override_beam_y",
"-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate"
).option(
None,
"--calib_dir",
"-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory"
).option(
None,
"--pickle_calib_dir",
"-P",
type="string",
default=None,
dest="pickle_calib_dir",
metavar="PATH",
help=
"pickle calibration directory specification. Replaces --calib_dir functionality."
).option(
None,
"--xtc_dir",
"-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory"
).option(
None,
"--use_ffb",
"-f",
action="store_true",
default=False,
dest="use_ffb",
help=
"Use the fast feedback filesystem at LCLS. Only for the active experiment!"
).option(
None,
"--gain_mask_value",
"-g",
type="float",
default=None,
dest="gain_mask_value",
help=
"Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
).option(
None,
"--min",
None,
action="store_true",
default=False,
dest="do_minimum_projection",
help="Output a minimum projection"
).option(
None,
"--minpath",
None,
type="string",
default="{experiment!l}_min-r{run:04d}",
dest="minpath",
metavar="PATH",
help=
"Path to output minimum image without extension. String substitution allowed"
)).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxint
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:idx" % (command_line.options.experiment,
command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s" % command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
command_line.options.experiment[0:3],
command_line.options.experiment)
if command_line.options.calib_dir is not None:
psana.setOption('psana.calib-dir', command_line.options.calib_dir)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)' % address)
nevent = np.array([0.])
if command_line.options.background_pickle is not None:
background = easy_pickle.load(
command_line.options.background_pickle)['DATA'].as_numpy_array()
for run in ds.runs():
runnumber = run.run()
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
# list of all events
if command_line.options.skipevents > 0:
print "Skipping first %d events" % command_line.options.skipevents
elif "Rayonix" in command_line.options.address:
print "Skipping first image in the Rayonix detector" # Shuttering issue
command_line.options.skipevents = 1
times = run.times()[command_line.options.skipevents:]
nevents = min(len(times), maxevents)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in range(nevents) if (i + rank) % size == 0]
for i in range(len(mytimes)):
if i % 10 == 0:
print 'Rank', rank, 'processing event', rank * len(
mytimes) + i, ', ', i, 'of', len(mytimes)
evt = run.event(mytimes[i])
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
data = evt.get(psana.Camera.FrameV1, src)
if data is None:
print "No data"
continue
data = data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
if command_line.options.raw_data:
data = get_psana_corrected_data(psana_det,
evt,
use_default=False,
dark=False,
common_mode=None,
apply_gain_mask=False,
per_pixel_gain=False)
else:
if command_line.options.gain_mask_value is None:
data = get_psana_corrected_data(psana_det,
evt,
use_default=True)
else:
data = get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=True,
gain_mask_value=command_line.options.
gain_mask_value,
per_pixel_gain=False)
if data is None:
print "No data"
continue
if command_line.options.background_pickle is not None:
data -= background
if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
distance = np.array([0.0])
wavelength = np.array([1.0])
else:
d = cspad_tbx.env_distance(address, run.env(),
command_line.options.detz_offset)
if d is None:
print "No distance, using distance", command_line.options.detz_offset
assert command_line.options.detz_offset is not None
if 'distance' not in locals():
distance = np.array([command_line.options.detz_offset])
else:
distance += command_line.options.detz_offset
else:
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print "No wavelength"
if 'wavelength' not in locals():
wavelength = np.array([1.0])
else:
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print "No timestamp, skipping shot"
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1] / 1000)
else:
timestamp = np.array([t[0] + (t[1] / 1000)])
if 'sum' in locals():
sum += data
else:
sum = np.array(data, copy=True)
if 'sumsq' in locals():
sumsq += data * data
else:
sumsq = data * data
if 'maximum' in locals():
maximum = np.maximum(maximum, data)
else:
maximum = np.array(data, copy=True)
if command_line.options.do_minimum_projection:
if 'minimum' in locals():
minimum = np.minimum(minimum, data)
else:
minimum = np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print "Synchronizing rank", rank
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent, totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum, sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq, sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum, maxall, op=MPI.MAX)
if command_line.options.do_minimum_projection:
minall = np.zeros(maximum.shape).astype(minimum.dtype)
comm.Reduce(minimum, minall, op=MPI.MIN)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength, waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance, distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp, timeall)
if rank == 0:
if size > 1:
print "Synchronized"
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [
cspad_tbx.pathsubst(command_line.options.averagepath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
ds.env())
]
if command_line.options.do_minimum_projection:
dest_paths.append(
cspad_tbx.pathsubst(command_line.options.minpath + extension,
evt, ds.env()))
dest_paths = [
os.path.join(command_line.options.outputdir, path)
for path in dest_paths
]
if 'Rayonix' in command_line.options.address:
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
command_line.options.bin_size)
beam_center = [
command_line.options.override_beam_x,
command_line.options.override_beam_y
]
active_areas = flex.int([0, 0, mean.focus()[1], mean.focus()[0]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip(all_data, dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
all_data = [mean, stddev, maxall]
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
d = cspad_tbx.dpack(address=old_style_address,
data=flex.double(data),
distance=distance,
pixel_size=0.1,
timestamp=timestamp,
wavelength=wavelength)
print "Saving", path
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(
evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
old_style_address,
reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.pickle_calib_dir is not None:
metro_path = command_line.options.pickle_calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
minall = flex.double(minall.astype(np.float64))
else:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
minall = flex.double(minall.astype(np.float64))
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print "Saving", path
d = cspad_tbx.dpack(active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print "Saving", path
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx,
data,
distance,
wavelength,
timestamp,
address,
round_to_int=False)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def test_tanh_fit():
x = flex.double(range(0, 100)) * 0.01
f = curve_fitting.tanh(0.5, 1.5)
yo = f(x)
yf = resolution_analysis.tanh_fit(x, yo)
assert yo == pytest.approx(yf, abs=1e-5)
def estimate_global_threshold(image, mask=None, plot=False):
n_above_threshold = flex.size_t()
threshold = flex.double()
for i in range(1, 20):
g = 1.5**i
g = int(g)
n_above_threshold.append((image > g).count(True))
threshold.append(g)
# Find the elbow point of the curve, in the same manner as that used by
# distl spotfinder for resolution method 1 (Zhang et al 2006).
# See also dials/algorithms/spot_finding/per_image_analysis.py
x = threshold.as_double()
y = n_above_threshold.as_double()
slopes = (y[-1] - y[:-1]) / (x[-1] - x[:-1])
p_m = flex.min_index(slopes)
x1 = matrix.col((x[p_m], y[p_m]))
x2 = matrix.col((x[-1], y[-1]))
gaps = flex.double()
v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize()
for i in range(p_m, len(x)):
x0 = matrix.col((x[i], y[i]))
r = x1 - x0
g = abs(v.dot(r))
gaps.append(g)
p_g = flex.max_index(gaps)
x_g_ = x[p_g + p_m]
y_g_ = y[p_g + p_m]
# more conservative, choose point 2 left of the elbow point
x_g = x[p_g + p_m - 2]
# y_g = y[p_g + p_m - 2]
if plot:
from matplotlib import pyplot
pyplot.figure(figsize=(16, 12))
pyplot.scatter(threshold, n_above_threshold, marker="+")
# for i in range(len(threshold)-1):
# pyplot.plot([threshold[i], threshold[-1]],
# [n_above_threshold[i], n_above_threshold[-1]])
# for i in range(1, len(threshold)):
# pyplot.plot([threshold[0], threshold[i]],
# [n_above_threshold[0], n_above_threshold[i]])
pyplot.plot([x_g, x_g], pyplot.ylim())
pyplot.plot(
[threshold[p_m], threshold[-1]],
[n_above_threshold[p_m], n_above_threshold[-1]],
)
pyplot.plot([x_g_, threshold[-1]], [y_g_, n_above_threshold[-1]])
pyplot.xlabel("Threshold")
pyplot.ylabel("Number of pixels above threshold")
pyplot.savefig("global_threshold.png")
pyplot.clf()
return x_g
def _index_prepare(self):
"""Prepare to do autoindexing - in XDS terms this will mean
calling xycorr, init and colspot on the input images."""
# decide on images to work with
Debug.write("XDS INDEX PREPARE:")
Debug.write("Wavelength: %.6f" % self.get_wavelength())
Debug.write("Distance: %.2f" % self.get_distance())
if self._indxr_images == []:
_select_images_function = getattr(
self, "_index_select_images_%s" % (self._index_select_images))
wedges = _select_images_function()
for wedge in wedges:
self.add_indexer_image_wedge(wedge)
self.set_indexer_prepare_done(True)
all_images = self.get_matching_images()
first = min(all_images)
last = max(all_images)
# next start to process these - first xycorr
xycorr = self.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
from dxtbx.serialize.xds import to_xds
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
for block in self._indxr_images:
xycorr.add_spot_range(block[0], block[1])
# FIXME need to set the origin here
xycorr.run()
for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]:
self._indxr_payload[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
imageset = self._indxr_imagesets[0]
masker = (imageset.get_format_class().get_instance(
imageset.paths()[0]).get_masker())
if masker is None:
# disable dynamic_shadowing
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing:
# find the region of the scan with the least predicted shadow
# to use for background determination in XDS INIT step
from dxtbx.model.experiment_list import ExperimentListFactory
imageset = self._indxr_imagesets[0]
xsweep = self._indxr_sweeps[0]
sweep_filename = os.path.join(
self.get_working_directory(),
"%s_indexed.expt" % xsweep.get_name())
ExperimentListFactory.from_imageset_and_crystal(
imageset, None).as_file(sweep_filename)
from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot
shadow_plot = ShadowPlot()
shadow_plot.set_working_directory(self.get_working_directory())
auto_logfiler(shadow_plot)
shadow_plot.set_sweep_filename(sweep_filename)
shadow_plot.set_json_filename(
os.path.join(
self.get_working_directory(),
"%s_shadow_plot.json" % shadow_plot.get_xpid(),
))
shadow_plot.run()
results = shadow_plot.get_results()
from scitbx.array_family import flex
fraction_shadowed = flex.double(results["fraction_shadowed"])
if flex.max(fraction_shadowed) == 0:
PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False
else:
scan_points = flex.double(results["scan_points"])
scan = imageset.get_scan()
oscillation = scan.get_oscillation()
if self._background_images is not None:
bg_images = self._background_images
bg_range_deg = (
scan.get_angle_from_image_index(bg_images[0]),
scan.get_angle_from_image_index(bg_images[1]),
)
bg_range_width = bg_range_deg[1] - bg_range_deg[0]
min_shadow = 100
best_bg_range = bg_range_deg
from libtbx.utils import frange
for bg_range_start in frange(
flex.min(scan_points),
flex.max(scan_points) - bg_range_width,
step=oscillation[1],
):
bg_range_deg = (bg_range_start,
bg_range_start + bg_range_width)
sel = (scan_points >= bg_range_deg[0]) & (
scan_points <= bg_range_deg[1])
mean_shadow = flex.mean(fraction_shadowed.select(sel))
if mean_shadow < min_shadow:
min_shadow = mean_shadow
best_bg_range = bg_range_deg
self._background_images = (
scan.get_image_index_from_angle(best_bg_range[0]),
scan.get_image_index_from_angle(best_bg_range[1]),
)
Debug.write("Setting background images: %s -> %s" %
self._background_images)
init = self.Init()
for file in ["X-CORRECTIONS.cbf", "Y-CORRECTIONS.cbf"]:
init.set_input_data_file(file, self._indxr_payload[file])
init.set_data_range(first, last)
if self._background_images:
init.set_background_range(self._background_images[0],
self._background_images[1])
else:
init.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
init.add_spot_range(block[0], block[1])
init.run()
# at this stage, need to (perhaps) modify the BKGINIT.cbf image
# to mark out the back stop
if PhilIndex.params.xds.backstop_mask:
Debug.write("Applying mask to BKGINIT.pck")
# copy the original file
cbf_old = os.path.join(init.get_working_directory(), "BKGINIT.cbf")
cbf_save = os.path.join(init.get_working_directory(),
"BKGINIT.sav")
shutil.copyfile(cbf_old, cbf_save)
# modify the file to give the new mask
from xia2.Toolkit.BackstopMask import BackstopMask
mask = BackstopMask(PhilIndex.params.xds.backstop_mask)
mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old)
init.reload()
for file in ["BLANK.cbf", "BKGINIT.cbf", "GAIN.cbf"]:
self._indxr_payload[file] = init.get_output_data_file(file)
if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder:
spotfinder = self.DialsSpotfinder()
for block in self._indxr_images:
spotfinder.add_spot_range(block[0], block[1])
spotfinder.run()
export = self.DialsExportSpotXDS()
export.set_input_data_file(
"observations.refl",
spotfinder.get_output_data_file("observations.refl"),
)
export.run()
for file in ["SPOT.XDS"]:
self._indxr_payload[file] = export.get_output_data_file(file)
else:
# next start to process these - then colspot
colspot = self.Colspot()
for file in [
"X-CORRECTIONS.cbf",
"Y-CORRECTIONS.cbf",
"BLANK.cbf",
"BKGINIT.cbf",
"GAIN.cbf",
]:
colspot.set_input_data_file(file, self._indxr_payload[file])
colspot.set_data_range(first, last)
colspot.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
for block in self._indxr_images:
colspot.add_spot_range(block[0], block[1])
colspot.run()
for file in ["SPOT.XDS"]:
self._indxr_payload[file] = colspot.get_output_data_file(file)
825 17
840 13
855 12
870 18
885 10
Will fit these data to the following functional form (bi-exponential decay):
y(x) = a0 + a1 * exp( -x/ a3 ) + a2 * exp( -x/ a4)
Initial values for parameters, to be refined:
10 900 80 27 225
"""
raw_strings = data.split("\n")
data_index = raw_strings.index("Time Counts")
x_obs = flex.double([float(line.split()[0]) for line in raw_strings[data_index+1:data_index+60]])
y_obs = flex.double([float(line.split()[1]) for line in raw_strings[data_index+1:data_index+60]])
w_obs = 1./(y_obs)
initial = flex.double([10, 900, 80, 27, 225])
#initial = flex.double([10.4, 958.3, 131.4, 33.9, 205])
from scitbx.examples.bevington import bevington_silver
class lbfgs_biexponential_fit(bevington_silver):
def __init__(self, x_obs, y_obs, w_obs, initial):
super(lbfgs_biexponential_fit,self).__init__()
self.set_cpp_data(x_obs,y_obs,w_obs)
assert x_obs.size() == y_obs.size()
self.x_obs = x_obs
self.y_obs = y_obs
self.w_obs = w_obs
self.n = len(initial)
def dump_crystal(entry, crystal, scan):
"""Export the crystal model."""
from scitbx.array_family import flex
# Get the sample
nx_sample = get_nx_sample(entry, "sample")
# Set the space group
nx_sample["unit_cell_group"] = crystal.get_space_group().type(
).hall_symbol()
# Get the unit cell and orientation matrix in the case of scan varying and
# scan static models
if crystal.num_scan_points:
num = crystal.num_scan_points
unit_cell = flex.double(flex.grid(num, 6))
orientation_matrix = flex.double(flex.grid(num, 9))
for i in range(num):
__cell = crystal.get_unit_cell_at_scan_point(i).parameters()
for j in range(6):
unit_cell[i, j] = __cell[j]
__matrix = crystal.get_U_at_scan_point(i)
for j in range(9):
orientation_matrix[i, j] = __matrix[j]
orientation_matrix = [[
tuple(orientation_matrix[i:i + 1, 0:3]),
tuple(orientation_matrix[i:i + 1, 3:6]),
tuple(orientation_matrix[i:i + 1, 6:9]),
] for i in range(num)]
unit_cell = [tuple(unit_cell[i:i + 1, :]) for i in range(num)]
average_unit_cell = crystal.get_unit_cell().parameters()
average_orientation_matrix = crystal.get_U()
average_orientation_matrix = [
average_orientation_matrix[0:3],
average_orientation_matrix[3:6],
average_orientation_matrix[6:9],
]
else:
unit_cell = [crystal.get_unit_cell().parameters()]
orientation_matrix = [crystal.get_U()]
orientation_matrix = [[
tuple(orientation_matrix[0][0:3]),
tuple(orientation_matrix[0][3:6]),
tuple(orientation_matrix[0][6:9]),
]]
average_unit_cell = unit_cell[0]
average_orientation_matrix = orientation_matrix[0]
# Save the unit cell data
nx_sample["name"] = "FROM_DIALS"
nx_sample["unit_cell"] = unit_cell
nx_sample["unit_cell"].attrs["angles_units"] = "deg"
nx_sample["unit_cell"].attrs["length_units"] = "angstrom"
# Save the orientation matrix
nx_sample["orientation_matrix"] = orientation_matrix
# Set an average unit cell etc for scan static stuff
nx_sample["average_unit_cell"] = average_unit_cell
nx_sample["average_unit_cell"].attrs["angles_units"] = "deg"
nx_sample["average_unit_cell"].attrs["length_units"] = "angstrom"
nx_sample["average_orientation_matrix"] = average_orientation_matrix
# Set depends on
if scan is not None:
nx_sample["depends_on"] = str(nx_sample["transformations/phi"].name)
else:
nx_sample["depends_on"] = "."
plot = '-p' in arguments
for file in files:
message = """Based on the file %s,
this program will compute incremental quadrant translations to circularize
powder rings on the inner four sensors. The algorithm treats each quadrant
independently and scores based on self-correlation upon 45-degree rotation.
""" % file
print message
image = dxtbx.load(file)
detector = image.get_detector()
beam = image.get_beam()
from xfel.metrology.quadrant import one_panel
ccs = flex.double()
for i_quad, quad in enumerate(detector.hierarchy()):
# find panel closest to the beam center
panels = []
def recursive_get_panels(group):
if group.is_group():
for child in group:
recursive_get_panels(child)
else:
panels.append(group)
recursive_get_panels(quad)
smallest_dist = float("inf")
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
import libtbx.load_env
usage = "%s [options] datablock.json" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=True,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0 or len(reflections) == 0:
parser.print_help()
exit(0)
imagesets = experiments.imagesets()
reflections = reflections[0]
shadowed = filter_shadowed_reflections(experiments, reflections)
print "# shadowed reflections: %i/%i (%.2f%%)" %(
shadowed.count(True), shadowed.size(),
shadowed.count(True)/shadowed.size() * 100)
expt = experiments[0]
x,y,z = reflections['xyzcal.px'].parts()
z_ = z * expt.scan.get_oscillation()[1]
zmin, zmax = expt.scan.get_oscillation_range()
hist_scan_angle = flex.histogram(z_.select(shadowed), n_slots=int(zmax-zmin))
#hist_scan_angle.show()
uc = experiments[0].crystal.get_unit_cell()
d_spacings = uc.d(reflections['miller_index'])
ds2 = uc.d_star_sq(reflections['miller_index'])
hist_res = flex.histogram(ds2.select(shadowed), flex.min(ds2), flex.max(ds2), n_slots=20, )
#hist_res.show()
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
plt.hist2d(z_.select(shadowed).as_numpy_array(),
ds2.select(shadowed).as_numpy_array(), bins=(40,40),
range=((flex.min(z_), flex.max(z_)),(flex.min(ds2), flex.max(ds2))))
yticks_dsq = flex.double(plt.yticks()[0])
from cctbx import uctbx
yticks_d = uctbx.d_star_sq_as_d(yticks_dsq)
plt.axes().set_yticklabels(['%.2f' %y for y in yticks_d])
plt.xlabel('Scan angle (degrees)')
plt.ylabel('Resolution (A^-1)')
cbar = plt.colorbar()
cbar.set_label('# shadowed reflections')
plt.savefig('n_shadowed_hist2d.png')
plt.clf()
plt.scatter(hist_scan_angle.slot_centers().as_numpy_array(), hist_scan_angle.slots().as_numpy_array())
plt.xlabel('Scan angle (degrees)')
plt.ylabel('# shadowed reflections')
plt.savefig("n_shadowed_vs_scan_angle.png")
plt.clf()
plt.scatter(hist_res.slot_centers().as_numpy_array(), hist_res.slots().as_numpy_array())
plt.xlabel('d_star_sq')
plt.savefig("n_shadowed_vs_resolution.png")
plt.clf()
def load_crystal(entry):
from cctbx import uctbx
from dxtbx.model import Crystal
from scitbx.array_family import flex
# Get the sample
nx_sample = get_nx_sample(entry, "sample")
# Set the space group
space_group_symbol = nx_sample["unit_cell_group"][()]
# Get depends on
if h5str(nx_sample["depends_on"][()]) != ".":
assert h5str(nx_sample["depends_on"][()]) == h5str(
nx_sample["transformations/phi"].name)
# Read the average unit cell data
average_unit_cell = flex.double(np.array(nx_sample["average_unit_cell"]))
assert nx_sample["average_unit_cell"].attrs["angles_units"] == "deg"
assert nx_sample["average_unit_cell"].attrs["length_units"] == "angstrom"
assert len(average_unit_cell.all()) == 1
assert len(average_unit_cell) == 6
average_orientation_matrix = flex.double(
np.array(nx_sample["average_orientation_matrix"]))
assert len(average_orientation_matrix.all()) == 2
assert average_orientation_matrix.all()[0] == 3
assert average_orientation_matrix.all()[1] == 3
# Get the real space vectors
uc = uctbx.unit_cell(tuple(average_unit_cell))
U = matrix.sqr(average_orientation_matrix)
B = matrix.sqr(uc.fractionalization_matrix()).transpose()
A = U * B
A = A.inverse()
real_space_a = A[0:3]
real_space_b = A[3:6]
real_space_c = A[6:9]
# Read the unit cell data
unit_cell = flex.double(np.array(nx_sample["unit_cell"]))
assert nx_sample["unit_cell"].attrs["angles_units"] == "deg"
assert nx_sample["unit_cell"].attrs["length_units"] == "angstrom"
# Read the orientation matrix
orientation_matrix = flex.double(np.array(nx_sample["orientation_matrix"]))
assert len(unit_cell.all()) == 2
assert len(orientation_matrix.all()) == 3
assert unit_cell.all()[0] == orientation_matrix.all()[0]
assert unit_cell.all()[1] == 6
assert orientation_matrix.all()[1] == 3
assert orientation_matrix.all()[2] == 3
# Construct the crystal model
crystal = Crystal(real_space_a, real_space_b, real_space_c,
space_group_symbol)
# Sort out scan points
if unit_cell.all()[0] > 1:
A_list = []
for i in range(unit_cell.all()[0]):
uc = uctbx.unit_cell(tuple(unit_cell[i:i + 1, :]))
U = matrix.sqr(tuple(orientation_matrix[i:i + 1, :, :]))
B = matrix.sqr(uc.fractionalization_matrix()).transpose()
A_list.append(U * B)
crystal.set_A_at_scan_points(A_list)
else:
assert unit_cell.all_eq(average_unit_cell)
assert orientation_matrix.all_eq(average_orientation_matrix)
# Return the crystal
return crystal
def extract_b(self):
return flex.double([a.b for a in self._atoms])
def find_matching_symmetry(unit_cell, target_space_group, max_delta=5):
cs = crystal.symmetry(unit_cell=unit_cell, space_group=sgtbx.space_group())
target_bravais_t = bravais_types.bravais_lattice(
group=target_space_group.info().reference_setting().group())
best_subgroup = None
best_angular_difference = 1e8
# code based on cctbx/sgtbx/lattice_symmetry.py but optimised to only
# look at subgroups with the correct bravais type
input_symmetry = cs
# Get cell reduction operator
cb_op_inp_minimum = input_symmetry.change_of_basis_op_to_minimum_cell()
# New symmetry object with changed basis
minimum_symmetry = input_symmetry.change_basis(cb_op_inp_minimum)
# Get highest symmetry compatible with lattice
lattice_group = sgtbx.lattice_symmetry_group(
minimum_symmetry.unit_cell(),
max_delta=max_delta,
enforce_max_delta_for_generated_two_folds=True)
# Get list of sub-spacegroups
subgrs = subgroups.subgroups(lattice_group.info()).groups_parent_setting()
# Order sub-groups
sort_values = flex.double()
for group in subgrs:
order_z = group.order_z()
space_group_number = sgtbx.space_group_type(group, False).number()
assert 1 <= space_group_number <= 230
sort_values.append(order_z * 1000 + space_group_number)
perm = flex.sort_permutation(sort_values, True)
for i_subgr in perm:
acentric_subgroup = subgrs[i_subgr]
acentric_supergroup = metric_supergroup(acentric_subgroup)
## Add centre of inversion to acentric lattice symmetry
#centric_group = sgtbx.space_group(acentric_subgroup)
#centric_group.expand_inv(sgtbx.tr_vec((0,0,0)))
# Make symmetry object: unit-cell + space-group
# The unit cell is potentially modified to be exactly compatible
# with the space group symmetry.
subsym = crystal.symmetry(unit_cell=minimum_symmetry.unit_cell(),
space_group=acentric_subgroup,
assert_is_compatible_unit_cell=False)
#supersym = crystal.symmetry(
#unit_cell=minimum_symmetry.unit_cell(),
#space_group=acentric_supergroup,
#assert_is_compatible_unit_cell=False)
# Convert subgroup to reference setting
cb_op_minimum_ref = subsym.space_group_info().type().cb_op()
ref_subsym = subsym.change_basis(cb_op_minimum_ref)
# Ignore unwanted groups
bravais_t = bravais_types.bravais_lattice(
group=ref_subsym.space_group())
if bravais_t != target_bravais_t:
continue
# Choose best setting for monoclinic and orthorhombic systems
cb_op_best_cell = ref_subsym.change_of_basis_op_to_best_cell(
best_monoclinic_beta=True)
best_subsym = ref_subsym.change_basis(cb_op_best_cell)
# Total basis transformation
cb_op_best_cell = change_of_basis_op(str(cb_op_best_cell),
stop_chars='',
r_den=144,
t_den=144)
cb_op_minimum_ref = change_of_basis_op(str(cb_op_minimum_ref),
stop_chars='',
r_den=144,
t_den=144)
cb_op_inp_minimum = change_of_basis_op(str(cb_op_inp_minimum),
stop_chars='',
r_den=144,
t_den=144)
cb_op_inp_best = cb_op_best_cell * cb_op_minimum_ref * cb_op_inp_minimum
# Use identity change-of-basis operator if possible
if best_subsym.unit_cell().is_similar_to(input_symmetry.unit_cell()):
cb_op_corr = cb_op_inp_best.inverse()
try:
best_subsym_corr = best_subsym.change_basis(cb_op_corr)
except RuntimeError as e:
if str(e).find(
"Unsuitable value for rational rotation matrix.") < 0:
raise
else:
if best_subsym_corr.space_group() == best_subsym.space_group():
cb_op_inp_best = cb_op_corr * cb_op_inp_best
max_angular_difference = find_max_delta(
reduced_cell=minimum_symmetry.unit_cell(),
space_group=acentric_supergroup)
if max_angular_difference < best_angular_difference:
#best_subgroup = subgroup
best_angular_difference = max_angular_difference
best_subgroup = {
'subsym': subsym,
#'supersym':supersym,
'ref_subsym': ref_subsym,
'best_subsym': best_subsym,
'cb_op_inp_best': cb_op_inp_best,
'max_angular_difference': max_angular_difference
}
if best_subgroup is not None:
return best_subgroup
def __init__(self, indices, parameter_vector):
self.indices = indices
parameter_vector = flex.double(parameter_vector)
self.constrained_value = flex.mean(parameter_vector.select(indices))
self._shifts = parameter_vector.select(indices) - self.constrained_value
def __init__(self,
reflections,
step_size=45,
tolerance=1.5,
max_height_fraction=0.25,
percentile=0.05,
histogram_binning='linear',
nn_per_bin=5):
self.tolerance = tolerance # Margin of error for max unit cell estimate
from scitbx.array_family import flex
NEAR = 10
self.NNBIN = nn_per_bin # target number of neighbors per histogram bin
self.histogram_binning = histogram_binning
direct = flex.double()
if 'entering' in reflections:
entering_flags = reflections['entering']
else:
entering_flags = flex.bool(reflections.size(), True)
rs_vectors = reflections['rlp']
phi_deg = reflections['xyzobs.mm.value'].parts()[2] * (180 / math.pi)
d_spacings = flex.double()
# nearest neighbor analysis
from annlib_ext import AnnAdaptor
for imageset_id in range(flex.max(reflections['imageset_id']) + 1):
sel_imageset = reflections['imageset_id'] == imageset_id
if sel_imageset.count(True) == 0:
continue
phi_min = flex.min(phi_deg.select(sel_imageset))
phi_max = flex.max(phi_deg.select(sel_imageset))
d_phi = phi_max - phi_min
n_steps = max(int(math.ceil(d_phi / step_size)), 1)
for n in range(n_steps):
sel_step = sel_imageset & (phi_deg >=
(phi_min + n * step_size)) & (
phi_deg <
(phi_min + (n + 1) * step_size))
for entering in (True, False):
sel_entering = sel_step & (entering_flags == entering)
if sel_entering.count(True) == 0:
continue
query = flex.double()
query.extend(rs_vectors.select(sel_entering).as_double())
if query.size() == 0:
continue
IS_adapt = AnnAdaptor(data=query, dim=3, k=1)
IS_adapt.query(query)
direct.extend(1 / flex.sqrt(IS_adapt.distances))
d_spacings.extend(1 / rs_vectors.norms())
assert len(direct) > NEAR, (
"Too few spots (%d) for nearest neighbour analysis." % len(direct))
perm = flex.sort_permutation(direct)
direct = direct.select(perm)
d_spacings = d_spacings.select(perm)
# eliminate nonsensical direct space distances
sel = direct > 1
direct = direct.select(sel)
d_spacings = d_spacings.select(sel)
# reject top 1% of longest distances to hopefully get rid of any outliers
n = int(math.floor(0.99 * len(direct)))
direct = direct[:n]
d_spacings = d_spacings[:n]
# determine the most probable nearest neighbor distance (direct space)
if self.histogram_binning == 'log':
hst = flex.histogram(flex.log10(direct),
n_slots=int(len(direct) / self.NNBIN))
else:
hst = flex.histogram(direct, n_slots=int(len(direct) / self.NNBIN))
centers = hst.slot_centers()
if self.histogram_binning == 'log':
self.slot_start = flex.double(
[10**(s - 0.5 * hst.slot_width()) for s in hst.slot_centers()])
self.slot_end = flex.double(
[10**(s + 0.5 * hst.slot_width()) for s in hst.slot_centers()])
self.slot_width = self.slot_end - self.slot_start
else:
self.slot_start = hst.slot_centers() - 0.5 * hst.slot_width()
self.slot_end = hst.slot_centers() + 0.5 * hst.slot_width()
self.slot_width = hst.slot_width()
self.relative_frequency = hst.slots().as_double() / self.slot_width
highest_bin_height = flex.max(self.relative_frequency)
if False: # to print out the histogramming analysis
smin, smax = flex.min(direct), flex.max(direct)
stats = flex.mean_and_variance(direct)
import sys
out = sys.stdout
print >> out, " range: %6.2f - %.2f" % (smin, smax)
print >> out, " mean: %6.2f +/- %6.2f on N = %d" % (
stats.mean(), stats.unweighted_sample_standard_deviation(),
direct.size())
hst.show(f=out, prefix=" ", format_cutoffs="%6.2f")
print >> out, ""
# choose a max cell based on bins above a given fraction of the highest bin height
# given multiple
isel = (self.relative_frequency.as_double() >
(max_height_fraction * highest_bin_height)).iselection()
self.max_cell = (self.tolerance *
self.slot_end[int(flex.max(isel.as_double()))])
# determine the 5th-percentile direct-space distance
perm = flex.sort_permutation(direct, reverse=True)
self.percentile = direct[perm[int(percentile * len(direct))]]
self.reciprocal_lattice_vectors = rs_vectors
self.d_spacings = d_spacings
self.direct = direct
self.histogram = hst
def _silhouette_analysis(self, cluster_labels, linkage_matrix, n_clusters,
min_silhouette_score):
"""Compare valid equal-sized clustering using silhouette scores.
Args:
cluster_labels (scitbx.array_family.flex.int):
linkage_matrix (numpy.ndarray): The hierarchical clustering of centroids of the
initial clustering as produced by
:func:`scipy.cluster.hierarchy.linkage`.
n_clusters (int): Optionally override the automatic determination of the
number of clusters.
min_silhouette_score (float): The minimum silhouette score to be used
in automatic determination of the number of clusters.
Returns:
cluster_labels (scitbx.array_family.flex.int): A label for each coordinate.
"""
eps = 1e-6
X = self.coords.as_numpy_array()
cluster_labels_input = cluster_labels
distances = linkage_matrix[::, 2]
distances = np.insert(distances, 0, 0)
silhouette_scores = flex.double()
thresholds = flex.double()
threshold_n_clusters = flex.size_t()
for threshold in distances[1:]:
cluster_labels = cluster_labels_input.deep_copy()
labels = hierarchy.fcluster(linkage_matrix,
threshold - eps,
criterion="distance").tolist()
counts = [labels.count(l) for l in set(labels)]
if len(set(counts)) > 1:
# only equal-sized clusters are valid
continue
n = len(set(labels))
if n == 1:
continue
elif n_clusters is not Auto and n != n_clusters:
continue
for i in range(len(labels)):
cluster_labels.set_selected(cluster_labels_input == i,
int(labels[i] - 1))
if len(set(cluster_labels)) == X.shape[0]:
# silhouette coefficient not defined if 1 dataset per cluster
# not sure what the default value should be
sample_silhouette_values = np.full(cluster_labels.size(), 0)
else:
# Compute the silhouette scores for each sample
sample_silhouette_values = metrics.silhouette_samples(
X, cluster_labels.as_numpy_array(), metric="cosine")
silhouette_avg = sample_silhouette_values.mean()
silhouette_scores.append(silhouette_avg)
thresholds.append(threshold)
threshold_n_clusters.append(n)
count_negative = (sample_silhouette_values < 0).sum()
logger.info("Clustering:")
logger.info(" Number of clusters: %i" % n)
logger.info(" Threshold score: %.3f (%.1f deg)" %
(threshold, math.degrees(math.acos(1 - threshold))))
logger.info(" Silhouette score: %.3f" % silhouette_avg)
logger.info(" -ve silhouette scores: %.1f%%" %
(100 * count_negative / sample_silhouette_values.size))
if n_clusters is Auto:
idx = flex.max_index(silhouette_scores)
else:
idx = flex.first_index(threshold_n_clusters, n_clusters)
if idx is None:
raise Sorry("No valid clustering with %i clusters" %
n_clusters)
if n_clusters is Auto and silhouette_scores[idx] < min_silhouette_score:
# assume single cluster
cluster_labels = flex.int(cluster_labels.size(), 0)
else:
threshold = thresholds[idx] - eps
labels = hierarchy.fcluster(linkage_matrix,
threshold,
criterion="distance")
cluster_labels = flex.double(cluster_labels.size(), -1)
for i in range(len(labels)):
cluster_labels.set_selected(cluster_labels_input == i,
float(labels[i] - 1))
return cluster_labels, threshold
def extract_occ(self):
return flex.double([a.occ for a in self._atoms])
def update_detail(self,horizon_phil,current_status,first_time_through,verbose):
assert(len(self.observed_spots) == len(self.predicted_spots))
if horizon_phil.indexing.outlier_detection.verbose:
classes=[str(current_status[i]) for i in xrange(len(self.observed_spots))]
class_types = set(classes)
class_counts = dict([[item,classes.count(item)] for item in class_types])
flex_counts = flex.int(class_counts.values())
assert flex.sum(flex_counts) == len(self.observed_spots)
#for pair in class_counts.items():
# print "%10s %6d"%pair
#print "%10s %6d"%("TOTAL",len(self.observed_spots))
if status_with_marked_outliers == None:
# status_with_marked_outliers==None is shorthand for identifying the first run through
print """After indexing on a subset of %d spots (from all images), %d were reclassified as
either lying on the spindle, or potential overlapped spots or ice rings."""%(
len(self.observed_spots),len(self.observed_spots)-class_counts["GOOD"])
else:
print """Rerefinement on just the well-fit spots followed by spot reclassification
leaves %d good spots on which to calculate a triclinic rmsd."""%(class_counts["GOOD"])
# check good spots
if (self.good is not None):
match = 0
for i in xrange(len(self.observed_spots)):
if ((current_status[i] == SpotClass.GOOD) and self.good[i]):
match = match + 1
if self.verbose:print "Number of GOOD spots matched with previous model =",match
# calculate differences for all spots
self.sorted_observed_spots = {}
self.dr = flex.double()
self.not_good_dr = flex.double()
self.dx = [0.0 for i in xrange(len(self.observed_spots))]
self.dy = [0.0 for i in xrange(len(self.observed_spots))]
for i in xrange(len(self.observed_spots)):
o = self.observed_spots[i]
p = self.predicted_spots[i]
self.dx[i] = o[0] - p[0]
self.dy[i] = o[1] - p[1]
self.sorted_observed_spots[
math.sqrt(self.dx[i]*self.dx[i] + self.dy[i]*self.dy[i])] = i
# separate GOOD spots
spotclasses = {SpotClass.GOOD:0,SpotClass.SPINDLE:0,SpotClass.OVERLAP:0,SpotClass.ICE:0,SpotClass.OUTLIER:0,SpotClass.NONE:0}
for key in sorted(self.sorted_observed_spots.keys()):
spotclass = current_status[self.sorted_observed_spots[key]]
spotclasses[spotclass]+=1
if (current_status[self.sorted_observed_spots[key]] == SpotClass.GOOD):
self.dr.append(key)
else:
self.not_good_dr.append(key)
if verbose: print ", ".join(["=".join([str(i[0]),"%d"%i[1]]) for i in spotclasses.items()]),
totalsp = sum([spotclasses.values()[iidx] for iidx in xrange(len(spotclasses))])
if verbose: print "Total=%d"%(totalsp),"# observed spots",len(self.observed_spots)
assert totalsp == len(self.observed_spots), "Some spot pairs have the same predicted-observed distances. Do you have duplicated images?"
self.x = flex.double(len(self.dr))
for i in xrange(len(self.x)):
self.x[i] = float(i)/float(len(self.x))
limit = int(self.fraction*len(self.dr))
if limit < 4: return # Basic sanity check, need at least a few good spots to fit the distribution
fitted_rayleigh = fit_cdf(x_data=self.dr[0:limit],
y_data=self.x[0:limit],distribution=rayleigh)
if False:
y_data=self.x[0:limit]
inv_cdf = [fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data]
from matplotlib import pyplot as plt
plt.plot(self.dr[0:limit],self.x[0:limit],"r+")
plt.plot(inv_cdf,y_data,"b.")
plt.show()
# store indices for spots used for fitting
self.fraction_spot_indices = []
for dr in self.dr[0:limit]:
self.fraction_spot_indices.append(self.sorted_observed_spots[dr])
# generate points for fitted distributions
rayleigh_cdf_x = flex.double(500)
for i in xrange(len(rayleigh_cdf_x)):
rayleigh_cdf_x[i] = float(i)/float(len(rayleigh_cdf_x))
rayleigh_cdf = flex.double(len(rayleigh_cdf_x))
for i in xrange(len(rayleigh_cdf_x)):
rayleigh_cdf[i] = fitted_rayleigh.distribution.cdf(x=rayleigh_cdf_x[i])
# generate points for pdf
dr_bins,dr_histogram = make_histogram_data(data=self.dr,n_bins=100)
rayleigh_pdf = flex.double(len(dr_bins))
for i in xrange(len(dr_bins)):
rayleigh_pdf[i] = fitted_rayleigh.distribution.pdf(x=dr_bins[i])
rayleigh_pdf = rayleigh_pdf/flex.sum(rayleigh_pdf)
dr_bins = flex.double(dr_bins)
dr_histogram = flex.double(dr_histogram)
# standard deviation for cdf
sd = math.sqrt((4.0-math.pi)/(2.0)*
fitted_rayleigh.x[0]*fitted_rayleigh.x[0])
if self.verbose:print 'Standard deviation of Rayleigh fit = %4.3f'%sd
sd_data = None
radius_outlier_index = None
limit_outlier = None
for i in xrange(len(rayleigh_cdf_x)):
mx = rayleigh_cdf_x[i]
my = rayleigh_cdf[i]
for j in xrange(1,len(self.dr)):
upper_x = self.dr[j]
upper_y = self.x[j]
lower_x = self.dr[j-1]
lower_y = self.x[j-1]
if ((my >= lower_y) and (my < upper_y)):
if ((sd <= (upper_x - mx)) and ((lower_x - mx) > 0.0)):
sd_data = ((mx,my),(lower_x,lower_y))
radius_outlier_index = j-1
limit_outlier = lower_x
if self.verbose:print "Width of green bar = %4.3f"%(lower_x - mx)
break
if (sd_data is not None):
break
if (radius_outlier_index is None):
radius_outlier_index = len(self.dr)
if (limit_outlier is None):
limit_outlier = self.dr[-1]
radius_95 = None
for i in xrange(len(rayleigh_cdf)):
if (rayleigh_cdf[i] >= 0.95):
radius_95 = rayleigh_cdf_x[i]
break
if (radius_95 is None):
radius_95 = rayleigh_cdf_x[-1]
upper_circle = []
lower_circle = []
d_radius = 2.0*radius_95/100.0
x = -radius_95
r2 = radius_95*radius_95
for i in xrange(100):
y = math.sqrt(r2 - x*x)
upper_circle.append((x,y))
lower_circle.append((x,-y))
x = x + d_radius
y = 0.0
upper_circle.append((x,y))
lower_circle.append((x,-y))
self.sqrtr2 = math.sqrt(r2)
# color code dx dy
dxdy_fraction = []
dxdy_inliers = []
dxdy_outliers = []
limit = self.dr[int(self.fraction*len(self.dr))]
trifold = dict(fraction=0,inlier=0,outlier=0,total=0)
for key in self.dr:
trifold["total"]+=1
i = self.sorted_observed_spots[key]
if (key < limit):
trifold["fraction"]+=1
if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or
(self.dy[i] > 1.0) or (self.dy[i] < -1.0))):
dxdy_fraction.append((self.dx[i],self.dy[i]))
elif (key < limit_outlier):
trifold["inlier"]+=1
if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or
(self.dy[i] > 1.0) or (self.dy[i] < -1.0))):
dxdy_inliers.append((self.dx[i],self.dy[i]))
else:
trifold["outlier"]+=1
if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or
(self.dy[i] > 1.0) or (self.dy[i] < -1.0))):
dxdy_outliers.append((self.dx[i],self.dy[i]))
if verbose: print ", ".join(["=".join([str(i[0]),"%d"%i[1]]) for i in trifold.items()])
# color code observed fractions
o_fraction = []
o_inliers = []
o_outliers = []
mr = format_data(x_data=rayleigh_cdf_x,y_data=rayleigh_cdf)
limit = int(self.fraction*len(self.dr))
for i in xrange(len(self.dr)):
if (self.dr[i] <= 1.0):
if (i < limit):
o_fraction.append((self.dr[i],self.x[i]))
elif (i < radius_outlier_index):
o_inliers.append((self.dr[i],self.x[i]))
else:
o_outliers.append((self.dr[i],self.x[i]))
if horizon_phil.indexing.outlier_detection.verbose:
o_outliers_for_severity = []
for i in xrange(radius_outlier_index, len(self.dr)):
o_outliers_for_severity.append((self.dr[i],self.x[i]))
# limit data range
for i in xrange(len(dr_bins)):
if (dr_bins[i] > 1.0):
dr_bins.resize(i)
dr_histogram.resize(i)
rayleigh_pdf.resize(i)
break
ho = format_data(x_data=dr_bins,y_data=dr_histogram)
hr = format_data(x_data=dr_bins,y_data=rayleigh_pdf)
# format data for graphing
self.plot_dxdy_data = [dxdy_fraction,dxdy_inliers,dxdy_outliers,
[(0.0,0.0)],[],[],[],[]]
self.framework = {4:dict(status=SpotClass.SPINDLE),
5:dict(status=SpotClass.OVERLAP),
6:dict(status=SpotClass.OUTLIER),
7:dict(status=SpotClass.ICE),
}
for key in self.not_good_dr:
i = self.sorted_observed_spots[key]
status = current_status[i]
if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or
(self.dy[i] > 1.0) or (self.dy[i] < -1.0))):
statuskey = [k for k in self.framework.keys() if self.framework[k]["status"]==status][0]
self.plot_dxdy_data[statuskey].append((self.dx[i],self.dy[i]))
self.plot_cdf_data = [mr,o_fraction,o_inliers,o_outliers]
if (sd_data is not None):
self.plot_cdf_data.append(sd_data)
self.plot_pdf_data = [ho,hr]
# mark outliers
if (first_time_through): #i.e., first time through the update() method
if (radius_outlier_index < len(self.dr)):
for i in xrange(radius_outlier_index,len(self.dr)):
current_status[self.sorted_observed_spots[self.dr[i]]] = SpotClass.OUTLIER
# reset good spots
self.good = [False for i in xrange(len(self.observed_spots))]
for i in xrange(len(self.observed_spots)):
if (current_status[i] == SpotClass.GOOD):
self.good[i] = True
count_outlier = 0
count_good = 0
for i in xrange(len(self.observed_spots)):
if (current_status[i] == SpotClass.OUTLIER):
count_outlier = count_outlier + 1
elif (current_status[i] == SpotClass.GOOD):
count_good = count_good + 1
if self.verbose:print 'Old GOOD =', len(self.dr),\
'OUTLIER =', count_outlier,\
'New GOOD =', count_good
if horizon_phil.indexing.outlier_detection.verbose and status_with_marked_outliers is None:
print "\nOf the remaining %d spots, %.1f%% were lattice outliers, leaving %d well-fit spots"%(
len(self.dr),100.*count_outlier/len(self.dr), count_good )
if count_outlier==0:return
#width of green bar is sd
delta_spread = o_outliers_for_severity[1][1]-o_outliers_for_severity[0][1]
severity = 0.
for item in o_outliers_for_severity:
delta_r = item[0] # obs - predicted deviation in mm
spread = item[1] # order of observed deviation on a scale from 0 to 1
# now invert the cdf to find expected delta r:
expected_delta_r = fitted_rayleigh.distribution.sigma * math.sqrt(
-2.* math.log(1.-spread) )
#print item, expected_delta_r, (delta_r - expected_delta_r) / sd
severity += ((delta_r - expected_delta_r) / sd)
severity *= delta_spread
print "The outlier severity is %.2f sigma [defined in J Appl Cryst (2010) 43, p.611 sec. 4].\n"%severity
return current_status
def _label_clusters_first_pass(self, n_datasets, n_sym_ops):
"""First pass labelling of clusters.
Labels points into clusters such that cluster contains exactly one copy
of each dataset.
Args:
n_datasets (int): The number of datasets.
n_sym_ops (int): The number of symmetry operations.
Returns:
cluster_labels (scitbx.array_family.flex.int): A label for each coordinate, labelled from
0 .. n_sym_ops.
"""
# initialise cluster labels: -1 signifies doesn't belong to a cluster
cluster_labels = flex.int(self.coords.all()[0], -1)
X_orig = self.coords.as_numpy_array()
cluster_id = 0
while cluster_labels.count(-1) > 0:
dataset_ids = (flex.int_range(n_datasets * n_sym_ops) %
n_datasets).as_numpy_array()
coord_ids = flex.int_range(dataset_ids.size).as_numpy_array()
# select only those points that don't already belong to a cluster
sel = np.where(cluster_labels == -1)
X = X_orig[sel]
dataset_ids = dataset_ids[sel]
coord_ids = coord_ids[sel]
# choose a high density point as seed for cluster
nbrs = NearestNeighbors(n_neighbors=min(11, len(X)),
algorithm="brute",
metric="cosine").fit(X)
distances, indices = nbrs.kneighbors(X)
average_distance = flex.double(
[dist[1:].mean() for dist in distances])
i = flex.min_index(average_distance)
d_id = dataset_ids[i]
cluster = np.array([coord_ids[i]])
cluster_dataset_ids = np.array([d_id])
xis = np.array([X[i]])
for j in range(n_datasets - 1):
# select only those rows that don't correspond to a dataset already
# present in current cluster
sel = np.where(dataset_ids != d_id)
X = X[sel]
dataset_ids = dataset_ids[sel]
coord_ids = coord_ids[sel]
assert len(X) > 0
# Find nearest neighbour in cosine-space to the current cluster centroid
nbrs = NearestNeighbors(n_neighbors=min(1, len(X)),
algorithm="brute",
metric="cosine").fit(X)
distances, indices = nbrs.kneighbors([xis.mean(axis=0)])
k = indices[0][0]
d_id = dataset_ids[k]
cluster = np.append(cluster, coord_ids[k])
cluster_dataset_ids = np.append(cluster_dataset_ids, d_id)
xis = np.append(xis, [X[k]], axis=0)
# label this cluster
cluster_labels.set_selected(flex.size_t(cluster.tolist()),
cluster_id)
cluster_id += 1
return cluster_labels
def run(argv=None):
if argv is None:
argv = sys.argv[1:]
command_line = (libtbx.option_parser.option_parser(
usage="%s [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files" % libtbx.env.dispatcher_name)
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--crop", "-c",
action="store_true",
default=False,
dest="crop",
help="Crop the image such that the beam center is in the middle")
.option(None, "--skip_converted", "-s",
action="store_true",
default=False,
dest="skip_converted",
help="Skip converting if an image already exist that matches the destination file name")
.option(None, "--wavelength", "-w",
type="float",
default=None,
dest="wavelength",
help="Override the image's wavelength (angstroms)")
.option(None, "--distance", "-d",
type="float",
default=None,
dest="distance",
help="Override the detector distance (mm)")
.option(None, "--pixel_size", "-p",
type="float",
default=None,
dest="pixel_size",
help="Override the detector pixel size (mm)")
.option(None, "--beam_x", "-x",
type="float",
default=None,
dest="beam_center_x",
help="Override the beam x position (pixels)")
.option(None, "--beam_y", "-y",
type="float",
default=None,
dest="beam_center_y",
help="Override the beam y position (pixels)")
.option(None, "--overload", "-o",
type="float",
default=None,
dest="overload",
help="Override the detector overload value (ADU)")
).process(args=argv)
paths = command_line.args
if len(paths) <= 0:
raise Usage("No files specified")
for imgpath in paths:
if command_line.options.verbose:
print "Reading %s"%(imgpath)
try:
img = dxtbx.load(imgpath)
except IOError:
img = None
pass
if img is None:
import numpy as np
try:
raw_data = np.loadtxt(imgpath)
from scitbx.array_family import flex
raw_data = flex.double(raw_data.astype(np.double))
except ValueError:
raise Usage("Couldn't load %s, no supported readers"%imgpath)
detector = None
beam = None
scan = None
is_multi_image = False
else:
try:
raw_data = img.get_raw_data()
is_multi_image = False
except TypeError:
raw_data = img.get_raw_data(0)
is_multi_image = True
detector = img.get_detector()
beam = img.get_beam()
scan = img.get_scan()
if detector is None:
if command_line.options.distance is None:
raise Usage("Can't get distance from image. Override with -d")
if command_line.options.pixel_size is None:
raise Usage("Can't get pixel size from image. Override with -p")
if command_line.options.overload is None:
raise Usage("Can't get overload value from image. Override with -o")
distance = command_line.options.distance
pixel_size = command_line.options.pixel_size
overload = command_line.options.overload
else:
detector = detector[0]
if command_line.options.distance is None:
distance = detector.get_distance()
else:
distance = command_line.options.distance
if command_line.options.pixel_size is None:
pixel_size = detector.get_pixel_size()[0]
else:
pixel_size = command_line.options.pixel_size
if command_line.options.overload is None:
overload = detector.get_trusted_range()[1]
else:
overload = command_line.options.overload
if beam is None:
if command_line.options.wavelength is None:
raise Usage("Can't get wavelength from image. Override with -w")
wavelength = command_line.options.wavelength
else:
if command_line.options.wavelength is None:
wavelength = beam.get_wavelength()
else:
wavelength = command_line.options.wavelength
if beam is None and detector is None:
if command_line.options.beam_center_x is None:
print "Can't get beam x position from image. Using image center. Override with -x"
beam_x = raw_data.focus()[0] * pixel_size
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
print "Can't get beam y position from image. Using image center. Override with -y"
beam_y = raw_data.focus()[1] * pixel_size
else:
beam_y = command_line.options.beam_center_y * pixel_size
else:
if command_line.options.beam_center_x is None:
beam_x = detector.get_beam_centre(beam.get_s0())[0]
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
beam_y = detector.get_beam_centre(beam.get_s0())[1]
else:
beam_y = command_line.options.beam_center_y * pixel_size
if scan is None:
timestamp = None
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec,msec))
if is_multi_image:
for i in xrange(img.get_num_images()):
save_image(command_line, imgpath, scan, img.get_raw_data(i), distance, pixel_size, wavelength, beam_x, beam_y, overload, timestamp, image_number = i)
else:
save_image(command_line, imgpath, scan, raw_data, distance, pixel_size, wavelength, beam_x, beam_y, overload, timestamp)
def exercise():
"""Test prepare_map_for_docking using data with known errors."""
# Generate two half-maps with same anisotropic signal,
# independent anisotropic noise. Test to see if
# simulation parameters are recovered.
# Start by working out how large the padding will have to be so that
# starting automatically-generated map will be large enough to contain
# sphere with room to spare around model.
n_residues = 25
d_min = 2.5
from cctbx.development.create_models_or_maps import generate_model
test_model = generate_model(n_residues=n_residues)
sites_cart = test_model.get_sites_cart()
cart_min = flex.double(sites_cart.min())
cart_max = flex.double(sites_cart.max())
box_centre = (cart_min+cart_max)/2
dsqrmax = flex.max( (sites_cart - tuple(box_centre)).norms() )**2
model_radius = math.sqrt(dsqrmax)
min_model_extent = flex.min(cart_max - cart_min)
pad_to_allow_cube = model_radius - min_model_extent/2
# Extra space needed for eventual masking
boundary_to_smoothing_ratio = 2
soft_mask_radius = d_min
padding = soft_mask_radius * boundary_to_smoothing_ratio
box_cushion = padding + pad_to_allow_cube
# Make map in box big enough to cut out cube containing sphere
mmm = map_model_manager()
mmm.generate_map(
n_residues=n_residues, d_min=d_min, k_sol=0.1, b_sol=50.,
box_cushion=box_cushion)
model = mmm.model()
sites_cart = model.get_sites_cart()
cart_min = flex.double(sites_cart.min())
cart_max = flex.double(sites_cart.max())
# Turn starting map into map coeffs for the signal
ucpars = mmm.map_manager().unit_cell().parameters()
d_max=max(ucpars[0], ucpars[1], ucpars[2])
start_map_coeffs = mmm.map_as_fourier_coefficients(
d_min=d_min, d_max=d_max)
# Apply anisotropic scaling to map coeffs
b_target = (100.,200.,300.,-50.,50.,100.)
b_model = (30.,30.,30.,0.,0.,0.) # All atoms in model have B=30
u_star = adptbx.u_cart_as_u_star(
start_map_coeffs.unit_cell(), adptbx.b_as_u(b_target))
b_expected = list((flex.double(b_target) + flex.double(b_model)))
scaled_map_coeffs = start_map_coeffs.apply_debye_waller_factors(u_star=u_star)
# Generate map coefficient errors for first half-map from complex normal
# distribution
b_target_e = (0.,0.,0.,-50.,-50.,100.) # Anisotropy for error terms
u_star_e = adptbx.u_cart_as_u_star(
start_map_coeffs.unit_cell(), adptbx.b_as_u(b_target_e))
se_target = 1. # Target for SigmaE variance term
rsigma = math.sqrt(se_target / 2.)
jj = 0.+1.j # Define I for generating complex numbers
random_complexes1 = flex.complex_double()
ncoeffs=start_map_coeffs.size()
random.seed(123457) # Make runs reproducible
for i in range(ncoeffs):
random_complexes1.append(random.gauss(0.,rsigma) + random.gauss(0.,rsigma)*jj)
rc1_miller = start_map_coeffs.customized_copy(data=random_complexes1)
mc1_delta = rc1_miller.apply_debye_waller_factors(u_star=u_star_e)
map1_coeffs = scaled_map_coeffs.customized_copy(
data=scaled_map_coeffs.data() + mc1_delta.data())
# Repeat for second half map with independent errors from same distribution
random_complexes2 = flex.complex_double()
for i in range(ncoeffs):
random_complexes2.append(random.gauss(0.,rsigma) + random.gauss(0.,rsigma)*jj)
rc2_miller = start_map_coeffs.customized_copy(data=random_complexes2)
mc2_delta = rc2_miller.apply_debye_waller_factors(u_star=u_star_e)
map2_coeffs = scaled_map_coeffs.customized_copy(
data=scaled_map_coeffs.data() + mc2_delta.data())
mmm.add_map_from_fourier_coefficients(
map1_coeffs, map_id = 'map_manager_1')
mmm.add_map_from_fourier_coefficients(
map2_coeffs, map_id = 'map_manager_2')
radius = model_radius + 5. # Sphere as large as possible to allow masking
results = run_refine_cryoem_errors(
mmm, d_min,
sphere_cent=tuple(box_centre), radius=radius, verbosity=0)
resultsdict = results.resultsdict
b_refined_a = resultsdict["a_baniso"]
# print("\nIdeal A tensor as Baniso: ", b_expected)
# print("Refined A tensor as Baniso", b_refined_a)
b_refined_e = resultsdict["sigmaE_baniso"] # Note: refers to intensity scale
# print("\nIdeal SigmaE tensor as Baniso: ",list(flex.double(b_target_e*2)))
# print("Refined SigmaE tensor as Baniso", b_refined_e)
for i in range(6):
assert(approx_equal(b_refined_a[i],b_expected[i], eps=25))
assert(approx_equal(b_refined_e[i],2*b_target_e[i], eps=25))
from __future__ import division, absolute_import, print_function
import sys, math
trial = sys.argv[1] # Like "005"
from scitbx.array_family import flex
if True:
angular_offset = flex.double()
for line in open(trial, "r"):
if line.find("angular offset") > 0:
value = line.split()[5]
angular_offset.append(float(value))
order = flex.sort_permutation(angular_offset)
print(list(angular_offset.select(order)))
median = angular_offset[order[len(order) // 2]]
rmsd = math.sqrt(flex.mean(angular_offset * angular_offset))
print(trial, "%6d measurements; rmsd %9.5f" % (len(angular_offset), rmsd),
"Median is %9.5f" % (median))
print("Max is %9.5f" % (flex.max(angular_offset)))
from matplotlib import pyplot as plt
zoom = "hi"
#nbins = dict(hi=300,lo=100)[zoom]
# to ensure bins always represent 0.001, multiply mx by 1000
nbins = int(1000. * flex.max(angular_offset))
n, bins, patches = plt.hist(angular_offset,
nbins,
normed=0,
facecolor="orange",
alpha=0.75)
def exercise_writer(use_mrcfile=None, output_axis_order=[3, 2, 1]):
from cctbx import uctbx, sgtbx
from scitbx.array_family import flex
mt = flex.mersenne_twister(0)
nxyz = (
4,
5,
6,
)
grid = flex.grid(nxyz)
real_map_data = mt.random_double(size=grid.size_1d())
real_map_data.reshape(grid)
unit_cell = uctbx.unit_cell((10, 10, 10, 90, 90, 90))
if use_mrcfile:
from iotbx.mrcfile import create_output_labels
labels = create_output_labels(
program_name='test',
limitations=['extract_unique'],
output_labels=['test label'],
)
iotbx.mrcfile.write_ccp4_map(
file_name="four_five_six.mrc",
unit_cell=unit_cell,
space_group=sgtbx.space_group_info("P1").group(),
map_data=real_map_data,
labels=labels,
output_axis_order=output_axis_order)
input_real_map = iotbx.mrcfile.map_reader(
file_name="four_five_six.mrc")
for x in input_real_map.labels:
print("LABEL: ", x)
assert str(input_real_map.labels).find('extract_unique') > -1
else:
iotbx.ccp4_map.write_ccp4_map(
file_name="four_five_six.map",
unit_cell=unit_cell,
space_group=sgtbx.space_group_info("P1").group(),
map_data=real_map_data,
labels=flex.std_string(["iotbx.ccp4_map.tst"]))
input_real_map = iotbx.ccp4_map.map_reader(
file_name="four_five_six.map")
input_map_data = input_real_map.map_data()
real_map_mmm = real_map_data.as_1d().min_max_mean()
input_map_mmm = input_map_data.as_1d().min_max_mean()
cc = flex.linear_correlation(real_map_data.as_1d(),
input_map_data.as_1d()).coefficient()
assert cc > 0.999
print("\nMRCFILE with 4x5x6 map and axis order %s %s" %
(output_axis_order, cc))
assert approx_equal(input_real_map.unit_cell_parameters,
unit_cell.parameters())
assert approx_equal(real_map_mmm.min, input_real_map.header_min, eps=0.001)
assert approx_equal(real_map_mmm.min, input_map_mmm.min, eps=0.001)
# random small maps of different sizes
for nxyz in flex.nested_loop((2, 1, 1), (4, 4, 4)):
mt = flex.mersenne_twister(0)
grid = flex.grid(nxyz)
real_map = mt.random_double(size=grid.size_1d())
real_map = real_map - 0.5
real_map.reshape(grid)
if use_mrcfile:
iotbx.mrcfile.write_ccp4_map(
file_name="random.mrc",
unit_cell=uctbx.unit_cell((1, 1, 1, 90, 90, 90)),
space_group=sgtbx.space_group_info("P1").group(),
gridding_first=(0, 0, 0),
gridding_last=tuple(grid.last(False)),
map_data=real_map,
labels=flex.std_string(["iotbx.ccp4_map.tst"]))
m = iotbx.mrcfile.map_reader(file_name="random.mrc")
else:
iotbx.ccp4_map.write_ccp4_map(
file_name="random.map",
unit_cell=uctbx.unit_cell((1, 1, 1, 90, 90, 90)),
space_group=sgtbx.space_group_info("P1").group(),
gridding_first=(0, 0, 0),
gridding_last=tuple(grid.last(False)),
map_data=real_map,
labels=flex.std_string(["iotbx.ccp4_map.tst"]))
m = iotbx.ccp4_map.map_reader(file_name="random.map")
mmm = flex.double(list(real_map)).min_max_mean()
m1 = real_map.as_1d()
m2 = m.map_data().as_1d()
cc = flex.linear_correlation(m1, m2).coefficient()
assert cc > 0.999
assert approx_equal(m.unit_cell_parameters, (1, 1, 1, 90, 90, 90))
assert approx_equal(mmm.min, m.header_min)
assert approx_equal(mmm.max, m.header_max)
#
# write unit_cell_grid explicitly to map
iotbx.ccp4_map.write_ccp4_map(
file_name="random_b.map",
unit_cell=uctbx.unit_cell((1, 1, 1, 90, 90, 90)),
space_group=sgtbx.space_group_info("P1").group(),
unit_cell_grid=real_map.all(),
map_data=real_map,
labels=flex.std_string(["iotbx.ccp4_map.tst"]))
m = iotbx.ccp4_map.map_reader(file_name="random_b.map")
m1 = real_map.as_1d()
m2 = m.map_data().as_1d()
cc = flex.linear_correlation(m1, m2).coefficient()
assert cc > 0.999
mmm = flex.double(list(real_map)).min_max_mean()
assert approx_equal(m.unit_cell_parameters, (1, 1, 1, 90, 90, 90))
assert approx_equal(mmm.min, m.header_min)
assert approx_equal(mmm.max, m.header_max)
#
#
gridding_first = (0, 0, 0)
gridding_last = tuple(grid.last(False))
map_box = maptbx.copy(real_map, gridding_first, gridding_last)
map_box.reshape(flex.grid(map_box.all()))
if use_mrcfile:
iotbx.mrcfile.write_ccp4_map(
file_name="random_box.mrc",
unit_cell=uctbx.unit_cell((1, 1, 1, 90, 90, 90)),
space_group=sgtbx.space_group_info("P1").group(),
map_data=map_box,
labels=flex.std_string(["iotbx.ccp4_map.tst"]))
else:
iotbx.ccp4_map.write_ccp4_map(
file_name="random_box.map",
unit_cell=uctbx.unit_cell((1, 1, 1, 90, 90, 90)),
space_group=sgtbx.space_group_info("P1").group(),
map_data=map_box,
labels=flex.std_string(["iotbx.ccp4_map.tst"]))
print("OK")
