def test_ann():
reference = flex.double()
for j in range(3 * 100):
reference.append(random.random())
query = flex.double()
for j in range(3 * 10):
query.append(random.random())
ann = ann_adaptor(data=reference, dim=3, k=1)
ann.query(query)
# workout code - see how far separated on average they are - which should
# in principle decrease as the number of positions in the reference set
# increases
offsets = []
for j in range(10):
q = matrix.col([query[3 * j + k] for k in range(3)])
r = matrix.col([reference[3 * ann.nn[j] + k] for k in range(3)])
offsets.append((q - r).length())
return meansd(offsets)
def exercise_derivatives(space_group_info, out):
crystal_symmetry = space_group_info.any_compatible_crystal_symmetry(
volume=1000)
space_group = space_group_info.group()
adp_constraints = space_group.adp_constraints()
m = adp_constraints.row_echelon_form()
print >> out, matrix.rec(m, (m.size()//6, 6)).mathematica_form(
one_row_per_line=True)
print >> out, list(adp_constraints.independent_indices)
u_cart_p1 = adptbx.random_u_cart()
u_star_p1 = adptbx.u_cart_as_u_star(crystal_symmetry.unit_cell(), u_cart_p1)
u_star = space_group.average_u_star(u_star_p1)
miller_set = miller.build_set(
crystal_symmetry=crystal_symmetry, d_min=3, anomalous_flag=False)
for h in miller_set.indices():
grads_fin = d_dw_d_u_star_finite(h=h, u_star=u_star)
print >> out, "grads_fin:", list(grads_fin)
grads_ana = d_dw_d_u_star_analytical(h=h, u_star=u_star)
print >> out, "grads_ana:", list(grads_ana)
compare_derivatives(grads_ana, grads_fin)
curvs_fin = d2_dw_d_u_star_d_u_star_finite(h=h, u_star=u_star)
print >> out, "curvs_fin:", list(curvs_fin)
curvs_ana = d2_dw_d_u_star_d_u_star_analytical(h=h, u_star=u_star)
print >> out, "curvs_ana:", list(curvs_ana)
compare_derivatives(curvs_ana, curvs_fin)
#
u_indep = adp_constraints.independent_params(u_star)
grads_indep_fin = d_dw_d_u_indep_finite(
adp_constraints=adp_constraints, h=h, u_indep=u_indep)
print >> out, "grads_indep_fin:", list(grads_indep_fin)
grads_indep_ana = flex.double(adp_constraints.independent_gradients(
all_gradients=list(grads_ana)))
print >> out, "grads_indep_ana:", list(grads_indep_ana)
compare_derivatives(grads_indep_ana, grads_indep_fin)
curvs_indep_fin = d2_dw_d_u_indep_d_u_indep_finite(
adp_constraints=adp_constraints, h=h, u_indep=u_indep)
print >> out, "curvs_indep_fin:", list(curvs_indep_fin)
curvs_indep_ana = adp_constraints.independent_curvatures(
all_curvatures=curvs_ana)
print >> out, "curvs_indep_ana:", list(curvs_indep_ana)
compare_derivatives(curvs_indep_ana, curvs_indep_fin)
#
curvs_indep_mm = None
if (str(space_group_info) == "P 1 2 1"):
assert list(adp_constraints.independent_indices) == [0,1,2,4]
curvs_indep_mm = p2_curv(h, u_star)
elif (str(space_group_info) == "P 4"):
assert list(adp_constraints.independent_indices) == [1,2]
curvs_indep_mm = p4_curv(h, u_star)
elif (str(space_group_info) in ["P 3", "P 6"]):
assert list(adp_constraints.independent_indices) == [2,3]
curvs_indep_mm = p3_curv(h, u_star)
elif (str(space_group_info) == "P 2 3"):
assert list(adp_constraints.independent_indices) == [2]
curvs_indep_mm = p23_curv(h, u_star)
if (curvs_indep_mm is not None):
curvs_indep_mm = flex.double(
curvs_indep_mm).matrix_symmetric_as_packed_u()
print >> out, "curvs_indep_mm:", list(curvs_indep_mm)
compare_derivatives(curvs_indep_ana, curvs_indep_mm)
def compare(xyz_to_hkl, xyz_to_hkl_ref):
# construct ann to perform search...
from cctbx.array_family import flex
from annlib_ext import AnnAdaptor as ann_adaptor
reference = flex.double()
xyzs = [xyz for xyz in xyz_to_hkl]
for xyz in xyzs:
reference.append(xyz[0])
reference.append(xyz[1])
reference.append(xyz[2])
ann = ann_adaptor(data = reference, dim = 3, k = 1)
n_correct = 0
n_wrong = 0
for xyz in xyz_to_hkl_ref:
query = flex.double(xyz)
ann.query(query)
nnxyz = xyzs[ann.nn[0]]
if xyz_to_hkl_ref[xyz] == xyz_to_hkl[nnxyz]:
n_correct += 1
else:
n_wrong += 1
return n_correct, n_wrong
def __init__(self,
fmodel,
map_type_str,
acentrics_scale = 2.0,
centrics_scale = 1.0):
"""
Compute x and y for x*Fobs-y*Fmodel given map type string
"""
self.fmodel = fmodel
mnm = mmtbx.map_names(map_name_string = map_type_str)
# R.Read, SIGMAA: 2mFo-DFc (acentrics) & mFo (centrics)
centric_flags = self.fmodel.f_obs().centric_flags().data()
acentric_flags = ~centric_flags
if(mnm.k != 0):
self.fo_scale = flex.double(self.fmodel.f_obs().size(), 1.0)
else: self.fo_scale = flex.double(self.fmodel.f_obs().size(), 0.0)
if(mnm.n != 0):
self.fc_scale = flex.double(self.fmodel.f_obs().size(), 1.0)
else: self.fc_scale = flex.double(self.fmodel.f_obs().size(), 0.0)
abs_kn = abs(mnm.k*mnm.n)
if(mnm.k != abs(mnm.n) and abs_kn > 1.e-6):
self.fo_scale.set_selected(acentric_flags, mnm.k)
self.fo_scale.set_selected( centric_flags, max(mnm.k-centrics_scale,0.))
self.fc_scale.set_selected(acentric_flags, mnm.n)
self.fc_scale.set_selected( centric_flags, max(mnm.n-centrics_scale,0.))
elif(mnm.k == abs(mnm.n) and abs_kn > 1.e-6):
fo_scale_k = self.fo_scale*mnm.k
self.fc_scale_n = self.fc_scale*mnm.n
self.fo_scale.set_selected(acentric_flags, fo_scale_k*acentrics_scale)
self.fo_scale.set_selected( centric_flags, fo_scale_k)
self.fc_scale.set_selected(acentric_flags, self.fc_scale_n*acentrics_scale)
self.fc_scale.set_selected( centric_flags, self.fc_scale_n)
else:
self.fo_scale *= mnm.k
self.fc_scale *= mnm.n
def calc_partiality_anisotropy_set(self, my_uc, rotx, roty, miller_indices,
ry, rz, r0, re, nu,
bragg_angle_set, alpha_angle_set, wavelength, crystal_init_orientation,
spot_pred_x_mm_set, spot_pred_y_mm_set, detector_distance_mm,
partiality_model, flag_beam_divergence):
#use III.4 in Winkler et al 1979 (A35; P901) for set of miller indices
O = sqr(my_uc.orthogonalization_matrix()).transpose()
R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose()
CO = crystal_orientation(O*R, basis_type.direct)
CO_rotate = CO.rotate_thru((1,0,0), rotx
).rotate_thru((0,1,0), roty)
A_star = sqr(CO_rotate.reciprocal_matrix())
S0 = -1*col((0,0,1./wavelength))
#caculate rs
rs_set = r0 + (re * flex.tan(bragg_angle_set))
if flag_beam_divergence:
rs_set += ((ry * flex.cos(alpha_angle_set))**2 + (rz * flex.sin(alpha_angle_set))**2)**(1/2)
#calculate rh
x = A_star.elems * miller_indices.as_vec3_double()
sd_array = x + S0.elems
rh_set = sd_array.norms() - (1/wavelength)
#calculate partiality
if partiality_model == "Lorentzian":
partiality_set = ((rs_set**2)/((2*(rh_set**2))+(rs_set**2)))
elif partiality_model == "Voigt":
partiality_set = self.voigt(rh_set, rs_set, nu)
elif partiality_model == "Lognormal":
partiality_set = self.lognpdf(rh_set, rs_set, nu)
#calculate delta_xy
d_ratio = -detector_distance_mm/sd_array.parts()[2]
calc_xy_array = flex.vec3_double(sd_array.parts()[0]*d_ratio, \
sd_array.parts()[1]*d_ratio, flex.double([0]*len(d_ratio)))
pred_xy_array = flex.vec3_double(spot_pred_x_mm_set, spot_pred_y_mm_set, flex.double([0]*len(d_ratio)))
delta_xy_set = (pred_xy_array - calc_xy_array).norms()
return partiality_set, delta_xy_set, rs_set, rh_set
def exercise_match_multi_indices():
h0 = flex.miller_index(((1,2,3), (-1,-2,-3), (2,3,4), (-2,-3,-4), (3,4,5)))
d0 = flex.double((1,2,3,4,5))
h1 = flex.miller_index(((1,2,3), (-2,-3,-4), (1,2,3), (2,3,4)))
d1 = flex.double((10,20,30,40))
mi = miller.match_multi_indices(h0, h0)
assert mi.have_singles() == 0
assert list(mi.pairs()) == zip(range(5), range(5))
mi = miller.match_multi_indices(h0, h1)
assert tuple(mi.singles(0)) == (1,4,)
assert tuple(mi.singles(1)) == ()
assert len(set(mi.pairs()) - set([(0,0), (0,2), (2,3), (3, 1)])) == 0
assert tuple(mi.number_of_matches(0)) == (2, 0, 1, 1, 0)
assert tuple(mi.pair_selection(0)) == (1, 0, 1, 1, 0)
assert tuple(mi.single_selection(0)) == (0, 1, 0, 0, 1)
assert tuple(mi.number_of_matches(1)) == (1, 1, 1, 1)
assert tuple(mi.pair_selection(1)) == (1, 1, 1, 1)
assert tuple(mi.single_selection(1)) == (0, 0, 0, 0)
assert tuple(mi.paired_miller_indices(0)) \
== tuple(h0.select(mi.pair_selection(0)))
l1 = list(mi.paired_miller_indices(1))
l2 = list(h1.select(mi.pair_selection(1)))
l1.sort()
l2.sort()
assert l1 == l2
try:
miller.match_multi_indices(h1, h0)
except RuntimeError, e:
pass
def exercise_map_to_asu(sg_symbol):
sg_type = sgtbx.space_group_type(sg_symbol)
index_abs_range = (4,4,4)
for anomalous_flag in (False,True):
m = miller.index_generator(
sg_type, anomalous_flag, index_abs_range).to_array()
a = flex.double()
p = flex.double()
c = flex.hendrickson_lattman()
for i in xrange(m.size()):
a.append(random.random())
p.append(random.random() * 2)
c.append([random.random() for j in xrange(4)])
f = flex.polar(a, p)
p = [p, p*(180/math.pi)]
m_random = flex.miller_index()
p_random = [flex.double(), flex.double()]
c_random = flex.hendrickson_lattman()
f_random = flex.complex_double()
for i,h_asym in enumerate(m):
h_eq = miller.sym_equiv_indices(sg_type.group(), h_asym)
i_eq = random.randrange(h_eq.multiplicity(anomalous_flag))
h_i = h_eq(i_eq)
m_random.append(h_i.h())
for deg in (False,True):
p_random[deg].append(h_i.phase_eq(p[deg][i], deg))
f_random.append(h_i.complex_eq(f[i]))
c_random.append(h_i.hendrickson_lattman_eq(c[i]))
m_random_copy = m_random.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
m_random_copy = m_random.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, f_random)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,f_asym in enumerate(f):
assert abs(f_asym - f_random[i]) < 1.e-6
m_random_copy = m_random.deep_copy()
a_random = a.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, a_random)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,a_asym in enumerate(a):
assert a_asym == a_random[i]
for deg in (False,True):
m_random_copy = m_random.deep_copy()
miller.map_to_asu(
sg_type, anomalous_flag, m_random_copy, p_random[deg], deg)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,p_asym in enumerate(p[deg]):
assert scitbx.math.phase_error(p_asym, p_random[deg][i], deg) < 1.e-5
m_random_copy = m_random.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, c_random)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,c_asym in enumerate(c):
for j in xrange(4):
assert abs(c_asym[j] - c_random[i][j]) < 1.e-5
def set_refinable_parameters(xray_structure, parameters, selections,
enforce_positivity=False):
# XXX PVA: Code below is terribly inefficient and MUST be moved into C++
sz = xray_structure.scatterers().size()
i = 0
for sel in selections:
# pre-check for positivity begin
# spread negative occupancies across i_seqs having positive ones
par_all = flex.double()
par_neg = flex.double()
i_p = i
for sel_ in sel:
p = parameters[i_p]
par_all.append(p)
if(p<0): par_neg.append(p)
i_p += 1
if(enforce_positivity and par_neg.size()>0):
par_all = par_all - flex.min(par_all)
fs = flex.sum(par_all)
if(fs != 0):
par_all = par_all / fs
# pre-check for positivity end
for j, sel_ in enumerate(sel):
sel__b = flex.bool(sz, flex.size_t(sel_))
xray_structure.set_occupancies(par_all[j], sel__b)
i+=1
def exercise_expand():
sg = sgtbx.space_group("P 41 (1,-1,0)")
h = flex.miller_index(((3,1,-2), (1,-2,0)))
assert tuple(sg.is_centric(h)) == (0, 1)
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=False, indices=h, build_iselection=False)
p1_i0 = ((-3,-1,2), (-1, 3,2),(3,1,2),(1,-3,2),(1,-2, 0),(2,1,0))
assert tuple(p1.indices) == p1_i0
assert p1.iselection.size() == 0
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=True, indices=h, build_iselection=False)
assert tuple(p1.indices) \
== ((3,1,-2), (1,-3,-2), (-3,-1,-2), (-1,3,-2),
(1,-2,0), (-2,-1,0), (-1,2,0), (2,1,0))
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=False, indices=h, build_iselection=True)
assert tuple(p1.indices) == p1_i0
assert tuple(p1.iselection) == (0,0,0,0,1,1)
a = flex.double((1,2))
p = flex.double((10,90))
p1 = miller.expand_to_p1_phases(
space_group=sg, anomalous_flag=False, indices=h, data=p, deg=True)
assert approx_equal(tuple(p1.data), (-10,110,110,-10, 90,30))
p1 = miller.expand_to_p1_phases(
space_group=sg, anomalous_flag=True, indices=h, data=p, deg=True)
assert approx_equal(tuple(p1.data), (10,-110,-110,10, 90,-30,-90,30))
p = flex.double([x * math.pi/180 for x in p])
v = [x * math.pi/180 for x in p1.data]
p1 = miller.expand_to_p1_phases(
space_group=sg, anomalous_flag=True, indices=h, data=p, deg=False)
assert approx_equal(tuple(p1.data), v)
f = flex.polar(a, p)
p1 = miller.expand_to_p1_complex(
space_group=sg, anomalous_flag=True, indices=h, data=f)
assert approx_equal(tuple(flex.abs(p1.data)), (1,1,1,1,2,2,2,2))
assert approx_equal(tuple(flex.arg(p1.data)), v)
hl = flex.hendrickson_lattman([(1,2,3,4), (5,6,7,8)])
p1 = miller.expand_to_p1_hendrickson_lattman(
space_group=sg, anomalous_flag=True, indices=h, data=hl)
assert approx_equal(p1.data, [
[1,2,3,4],
[1.232051,-1.866025,-4.964102,0.5980762],
[1.232051,-1.866025,-4.964102,0.5980762],
[1,2,3,4],
[5,6,7,8],
[2.696152,-7.330127,-10.4282,2.062178],
[-5,-6,7,8],
[7.696152,-1.330127,3.428203,-10.06218]])
b = flex.bool([True,False])
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=True, indices=h, build_iselection=True)
assert b.select(p1.iselection).all_eq(
flex.bool([True, True, True, True, False, False, False, False]))
i = flex.int([13,17])
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=True, indices=h, build_iselection=True)
assert i.select(p1.iselection).all_eq(flex.int([13,13,13,13,17,17,17,17]))
#
assert approx_equal(miller.statistical_mean(sg, False, h, a), 4/3.)
assert approx_equal(miller.statistical_mean(sg, True, h, a), 3/2.)
def df2m(df, cell, spgr, data=None, sigmas=None):
"""Constructs a miller.array from the columns specified by data/sigmas in the dataframe,
if both are None, returns just the indices.
needs cell and spgr to generate a symmetry object."""
anomalous_flag = False
if isinstance(df, pd.DataFrame):
try:
sel = df[data].notnull() # select notnull data items for index
except ValueError:
index = df.index
else:
index = df.index[sel]
else:
index = df
indices = flex.miller_index(index)
if data:
data = flex.double(df[data][sel])
if sigmas:
sigmas = flex.double(df[sigmas][sel])
symm = make_symmetry(cell, spgr)
ms = miller.set(
crystal_symmetry=symm, indices=indices, anomalous_flag=anomalous_flag)
return miller.array(ms, data=data, sigmas=sigmas)
def detwin_miller_array(miller_obs,
twin_law,
twin_fraction):
# for the moment, ignore incompleten twin pairs please
cb_op = sgtbx.change_of_basis_op( twin_law )
twin_related_miller = miller_obs.change_basis( cb_op ).set_observation_type(
miller_obs )
set1, set2 = miller_obs.common_sets( twin_related_miller )\
.set_observation_type(miller_obs )
assert miller_obs.observation_type() is not None
assert miller_obs.sigmas() is not None
if set1.is_xray_amplitude_array():
set1 = set1.f_as_f_sq()
set2 = set1.f_as_f_sq()
detwinned_f = flex.double()
detwinned_sigma = flex.double()
if set1.is_xray_intensity_array():
if set2.is_xray_intensity_array():
for i1,s1,i2,s2 in zip( set1.data(), set1.sigmas(),
set2.data(), set2.sigmas() ):
tmp_detwinner = detwin(i1,s1,i2,s2)
# we do some double work here actually
ni1 = tmp_detwinner.xm
ns1 = math.sqrt( math.abs(self.vcv[0]) )
detwinned_f.append( ni1 )
detwinned_s.append( ns1 )
set1 = set1.f_sq_as_f()
new_f = set1.customized_copy
def exercise_singular_least_squares():
obs = flex.double([1.234])
weights_2345 = flex.double([2.345])
weights_zero = flex.double([0])
r_free_flags = flex.bool([False])
a = flex.double([0])
b = flex.double([0])
for obs_type in ["F", "I"]:
for weights,scale_factor in [
(weights_2345, 3.456),
(weights_zero, 0)]:
tg = ext.targets_least_squares(
compute_scale_using_all_data=False,
obs_type=obs_type,
obs=obs,
weights=weights,
r_free_flags=r_free_flags,
f_calc=flex.complex_double(a, b),
derivatives_depth=2,
scale_factor=scale_factor)
if (weights is weights_2345):
assert approx_equal(tg.scale_factor(), scale_factor)
assert list(tg.gradients_work()) == [0j]
assert list(tg.hessians_work()) == [(1,1,1)]
else:
assert tg.scale_factor() is None
assert tg.target_work() is None
assert tg.target_test() is None
assert tg.gradients_work().size() == 0
assert tg.hessians_work().size() == 0
def exercise_f_model_no_scales(symbol = "C 2"):
random.seed(0)
flex.set_random_seed(0)
x = random_structure.xray_structure(
space_group_info = sgtbx.space_group_info(symbol=symbol),
elements =(("O","N","C")*5+("H",)*10),
volume_per_atom = 200,
min_distance = 1.5,
general_positions_only = True,
random_u_iso = True,
random_occupancy = False)
f_obs = abs(x.structure_factors(d_min = 1.5, algorithm="fft").f_calc())
x.shake_sites_in_place(mean_distance=1)
k_iso = flex.double(f_obs.data().size(), 2)
k_aniso = flex.double(f_obs.data().size(), 3)
fmodel = mmtbx.f_model.manager(
xray_structure = x,
k_isotropic = k_iso,
k_anisotropic = k_aniso,
f_obs = f_obs)
fc = abs(fmodel.f_calc()).data()
fm = abs(fmodel.f_model()).data()
fmns = abs(fmodel.f_model_no_scales()).data()
assert approx_equal(flex.mean(fm/fc), 6)
assert approx_equal(flex.mean(fmns/fc), 1)
def test_grouped_observations():
# some dummy observations for one reflection
I1, I2, I3 = 101, 100, 99
w1, w2, w3 = 0.9, 1.0, 0.8
g1, g2, g3 = 1.01, 1.0, 0.99
# combine with observations from some other group to make a dataset
hkl = flex.miller_index([(0,0,1)] * 3 + [(0,0,2)] * 2)
intensity = flex.double([I1, I2, I3, 10, 10])
weight = flex.double([w1, w2, w3, 1.0, 1.0])
phi = flex.double(len(hkl), 0) # dummy
scale = flex.double([g1, g2, g3, 1.0, 1.0])
# group the observations by Miller index
go = GroupedObservations(hkl,
intensity,
weight,
phi,
scale)
# ensure there are two groups, the first of size 3, the second of size 2
assert list(go.get_group_size()) == [3,2]
# the first group has an average intensity given by the HRS formula expanded:
avI = (w1*g1*I1 + w2*g2*I2 + w3*g3*I3) / (w1*g1*g1 + w2*g2*g2 + w3*g3*g3)
assert approx_equal(avI, go.get_average_intensity()[0])
print "OK"
def __init__(self, params,
fo,
hl_coeffs,
ncs_object=None,
map_coeffs=None,
model_map_coeffs=None,
log=None,
as_gui_program=False):
self.model_map_coeffs = model_map_coeffs
self.correlation_coeffs = flex.double()
self.mean_phase_errors = flex.double()
density_modification.density_modification.__init__(
self, params, fo, hl_coeffs,
ncs_object=ncs_object,
map_coeffs=map_coeffs,
log=log,
as_gui_program=as_gui_program)
if len(self.correlation_coeffs) > 1:
model_coeffs, start_coeffs = self.model_map_coeffs.common_sets(self.map_coeffs_start)
model_fft_map = model_coeffs.fft_map(
resolution_factor=self.params.grid_resolution_factor).apply_sigma_scaling()
fft_map = start_coeffs.fft_map(
resolution_factor=self.params.grid_resolution_factor
).apply_sigma_scaling()
corr = flex.linear_correlation(
model_fft_map.real_map_unpadded().as_1d(), fft_map.real_map_unpadded().as_1d())
print "Starting dm/model correlation: %.6f" %corr.coefficient()
print "Final dm/model correlation: %.6f" %self.correlation_coeffs[-1]
fft_map.as_ccp4_map(file_name="starting.map", labels=[])
def correlation(G_ref, I_ref, G_model, I_model, mi=None, plot=False):
if G_ref:
ref = open(G_ref)
I_ref_sub = []
for i in ref:
I_ref_sub.append(float(i))
R = flex.linear_correlation(flex.double(G_model), flex.double(I_ref_sub)).coefficient()
print "The Gm correlation: ", round(R, 5)
if I_ref:
ref = open(I_ref)
I_ref_sub = []
I_ref_match = []
for i in ref:
I_ref_sub.append(float(i))
if mi:
for j in mi:
I_ref_match.append(I_ref_sub[j])
else:
model_match = []
for i, j in zip(I_model, I_ref_sub):
if i != 0:
model_match.append(i)
I_ref_match.append(j)
I_model = model_match
R = flex.linear_correlation(flex.double(I_model), flex.double(I_ref_match)).coefficient()
print "The Ih correlation: ", round(R, 5)
def __init__(self, centers_of_mass,
sites_cart,
target_functor,
rot_objs,
selections,
suppress_gradients):
t_r = target_functor(compute_gradients=not suppress_gradients)
self.f = t_r.target_work()
if (suppress_gradients):
self.grads_wrt_r = None
self.grads_wrt_t = None
return
target_grads_wrt_xyz = t_r.gradients_wrt_atomic_parameters(site=True)
self.grads_wrt_r = []
self.grads_wrt_t = []
target_grads_wrt_xyz = flex.vec3_double(target_grads_wrt_xyz.packed())
for sel,rot_obj, cm in zip(selections, rot_objs, centers_of_mass):
sites_cart_cm = sites_cart.select(sel) - cm
target_grads_wrt_xyz_sel = target_grads_wrt_xyz.select(sel)
target_grads_wrt_r = matrix.sqr(
sites_cart_cm.transpose_multiply(target_grads_wrt_xyz_sel))
self.grads_wrt_t.append(flex.double(target_grads_wrt_xyz_sel.sum()))
g_phi = (rot_obj.r_phi() * target_grads_wrt_r).trace()
g_psi = (rot_obj.r_psi() * target_grads_wrt_r).trace()
g_the = (rot_obj.r_the() * target_grads_wrt_r).trace()
self.grads_wrt_r.append(flex.double([g_phi, g_psi, g_the]))
def show_terms(structure, term_table, coseq_dict=None):
assert len(term_table) == structure.scatterers().size()
for scatterer,terms in zip(structure.scatterers(), term_table):
print scatterer.label, list(terms),
if (coseq_dict is not None):
terms_to_match = list(terms[1:])
have_match = False
tags = coseq_dict.keys()
tags.sort()
for tag in tags:
for coseq_terms in coseq_dict[tag]:
n = min(len(coseq_terms), len(terms_to_match))
if (coseq_terms[:n] == terms_to_match[:n]):
print tag,
have_match = True
if (not have_match):
print "Unknown",
print
sums_terms = flex.double()
multiplicities = flex.double()
for scatterer,terms in zip(structure.scatterers(), term_table):
sums_terms.append(flex.sum(flex.size_t(list(terms))))
multiplicities.append(scatterer.multiplicity())
print "TD%d: %.2f" % (
len(terms)-1, flex.mean_weighted(sums_terms, multiplicities))
def jacobian_callable(pfh,current_values):
rotx = current_values[0]
roty = current_values[1]
from scitbx.matrix import sqr
Ai = sqr(OO.input_orientation.reciprocal_matrix())
Rx = col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(rotx)
Ry = col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(roty)
Rz = col((0,0,1)).axis_and_angle_as_r3_rotation_matrix(0.0)
dRx_drotx = col((1,0,0)).axis_and_angle_as_r3_derivative_wrt_angle(rotx)
dRy_droty = col((0,1,0)).axis_and_angle_as_r3_derivative_wrt_angle(roty)
dA_drotx = Rz * Ry * dRx_drotx * Ai
dA_droty = Rz * dRy_droty * Rx * Ai
dexc_drotx = [
OO.ucbp3.simple_part_excursion_part_rotxy(
wavelength = OO.central_wavelength_ang,
observation_no = obsno,
dA_drotxy = dA_drotx)
for obsno in xrange(len(OO.parent.indexed_pairs))]
dexc_droty = [
OO.ucbp3.simple_part_excursion_part_rotxy(
wavelength = OO.central_wavelength_ang,
observation_no = obsno,
dA_drotxy = dA_droty)
for obsno in xrange(len(OO.parent.indexed_pairs))]
return flex.double(dexc_drotx)/(2.*math.pi), flex.double(dexc_droty)/(2.*math.pi)
def pack_parameters(O):
O.x = flex.double()
O.l = flex.double()
O.u = flex.double()
O.nbd = flex.int()
sstab = O.xray_structure.site_symmetry_table()
for i_sc,sc in enumerate(O.xray_structure.scatterers()):
assert sc.flags.use_u_iso()
assert not sc.flags.use_u_aniso()
#
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
p = sc.site
else:
p = site_symmetry.site_constraints().independent_params(
all_params=sc.site)
O.x.extend(flex.double(p))
O.l.resize(O.x.size(), 0)
O.u.resize(O.x.size(), 0)
O.nbd.resize(O.x.size(), 0)
#
O.x.append(sc.u_iso)
O.l.append(0)
O.u.append(0)
O.nbd.append(1)
O.x *= O.p_as_x
def get_f_masks(xrs, miller_array):
crystal_gridding = maptbx.crystal_gridding(
unit_cell = xrs.unit_cell(),
d_min = miller_array.d_min(),
resolution_factor = 1./4,
symmetry_flags = maptbx.use_space_group_symmetry,
space_group_info = xrs.space_group_info())
mp = mmtbx.masks.mask_master_params.extract()
mask_data = mmtbx.masks.mask_from_xray_structure(
xray_structure = xrs,
p1 = True,
solvent_radius = mp.solvent_radius,
shrink_truncation_radius = mp.shrink_truncation_radius,
for_structure_factors = True,
n_real = crystal_gridding.n_real()).mask_data
n = mask_data.all()
mask_data1 = flex.double(flex.grid(n), 0)
mask_data2 = flex.double(flex.grid(n), 0)
I,J,K = xrange(n[0]), xrange(n[1]), xrange(n[2])
for i in I:
for j in J:
for k in K:
if(i < n[0]//2 and j < n[1]//2 and k < n[2]//2):
mask_data1[i,j,k]=mask_data[i,j,k]
else:
mask_data2[i,j,k]=mask_data[i,j,k]
f_mask1 = miller_array.structure_factors_from_map(map=mask_data1,
use_scale = True, anomalous_flag = False, use_sg = False)
f_mask2 = miller_array.structure_factors_from_map(map=mask_data2,
use_scale = True, anomalous_flag = False, use_sg = False)
return [f_mask1.data(), f_mask2.data()]
def calc_scales(self, params_in):
""" Calculate an array of scales based on scale, B and wavelength using the
equation $scale * exp(-2*B*(sin(theta)/wavelength)^2)$
Reuturn a scale vector for all the reflections in self, using the
parameters defined in the array params.
:params: a tuple of the form appropriate for the crystal symetry, such as
one produced by get_x0(). This method only uses params[0] (scale) and
params[1] (B)
:return: a list of scales for all the miller indicies in self
"""
if self.use_scales:
scale = params_in[0]
B = params_in[1]
sin_sq_theta = self.miller_array.two_theta(wavelength=self.wavelength) \
.sin_theta_over_lambda_sq().data()
scales = scale * self.miller_array.data()
exp_arg = flex.double(-2 * B * sin_sq_theta)
return flex.double(flex.double(scales) * flex.exp(exp_arg))
else:
# Horrible way to get vector of ones...
return flex.double(self.miller_array.data()/self.miller_array.data())
def read_table(file_name):
atomic_number = None
number_of_electrons = None
x = flex.double()
y = flex.double()
sigmas = flex.double()
lf = line_feeder(open(file_name))
while 1:
line = lf.next()
if lf.eof:
break
if line.startswith(" FORM: ATOMIC NUMBER="):
atomic_number = float(line.split("=")[1])
assert int(atomic_number) == atomic_number
elif line.startswith(" FORM: # ELECTRONS="):
number_of_electrons = float(line.split("=")[1])
assert int(number_of_electrons) == number_of_electrons
elif line.startswith(" X (1/A)"):
assert atomic_number == number_of_electrons
while 1:
line = lf.next()
assert not lf.eof
if line == " *** END OF DATA ***":
lf.eof = True
break
vals_str = line.split()
for val_str in vals_str:
assert len(val_str) == 13
x.append(float(vals_str[0]))
y.append(float(vals_str[1]))
assert vals_str[1][-4] == "E"
sigmas.append(float("0.00000005" + vals_str[1][-4:]))
return table(int(atomic_number), x, y * atomic_number, sigmas)
def join(self,data_dict):
"""The join() function closes the database.
"""
# Terminate the consumer process by feeding it a None command and
# wait for it to finish.
self._db_commands_queue.put(None)
self._db_commands_queue.join()
self._db_results_queue.join()
self._semaphore.acquire()
nrows = data_dict["rows"].get_obj()[0]
print "writing observation pickle with %d rows"%nrows
kwargs = dict(
miller_lookup = flex.size_t(data_dict["miller_proxy"].get_obj()[:nrows]),
observed_intensity = flex.double(data_dict["intensity_proxy"].get_obj()[:nrows]),
observed_sigI = flex.double(data_dict["sigma_proxy"].get_obj()[:nrows]),
frame_lookup = flex.size_t(data_dict["frame_proxy"].get_obj()[:nrows]),
original_H = flex.int (data_dict["H_proxy"].get_obj()[:nrows]),
original_K = flex.int (data_dict["K_proxy"].get_obj()[:nrows]),
original_L = flex.int (data_dict["L_proxy"].get_obj()[:nrows]),
)
import cPickle as pickle
pickle.dump(kwargs, open(self.params.output.prefix+"_observation.pickle","wb"),
pickle.HIGHEST_PROTOCOL)
pickle.dump(self.miller, open(self.params.output.prefix+"_miller.pickle","wb"),
pickle.HIGHEST_PROTOCOL)
pickle.dump(data_dict["xtal_proxy"].get_obj().raw.replace('\0','').strip(),
open(self.params.output.prefix+"_frame.pickle","wb"),pickle.HIGHEST_PROTOCOL)
return kwargs
def exercise_miller_array_data_types():
miller_set = crystal.symmetry(unit_cell=(10, 10, 10, 90, 90, 90), space_group_symbol="P1").miller_set(
indices=flex.miller_index([(1, 2, 3), (4, 5, 6)]), anomalous_flag=False
)
for data in [
flex.bool([False, True]),
flex.int([0, 1]),
flex.size_t([0, 1]),
flex.double([0, 1]),
flex.complex_double([0, 1]),
]:
miller_array = miller_set.array(data=data)
if op.isfile("tmp_iotbx_mtz.mtz"):
os.remove("tmp_iotbx_mtz.mtz")
assert not op.isfile("tmp_iotbx_mtz.mtz")
miller_array.as_mtz_dataset(column_root_label="DATA").mtz_object().write(file_name="tmp_iotbx_mtz.mtz")
assert op.isfile("tmp_iotbx_mtz.mtz")
mtz_obj = mtz.object(file_name="tmp_iotbx_mtz.mtz")
miller_arrays_read_back = mtz_obj.as_miller_arrays()
assert len(miller_arrays_read_back) == 1
miller_array_read_back = miller_arrays_read_back[0]
assert miller_array_read_back.indices().all_eq(miller_array.indices())
if miller_array.is_integer_array() or miller_array.is_bool_array():
assert miller_array_read_back.data().all_eq(flex.int([0, 1]))
elif miller_array.is_real_array():
assert miller_array_read_back.data().all_eq(flex.double([0, 1]))
elif miller_array.is_complex_array():
assert miller_array_read_back.data().all_eq(flex.complex_double([0, 1]))
else:
raise RuntimeError("Programming error.")
def testing_function_for_rsfit(basic_map,delta_h,xray_structure,out):
for i_trial in xrange(100):
sites_cart = flex.vec3_double((flex.random_double(size=3)-0.5)*1)
tmp_sites_cart = sites_cart.deep_copy()
for i in xrange(3):
ref = lbfgs(
basic_map=basic_map,
sites_cart=tmp_sites_cart,
delta_h=delta_h)
temp = flex.double(ref.sites_cart[0])-flex.double((0,0,0))
temp = math.sqrt(temp.dot(temp))
if temp <= 2*delta_h:
break
print >> out, "recycling:", ref.sites_cart[0]
tmp_sites_cart = ref.sites_cart
for site,sitec in zip(ref.sites_cart,xray_structure.sites_cart()):
print >> out, i_trial
print >> out, sitec
print >> out, sites_cart[0]
print >> out, site
temp = flex.double(site)-flex.double(sitec)
temp = math.sqrt(temp.dot(temp))
print >> out, temp, delta_h
assert temp <= delta_h*2
print >> out
def __init__(self):
self.data_obs1 = flex.double(2,1.0)
self.data_obs2 = flex.double(2,3.0)
self.sigma_obs1 = flex.double(2,0.1)
self.sigma_obs2 = flex.double(2,1)
self.unit_cell = uctbx.unit_cell('20, 30, 40, 90.0, 90.0, 90.0')
#mi = flex.miller_index(((1,2,3), (1,2,3)))
self.mi = flex.miller_index(((1,2,3), (5,6,7)))
self.xs = crystal.symmetry((20,30,40), "P 2 2 2")
self.ms = miller.set(self.xs, self.mi)
self.u = [1,2,3,4,5,6]
self.p_scale = 0.40
#self.u = [0,0,0,0,0,0]
#self.p_scale = 0.00
self.ls_i_wt = scaling.least_squares_on_i_wt(
self.mi,
self.data_obs1,
self.sigma_obs1,
self.data_obs2,
self.sigma_obs2,
self.p_scale,
self.unit_cell,
self.u)
self.ls_i = scaling.least_squares_on_i(
self.mi,
self.data_obs1,
self.sigma_obs1,
self.data_obs2,
self.sigma_obs2,
self.p_scale,
self.unit_cell,
self.u)
self.ls_f_wt = scaling.least_squares_on_f_wt(
self.mi,
self.data_obs1,
self.sigma_obs1,
self.data_obs2,
self.sigma_obs2,
self.p_scale,
self.unit_cell,
self.u)
self.ls_f = scaling.least_squares_on_f(
self.mi,
self.data_obs1,
self.sigma_obs1,
self.data_obs2,
self.sigma_obs2,
self.p_scale,
self.unit_cell,
self.u)
self.tst_ls_f_wt()
self.tst_ls_i_wt()
self.tst_ls_f()
self.tst_ls_i()
self.tst_hes_ls_i_wt()
self.tst_hes_ls_f_wt()
self.tst_hes_ls_i()
self.tst_hes_ls_f()
def exercise_x1(): u_iso = flex.double([1.0,2.0,3.0,4.0,5.0]) t = tools.t_from_u_cart(u_iso, 1.e-9) assert t == (1.0, 1.0, 1.0, 0.0, 0.0, 0.0) u_iso = flex.double([7.0,2.0,3.0,9.0,5.0]) t = tools.t_from_u_cart(u_iso, 1.e-9) assert t == (2.0, 2.0, 2.0, 0.0, 0.0, 0.0)
def set_fstats(self, fstats):
self.all_spots = None
self.spots = collections.OrderedDict()
self.total_integrated_signal = {}
self.mean_integrated_signal= {}
self.median_integrated_signal= {}
self.n_spots = {}
for k in sorted(fstats.nodes.keys()):
node = fstats.nodes[k] # XXX some data in node.data will be corrupted (e.g. resolution, wts) but x and y coordinates look ok (it works later). why?
if node.descriptor == "spots_total":
self.all_spots = node.data
else:
self.spots[node.descriptor] = node.data
# Pre-calculate stats
for k in self.keys():
summed_wts = [flex.sum(spot.wts) for spot in self.get_spots(k)]
self.intensities[k] = summed_wts
self.resolutions[k] = [spot.resolution for spot in self.get_spots(k)]
total_summed = flex.sum(flex.double(summed_wts))
if len(summed_wts) > 0:
self.mean_integrated_signal[k] = total_summed / len(summed_wts)
self.median_integrated_signal[k] = flex.median(flex.double(summed_wts))
else:
self.mean_integrated_signal[k] = 0.
self.median_integrated_signal[k] = 0.
self.total_integrated_signal[k] = total_summed
self.n_spots[k] = len(summed_wts)
def __init__(self, file_name=None, file_object=None, max_header_lines=30):
assert [file_name, file_object].count(None) == 1
if (file_object is None):
file_object = open(file_name)
self._names = None
self._indices = flex.miller_index()
self._data = flex.double()
self._sigmas = flex.double()
have_data = False
self.n_lines = 0
for raw_line in file_object:
self.n_lines += 1
ifs = index_fobs_sigma_line(raw_line)
if (not ifs.is_complete):
if (raw_line.strip().lower() == "end"):
break
if (self.n_lines == max_header_lines or have_data):
raise RuntimeError, "Unkown file format."
else:
if (self._names is None): self._names = ifs.names
self._indices.append(ifs.index)
self._data.append(ifs.fobs)
self._sigmas.append(ifs.sigma)
have_data = True
if (not have_data):
raise RuntimeError, "No data found in file."
def load_result(file_name, ref_bravais_type, reference_cell, params,
reindex_op, out):
# Pull relevant information from integrated.pickle and refined_experiments.json
# files to construct the equivalent of a single integration pickle (frame).
try:
frame = frame_extractor.ConstructFrameFromFiles(
eval_ending(file_name)[2],
eval_ending(file_name)[1]).make_frame()
except Exception:
return None
# If @p file_name cannot be read, the load_result() function returns
# @c None.
print "Step 2. Load frame obj and filter on lattice & cell with", reindex_op
"""
Take a frame with all expected contents of an integration pickle, confirm
that it contains the appropriate data, and check the lattice type and unit
cell against the reference settings - if rejected, raises an exception
(for tracking statistics).
"""
# Ignore frames with no integrated reflections.
obj = frame
if ("observations" not in obj):
return None
if reindex_op == "h,k,l":
pass
else:
obj["observations"][0].apply_change_of_basis(reindex_op)
pass
result_array = obj["observations"][0]
unit_cell = result_array.unit_cell()
sg_info = result_array.space_group_info()
print >> out, ""
print >> out, "-" * 80
print >> out, file_name
print >> out, sg_info
print >> out, unit_cell
#Check for pixel size (at this point we are assuming we have square pixels, all experiments described in one
#refined_experiments.json file use the same detector, and all panels on the detector have the same pixel size)
if params.pixel_size is not None:
pixel_size = params.pixel_size
elif "pixel_size" in obj:
pixel_size = obj["pixel_size"]
else:
raise Sorry(
"Cannot find pixel size. Specify appropriate pixel size in mm for your detector in phil file."
)
#Calculate displacements based on pixel size
assert obj['mapped_predictions'][0].size() == obj["observations"][0].size()
mm_predictions = pixel_size * (obj['mapped_predictions'][0])
mm_displacements = flex.vec3_double()
cos_two_polar_angle = flex.double()
for pred in mm_predictions:
mm_displacements.append(
(pred[0] - obj["xbeam"], pred[1] - obj["ybeam"], 0.0))
cos_two_polar_angle.append(
math.cos(
2. *
math.atan2(pred[1] - obj["ybeam"], pred[0] - obj["xbeam"])))
obj["cos_two_polar_angle"] = cos_two_polar_angle
#then convert to polar angle and compute polarization correction
if (not bravais_lattice(sg_info.type().number()) == ref_bravais_type):
raise WrongBravaisError(
"Skipping cell in different Bravais type (%s)" % str(sg_info))
if (not unit_cell.is_similar_to(
other=reference_cell,
relative_length_tolerance=params.unit_cell_length_tolerance,
absolute_angle_tolerance=params.unit_cell_angle_tolerance)):
raise OutlierCellError(
"Skipping cell with outlier dimensions (%g %g %g %g %g %g" %
unit_cell.parameters())
print >> out, "Integrated data:"
result_array.show_summary(f=out, prefix=" ")
# XXX don't force reference setting here, it will be done later, after the
# original unit cell is recorded
return obj
def visualise_orientational_distribution(self,
axes_to_return=None,
cbar=True):
""" Creates a plot of the orientational distribution of the unit cells.
:param axes_to_return: if None, print to screen, otherwise, requires 3 axes objects, and will return them.
:param cbar: boolean to specify if a color bar should be used.
"""
import matplotlib.pyplot as plt
import matplotlib.patheffects as patheffects
from mpl_toolkits.basemap import Basemap
import scipy.ndimage as ndi
def cart2sph(x, y, z):
# cctbx (+z to source, y to ceiling) to
# lab frame (+x to source, z to ceiling)
z, x, y = x, y, z
dxy = np.sqrt(x**2 + y**2)
r = np.sqrt(dxy**2 + z**2)
theta = np.arctan2(y, x)
phi = np.arctan2(z,
dxy) # angle of the z axis relative to xy plane
theta, phi = np.rad2deg([theta, phi])
return theta % 360, phi, r
def xy_lat_lon_from_orientation(orientation_array, axis_id):
logger.debug("axis_id: {}".format(axis_id))
dist = math.sqrt(orientation_array[axis_id][0]**2 +
orientation_array[axis_id][1]**2 +
orientation_array[axis_id][2]**2)
flon, flat, bla = cart2sph(orientation_array[axis_id][0] / dist,
orientation_array[axis_id][1] / dist,
orientation_array[axis_id][2] / dist)
x, y = euler_map(flon, flat)
return x, y, flon, flat
orientations = [
flex.vec3_double(flex.double(image.orientation.direct_matrix()))
for image in self.members
]
space_groups = [
image.orientation.unit_cell().lattice_symmetry_group()
for image in self.members
]
# Now do all the plotting
if axes_to_return is None:
plt.figure(figsize=(10, 14))
axes_to_return = [
plt.subplot2grid((3, 1), (0, 0)),
plt.subplot2grid((3, 1), (1, 0)),
plt.subplot2grid((3, 1), (2, 0))
]
show_image = True
else:
assert len(axes_to_return) == 3, "If using axes option, must hand" \
" 3 axes to function."
show_image = False
axis_ids = [0, 1, 2]
labels = ["a", "b", "c"]
for ax, axis_id, label in zip(axes_to_return, axis_ids, labels):
# Lists of x,y,lat,long for the master orientation, and for all
# symmetry mates.
x_coords = []
y_coords = []
lon = []
lat = []
sym_x_coords = []
sym_y_coords = []
sym_lon = []
sym_lat = []
euler_map = Basemap(projection='eck4', lon_0=0, ax=ax)
for orientation, point_group_type in zip(orientations,
space_groups):
# Get position of main spots.
main_x, main_y, main_lon, main_lat \
= xy_lat_lon_from_orientation(list(orientation), axis_id)
x_coords.append(main_x)
y_coords.append(main_y)
lon.append(main_lon)
lat.append(main_lat)
# Get position of symetry mates
symmetry_operations = list(point_group_type.smx())[1:]
for mx in symmetry_operations:
rotated_orientation = list(mx.r().as_double() *
orientation) # <--
# should make sense if orientation was a vector, not clear what is
# going on since orientation is a matrix. Or, make some test cases
# with 'orientation' and see if the behave as desired.
sym_x, sym_y, sym_lo, sym_la \
= xy_lat_lon_from_orientation(rotated_orientation, axis_id)
#assert (sym_x, sym_y) != (main_x, main_y)
sym_x_coords.append(sym_x)
sym_y_coords.append(sym_y)
sym_lon.append(sym_lo)
sym_lat.append(sym_la)
# Plot each image as a yellow sphere
logger.debug(len(x_coords))
euler_map.plot(x_coords,
y_coords,
'oy',
markersize=4,
markeredgewidth=0.5)
# Plot the symetry mates as black crosses
#euler_map.plot(sym_x_coords, sym_y_coords, 'kx')
# Use a histogram to bin the data in lattitude/longitude space, smooth it,
# then plot this as a contourmap. This is for all the symetry-related
# copies
#density_hist = np.histogram2d(lat + sym_lat, lon + sym_lon,
# bins=[range(-90, 91), range(0, 361)])
# No symmetry mates until we can verify what the cctbx libs are doing
density_hist = np.histogram2d(lat,
lon,
bins=[range(-90, 91),
range(0, 361)])
smoothed = ndi.gaussian_filter(density_hist[0], (15, 15),
mode='wrap')
local_intensity = []
x_for_plot = []
y_for_plot = []
for _lat in range(0, 180):
for _lon in range(0, 360):
_x, _y = euler_map(density_hist[2][_lon],
density_hist[1][_lat])
x_for_plot.append(_x)
y_for_plot.append(_y)
local_intensity.append(smoothed[_lat, _lon])
cs = euler_map.contourf(np.array(x_for_plot),
np.array(y_for_plot),
np.array(local_intensity),
tri=True)
# Pretty up graph
if cbar:
_cbar = plt.colorbar(cs, ax=ax)
_cbar.ax.set_ylabel('spot density [AU]')
middle = euler_map(0, 0)
path_effect = [patheffects.withStroke(linewidth=3, foreground="w")]
euler_map.plot(middle[0],
middle[1],
'o',
markersize=10,
mfc='none')
euler_map.plot(middle[0], middle[1], 'x', markersize=8)
ax.annotate("beam",
xy=(0.52, 0.52),
xycoords='axes fraction',
size='medium',
path_effects=path_effect)
euler_map.drawmeridians(np.arange(0, 360, 60),
labels=[0, 0, 1, 0],
fontsize=10)
euler_map.drawparallels(np.arange(-90, 90, 30),
labels=[1, 0, 0, 0],
fontsize=10)
ax.annotate(label,
xy=(-0.05, 0.9),
xycoords='axes fraction',
size='x-large',
weight='demi')
if show_image:
plt.show()
return axes_to_return
def __init__(self,
map_1,
xray_structure,
fft_map,
atom_radius,
hydrogen_atom_radius,
model_i,
number_previous_scatters,
ignore_hd=False,
residue_detail=True,
selection=None,
pdb_hierarchy=None):
self.xray_structure = xray_structure
self.selection = selection
self.pdb_hierarchy = pdb_hierarchy
self.result = []
self.map_1_size = map_1.size()
self.map_1_stat = maptbx.statistics(map_1)
self.atoms_with_labels = None
self.residue_detail = residue_detail
self.model_i = model_i
if (pdb_hierarchy is not None):
self.atoms_with_labels = list(pdb_hierarchy.atoms_with_labels())
scatterers = self.xray_structure.scatterers()
sigma_occ = flex.double()
if (self.selection is None):
self.selection = flex.bool(scatterers.size(), True)
real_map_unpadded = fft_map.real_map_unpadded()
sites_cart = self.xray_structure.sites_cart()
if not self.residue_detail:
self.gifes = [
None,
] * scatterers.size()
self._result = [
None,
] * scatterers.size()
#
atom_radii = flex.double(scatterers.size(), atom_radius)
for i_seq, sc in enumerate(scatterers):
if (self.selection[i_seq]):
if (sc.element_symbol().strip().lower() in ["h", "d"]):
atom_radii[i_seq] = hydrogen_atom_radius
#
for i_seq, site_cart in enumerate(sites_cart):
if (self.selection[i_seq]):
sel = maptbx.grid_indices_around_sites(
unit_cell=self.xray_structure.unit_cell(),
fft_n_real=real_map_unpadded.focus(),
fft_m_real=real_map_unpadded.all(),
sites_cart=flex.vec3_double([site_cart]),
site_radii=flex.double([atom_radii[i_seq]]))
self.gifes[i_seq] = sel
m1 = map_1.select(sel)
ed1 = map_1.eight_point_interpolation(
scatterers[i_seq].site)
sigma_occ.append(ed1)
a = None
if (self.atoms_with_labels is not None):
a = self.atoms_with_labels[i_seq]
self._result[i_seq] = group_args(atom=a,
m1=m1,
ed1=ed1,
xyz=site_cart)
self.xray_structure.set_occupancies(sigma_occ)
### For testing other residue averaging options
residues = self.extract_residues(
model_i=model_i,
number_previous_scatters=number_previous_scatters)
self.xray_structure.residue_selections = residues
# Residue detail
if self.residue_detail:
assert self.pdb_hierarchy is not None
residues = self.extract_residues(
model_i=model_i,
number_previous_scatters=number_previous_scatters)
self.gifes = [
None,
] * len(residues)
self._result = [
None,
] * len(residues)
for i_seq, residue in enumerate(residues):
residue_sites_cart = sites_cart.select(residue.selection)
if 0: print(i_seq, list(residue.selection)) # DEBUG
sel = maptbx.grid_indices_around_sites(
unit_cell=self.xray_structure.unit_cell(),
fft_n_real=real_map_unpadded.focus(),
fft_m_real=real_map_unpadded.all(),
sites_cart=residue_sites_cart,
site_radii=flex.double(residue.selection.size(),
atom_radius))
self.gifes[i_seq] = sel
m1 = map_1.select(sel)
ed1 = flex.double()
for i_seq_r in residue.selection:
ed1.append(
map_1.eight_point_interpolation(
scatterers[i_seq_r].site))
self._result[i_seq] = \
group_args(residue = residue, m1 = m1, ed1 = flex.mean(ed1),
xyz=residue_sites_cart.mean(), n_atoms=residue_sites_cart.size())
residue_scatterers = scatterers.select(residue.selection)
residue_ed1 = flex.double()
for n, scatter in enumerate(residue_scatterers):
if ignore_hd:
if scatter.element_symbol() not in ['H', 'D']:
residue_ed1.append(ed1[n])
else:
residue_ed1.append(ed1[n])
for x in range(ed1.size()):
sigma_occ.append(flex.mean(residue_ed1))
self.xray_structure.set_occupancies(sigma_occ)
self.xray_structure.residue_selections = residues
del map_1
def run(self, args, command_name, out=sys.stdout):
command_line = (iotbx_option_parser(
usage="%s [options]" % command_name,
description='Example: %s data.mtz data.mtz ref_model.pdb' %
command_name).option(
None,
"--show_defaults",
action="store_true",
help="Show list of parameters.")).process(args=args)
cif_file = None
processed_args = utils.process_command_line_args(
args=args, log=sys.stdout, master_params=master_phil)
params = processed_args.params
if (params is None): params = master_phil
self.params = params.extract().ensemble_probability
pdb_file_names = processed_args.pdb_file_names
if len(pdb_file_names) != 1:
raise Sorry("Only one PDB structure may be used")
pdb_file = file_reader.any_file(pdb_file_names[0])
self.log = multi_out()
self.log.register(label="stdout", file_object=sys.stdout)
self.log.register(label="log_buffer",
file_object=StringIO(),
atexit_send_to=None)
sys.stderr = self.log
log_file = open(
pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.log', "w")
self.log.replace_stringio(old_label="log_buffer",
new_label="log",
new_file_object=log_file)
utils.print_header(command_name, out=self.log)
params.show(out=self.log)
#
f_obs = None
r_free_flags = None
reflection_files = processed_args.reflection_files
if self.params.fobs_vs_fcalc_post_nll:
if len(reflection_files) == 0:
raise Sorry(
"Fobs from input MTZ required for fobs_vs_fcalc_post_nll")
if len(reflection_files) > 0:
crystal_symmetry = processed_args.crystal_symmetry
print('Reflection file : ',
processed_args.reflection_file_names[0],
file=self.log)
utils.print_header("Model and data statistics", out=self.log)
rfs = reflection_file_server(
crystal_symmetry=crystal_symmetry,
reflection_files=processed_args.reflection_files,
log=self.log)
parameters = utils.data_and_flags_master_params().extract()
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=parameters,
data_parameter_scope="refinement.input.xray_data",
flags_parameter_scope="refinement.input.xray_data.r_free_flags",
data_description="X-ray data",
keep_going=True,
log=self.log)
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(
data=flex.bool(f_obs.data().size(), False))
# process PDB
pdb_file.assert_file_type("pdb")
#
pdb_in = hierarchy.input(file_name=pdb_file.file_name)
ens_pdb_hierarchy = pdb_in.construct_hierarchy()
ens_pdb_hierarchy.atoms().reset_i_seq()
ens_pdb_xrs_s = pdb_in.input.xray_structures_simple()
number_structures = len(ens_pdb_xrs_s)
print('Number of structure in ensemble : ',
number_structures,
file=self.log)
# Calculate sigmas from input map only
if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None:
# process MTZ
input_file = file_reader.any_file(
self.params.ensemble_sigma_map_input)
if input_file.file_type == "hkl":
if input_file.file_object.file_type() != "ccp4_mtz":
raise Sorry("Only MTZ format accepted for map input")
else:
mtz_file = input_file
else:
raise Sorry("Only MTZ format accepted for map input")
miller_arrays = mtz_file.file_server.miller_arrays
map_coeffs_1 = miller_arrays[0]
#
xrs_list = []
for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
# get sigma levels from ensemble fc for each structure
xrs = get_map_sigma(ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
map_coeffs_1=map_coeffs_1,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
log=self.log)
xrs_list.append(xrs)
# write ensemble pdb file, occupancies as sigma level
filename = pdb_file_names[0].split('/')[-1].replace(
'.pdb',
'') + '_vs_' + self.params.ensemble_sigma_map_input.replace(
'.mtz', '') + '_pensemble.pdb'
write_ensemble_pdb(filename=filename,
xrs_list=xrs_list,
ens_pdb_hierarchy=ens_pdb_hierarchy)
# Do full analysis vs Fobs
else:
model_map_coeffs = []
fmodel = None
# Get <fcalc>
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
ens_pdb_xrs.set_occupancies(1.0)
if model == 0:
# If mtz not supplied get fobs from xray structure...
# Use input Fobs for scoring against nll
if self.params.fobs_vs_fcalc_post_nll:
dummy_fobs = f_obs
else:
if f_obs == None:
if self.params.fcalc_high_resolution == None:
raise Sorry(
"Please supply high resolution limit or input mtz file."
)
dummy_dmin = self.params.fcalc_high_resolution
dummy_dmax = self.params.fcalc_low_resolution
else:
print(
'Supplied mtz used to determine high and low resolution cuttoffs',
file=self.log)
dummy_dmax, dummy_dmin = f_obs.d_max_min()
#
dummy_fobs = abs(
ens_pdb_xrs.structure_factors(
d_min=dummy_dmin).f_calc())
dummy_fobs.set_observation_type_xray_amplitude()
# If mtz supplied, free flags are over written to prevent array size error
r_free_flags = dummy_fobs.array(
data=flex.bool(dummy_fobs.data().size(), False))
#
fmodel = utils.fmodel_simple(
scattering_table="wk1995",
xray_structures=[ens_pdb_xrs],
f_obs=dummy_fobs,
target_name='ls',
bulk_solvent_and_scaling=False,
r_free_flags=r_free_flags)
f_calc_ave = fmodel.f_calc().array(
data=fmodel.f_calc().data() * 0).deep_copy()
# XXX Important to ensure scale is identical for each model and <model>
fmodel.set_scale_switch = 1.0
f_calc_ave_total = fmodel.f_calc().data().deep_copy()
else:
fmodel.update_xray_structure(xray_structure=ens_pdb_xrs,
update_f_calc=True,
update_f_mask=False)
f_calc_ave_total += fmodel.f_calc().data().deep_copy()
print('Model :', model + 1, file=self.log)
print("\nStructure vs real Fobs (no bulk solvent or scaling)",
file=self.log)
print('Rwork : %5.4f ' % fmodel.r_work(),
file=self.log)
print('Rfree : %5.4f ' % fmodel.r_free(),
file=self.log)
print('K1 : %5.4f ' % fmodel.scale_k1(),
file=self.log)
fcalc_edm = fmodel.electron_density_map()
fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type='Fc')
fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset(
column_root_label='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_mtz_dataset.mtz_object().write(
file_name=str(model + 1) + "_Fc.mtz")
model_map_coeffs.append(fcalc_map_coeffs.deep_copy())
fmodel.update(f_calc=f_calc_ave.array(f_calc_ave_total /
number_structures))
print("\nEnsemble vs real Fobs (no bulk solvent or scaling)",
file=self.log)
print('Rwork : %5.4f ' % fmodel.r_work(), file=self.log)
print('Rfree : %5.4f ' % fmodel.r_free(), file=self.log)
print('K1 : %5.4f ' % fmodel.scale_k1(), file=self.log)
# Get <Fcalc> map
fcalc_ave_edm = fmodel.electron_density_map()
fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients(
map_type='Fc').deep_copy()
fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset(
column_root_label='Fc')
if self.params.output_model_and_model_ave_mtz:
fcalc_ave_mtz_dataset.mtz_object().write(file_name="aveFc.mtz")
fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map()
fcalc_ave_map_coeffs.apply_volume_scaling()
fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded()
fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data)
print("<Fcalc> Map Stats :", file=self.log)
fcalc_ave_map_stats.show_summary(f=self.log)
offset = fcalc_ave_map_stats.min()
model_neg_ll = []
number_previous_scatters = 0
# Run through structure list again and get probability
xrs_list = []
for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s):
if self.params.verbose:
print('\n\nModel : ',
model + 1,
file=self.log)
# Get model atom sigmas vs Fcalc
fcalc_map = model_map_coeffs[model].fft_map()
fcalc_map.apply_volume_scaling()
fcalc_map_data = fcalc_map.real_map_unpadded()
fcalc_map_stats = maptbx.statistics(fcalc_map_data)
if self.params.verbose:
print("Fcalc map stats :", file=self.log)
fcalc_map_stats.show_summary(f=self.log)
xrs = get_map_sigma(
ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
fft_map_1=fcalc_map,
model_i=model,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
number_previous_scatters=number_previous_scatters,
log=self.log)
fcalc_sigmas = xrs.scatterers().extract_occupancies()
del fcalc_map
# Get model atom sigmas vs <Fcalc>
xrs = get_map_sigma(
ens_pdb_hierarchy=ens_pdb_hierarchy,
ens_pdb_xrs=ens_pdb_xrs,
fft_map_1=fcalc_ave_map_coeffs,
model_i=model,
residue_detail=self.params.residue_detail,
ignore_hd=self.params.ignore_hd,
number_previous_scatters=number_previous_scatters,
log=self.log)
### For testing other residue averaging options
#print xrs.residue_selections
fcalc_ave_sigmas = xrs.scatterers().extract_occupancies()
# Probability of model given <model>
prob = fcalc_ave_sigmas / fcalc_sigmas
# XXX debug option
if False:
for n, p in enumerate(prob):
print(' {0:5d} {1:5.3f}'.format(n, p), file=self.log)
# Set probabilty between 0 and 1
# XXX Make Histogram / more stats
prob_lss_zero = flex.bool(prob <= 0)
prob_grt_one = flex.bool(prob > 1)
prob.set_selected(prob_lss_zero, 0.001)
prob.set_selected(prob_grt_one, 1.0)
xrs.set_occupancies(prob)
xrs_list.append(xrs)
sum_neg_ll = sum(-flex.log(prob))
model_neg_ll.append((sum_neg_ll, model))
if self.params.verbose:
print('Model probability stats :', file=self.log)
print(prob.min_max_mean().show(), file=self.log)
print(' Count < 0.0 : ',
prob_lss_zero.count(True),
file=self.log)
print(' Count > 1.0 : ',
prob_grt_one.count(True),
file=self.log)
# For averaging by residue
number_previous_scatters += ens_pdb_xrs.sites_cart().size()
# write ensemble pdb file, occupancies as sigma level
write_ensemble_pdb(
filename=pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.pdb',
xrs_list=xrs_list,
ens_pdb_hierarchy=ens_pdb_hierarchy)
# XXX Test ordering models by nll
# XXX Test removing nth percentile atoms
if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll:
for percentile in [1.0, 0.975, 0.95, 0.9, 0.8, 0.6, 0.2]:
model_neg_ll = sorted(model_neg_ll)
f_calc_ave_total_reordered = None
print_list = []
for i_neg_ll in model_neg_ll:
xrs = xrs_list[i_neg_ll[1]]
nll_occ = xrs.scatterers().extract_occupancies()
# Set q=0 nth percentile atoms
sorted_nll_occ = sorted(nll_occ, reverse=True)
number_atoms = len(sorted_nll_occ)
percentile_prob_cutoff = sorted_nll_occ[
int(number_atoms * percentile) - 1]
cutoff_selections = flex.bool(
nll_occ < percentile_prob_cutoff)
cutoff_nll_occ = flex.double(nll_occ.size(),
1.0).set_selected(
cutoff_selections,
0.0)
#XXX Debug
if False:
print('\nDebug')
for x in range(len(cutoff_selections)):
print(cutoff_selections[x], nll_occ[x],
cutoff_nll_occ[x])
print(percentile)
print(percentile_prob_cutoff)
print(cutoff_selections.count(True))
print(cutoff_selections.size())
print(cutoff_nll_occ.count(0.0))
print('Count q = 1 : ',
cutoff_nll_occ.count(1.0))
print('Count scatterers size : ',
cutoff_nll_occ.size())
xrs.set_occupancies(cutoff_nll_occ)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True,
update_f_mask=True)
if f_calc_ave_total_reordered == None:
f_calc_ave_total_reordered = fmodel.f_calc().data(
).deep_copy()
f_mask_ave_total_reordered = fmodel.f_masks(
)[0].data().deep_copy()
cntr = 1
else:
f_calc_ave_total_reordered += fmodel.f_calc().data(
).deep_copy()
f_mask_ave_total_reordered += fmodel.f_masks(
)[0].data().deep_copy()
cntr += 1
fmodel.update(
f_calc=f_calc_ave.array(
f_calc_ave_total_reordered / cntr).deep_copy(),
f_mask=f_calc_ave.array(
f_mask_ave_total_reordered / cntr).deep_copy())
# Update solvent and scale
# XXX Will need to apply_back_trace on latest version
fmodel.set_scale_switch = 0
fmodel.update_all_scales()
# Reset occ for outout
xrs.set_occupancies(nll_occ)
# k1 updated vs Fobs
if self.params.fobs_vs_fcalc_post_nll:
print_list.append([
cntr, i_neg_ll[0], i_neg_ll[1],
fmodel.r_work(),
fmodel.r_free()
])
# Order models by nll and print summary
print(
'\nModels ranked by nll <Fcalc> R-factors recalculated',
file=self.log)
print('Percentile cutoff : {0:5.3f}'.format(percentile),
file=self.log)
xrs_list_sorted_nll = []
print(' | NLL <Rw> <Rf> Ens Model',
file=self.log)
for info in print_list:
print(' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'.
format(
info[0],
info[1],
info[3],
info[4],
info[2] + 1,
),
file=self.log)
xrs_list_sorted_nll.append(xrs_list[info[2]])
# Output nll ordered ensemble
write_ensemble_pdb(
filename='nll_ordered_' +
pdb_file_names[0].split('/')[-1].replace('.pdb', '') +
'_pensemble.pdb',
xrs_list=xrs_list_sorted_nll,
ens_pdb_hierarchy=ens_pdb_hierarchy)
assume_index_matching=True)
r1_factor_bin.show()
r1_factor = ref_match.r1_factor(obs_match,
use_binning=False,
assume_index_matching=True)
print 'Overall R-factor:', r1_factor
#calculate r_factor for partiality-corrected scaled data
pixel_size_mm = 0.079346
crystal_init_orientation = observations_pickle["current_orientation"][0]
wavelength = observations_pickle["wavelength"]
detector_distance_mm = observations_pickle['distance']
mm_predictions = pixel_size_mm * (
observations_pickle['mapped_predictions'][0])
xbeam = observations_pickle["xbeam"]
ybeam = observations_pickle["ybeam"]
alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \
for pred in mm_predictions])
spot_pred_x_mm = flex.double([pred[0] - xbeam for pred in mm_predictions])
spot_pred_y_mm = flex.double([pred[1] - ybeam for pred in mm_predictions])
ph = partiality_handler()
r0 = ph.calc_spot_radius(sqr(crystal_init_orientation.reciprocal_matrix()),
observations.indices(), wavelength)
ry, rz, re, voigt_nu, rotx, roty = (0, 0, 0.003, 0.5, 0, 0)
partiality_init, delta_xy_init, rs_init, rh_init = ph.calc_partiality_anisotropy_set(\
crystal_init_orientation.unit_cell(),
rotx, roty, observations.indices(),
ry, rz, r0, re, voigt_nu,
two_theta, alpha_angle_obs, wavelength,
crystal_init_orientation,
spot_pred_x_mm, spot_pred_y_mm,
detector_distance_mm, "Voigt",
False)
if (command_line.options.use_model):
crystal_symmetry = crystal_symmetry_from_pdb.extract_from(
file_name=command_line.options.use_model)
if (crystal_symmetry.unit_cell() is None
or crystal_symmetry.space_group_info() is None):
raise Sorry(
"Crystal symmetry is not defined. Please use the --symmetry option.\n"
"Type %s without arguments to see more options." % command_name)
if (len(command_line.args) > 1):
print "%d arguments are given from the command line:"% \
len(command_line.args), command_line.args
raise Sorry("Please specify one reflection csv file.")
file_name = command_line.args[0]
if (not os.path.isfile(file_name)):
raise Sorry("File is not found: %s" % file_name)
data = flex.double()
sigmas = flex.double()
flags = flex.int()
column_ids, columns = parse_csv_file(file_name=file_name)
data_label_root = column_ids[3]
ms = miller.set(crystal_symmetry, flex.miller_index(columns[0]))
for d in columns[1]:
data.append(float(d))
for sig in columns[2]:
sigmas.append(float(sig))
for flag in columns[3]:
flags.append(int(flag))
assert len(data) == len(sigmas)
assert len(flags) == len(data)
ma = miller.array(ms, data, sigmas)
if data_label_root.startswith('F'):
def exercise_sort():
expected_unsorted_data = """ 4 I 4
1 0 0 0 1 0 0 0 1
0 0 0
0 -1 0 1 0 0 0 0 1
0 0 0
0 1 0 -1 0 0 0 0 1
0 0 0
-1 0 0 0 -1 0 0 0 1
0 0 0
0 0 4 0 0 4 0 1 0 1 80331.8 8648.0
0 0 12 0 0 12 0 1 0 1104526.7 11623.3
0 0 -4 0 0 4 0 2 0 1 44883.6 4527.6
0 0 -8 0 0 8 0 2 0 1 41134.1 4431.9
0 0 8 0 0 8 0 1 0 1 50401.8 5464.6
0 0 8 0 0 8 0 1 0 1 53386.0 5564.1
0 0 -8 0 0 8 0 2 0 1119801.4 12231.2
0 0 12 0 0 12 0 1 0 1105312.9 12602.2
0 0 16 0 0 16 0 1 0 1 14877.6 2161.5
"""
expected_sorted_data = """ 4 I 4
1 0 0 0 1 0 0 0 1
0 0 0
0 -1 0 1 0 0 0 0 1
0 0 0
0 1 0 -1 0 0 0 0 1
0 0 0
-1 0 0 0 -1 0 0 0 1
0 0 0
0 0 4 0 0 4 0 1 0 1 80331.8 8648.0
0 0 -4 0 0 4 0 2 0 1 44883.6 4527.6
0 0 8 0 0 8 0 1 0 1 50401.8 5464.6
0 0 8 0 0 8 0 1 0 1 53386.0 5564.1
0 0 -8 0 0 8 0 2 0 1 41134.1 4431.9
0 0 -8 0 0 8 0 2 0 1119801.4 12231.2
0 0 12 0 0 12 0 1 0 1104526.7 11623.3
0 0 12 0 0 12 0 1 0 1105312.9 12602.2
0 0 16 0 0 16 0 1 0 1 14877.6 2161.5
"""
from cctbx.xray import observation_types
from cctbx.array_family import flex
xs = crystal.symmetry((113.949, 113.949, 32.474, 90.000, 90.000, 90.000),
"I4")
mi = flex.miller_index((
(0, 0, 4),
(0, 0, 12),
(0, 0, -4),
(0, 0, -8),
(0, 0, 8),
(0, 0, 8),
(0, 0, -8),
(0, 0, 12),
(0, 0, 16),
))
data = flex.double((
80331.8,
104526.7,
44883.6,
41134.1,
50401.8,
53386.0,
119801.4,
105312.9,
14877.6,
))
sigmas = flex.double((
8648.0,
11623.3,
4527.6,
4431.9,
5464.6,
5564.1,
12231.2,
12602.2,
2161.5,
))
ms = miller.set(xs, mi, anomalous_flag=True)
i_obs = miller.array(ms, data=data, sigmas=sigmas).set_observation_type(
observation_types.intensity())
i_obs.export_as_scalepack_unmerged(
file_name='unsorted.sca', scale_intensities_for_scalepack_merge=True)
i_obs = i_obs.sort(by_value="asu_indices")
i_obs.export_as_scalepack_unmerged(
file_name='sorted.sca', scale_intensities_for_scalepack_merge=True)
with open('unsorted.sca') as f:
unsorted = f.read()
with open('sorted.sca') as f:
sorted = f.read()
assert unsorted == expected_unsorted_data
assert sorted == expected_sorted_data
def contour_plot_DEPRECATED_DOES_NOT_APPLY_SUBPIXEL_METROLOGY(OO):
self = OO.parent
# see if I can reproduce the predicted positions
pxlsz = self.pixel_size # mm/pixel
SIGN = -1.
# get a simple rmsd obs vs predicted
sumsq = 0.
nspot = 0
for pair in self.indexed_pairs:
deltax = self.spots[pair["spot"]].ctr_mass_x() - self.predicted[pair["pred"]][0]/pxlsz
deltay = self.spots[pair["spot"]].ctr_mass_y() - self.predicted[pair["pred"]][1]/pxlsz
sumsq += deltax*deltax + deltay*deltay
nspot+=1
print "RMSD obs vs pred in pixels: %7.2f"%(math.sqrt(sumsq/nspot))
excursi = flex.double()
rmsdpos = flex.double()
for irotx in OO.grid:
rotx = (0.02 * irotx) * math.pi/180.
for iroty in OO.grid:
roty = (0.02 * iroty) * math.pi/180.
#Not necessary to apply the 3 offset rotations; they have apparently
# been applied already. rotz (0,0,1) is the direct beam
effective_orientation = OO.input_orientation.rotate_thru((1,0,0),rotx
).rotate_thru((0,1,0),roty
).rotate_thru((0,0,1),0.0)
OO.ucbp3.set_orientation(effective_orientation)
OO.ucbp3.gaussian_fast_slow()
mean_position = OO.ucbp3.mean_position
print mean_position
sumsq = 0.
nspot = 0
for pair in self.indexed_pairs:
deltax = mean_position[nspot][1] - self.predicted[pair["pred"]][0]/pxlsz
deltay = mean_position[nspot][0] - self.predicted[pair["pred"]][1]/pxlsz
sumsq += deltax*deltax + deltay*deltay
nspot+=1
print "RMSD markmodel vs rossmanpred in pixels: %7.2f"%(math.sqrt(sumsq/nspot))
#from matplotlib import pyplot as plt
#plt.plot([mpos[0] for mpos in mean_position],[mpos[1] for mpos in mean_position],"r+")
#plt.plot([self.predicted[pair["pred"]][1]/pxlsz for pair in indexed_pairs],
# [self.predicted[pair["pred"]][0]/pxlsz for pair in indexed_pairs], "b.")
#plt.show()
sumsq = 0.
nspot = 0
for pair in self.indexed_pairs:
deltax = self.spots[pair["spot"]].ctr_mass_x() - mean_position[nspot][1]
deltay = self.spots[pair["spot"]].ctr_mass_y() - mean_position[nspot][0]
sumsq += deltax*deltax + deltay*deltay
nspot+=1
rmsdposition = math.sqrt(sumsq/nspot)
rmsdpos.append(rmsdposition)
print "RMSD obs vs markmodel in pixels: %8.4f"%(rmsdposition)
excursions = flex.double(
[OO.ucbp3.simple_forward_calculation_spot_position(
wavelength = OO.central_wavelength_ang,
observation_no = obsno).rotax_excursion_rad*180./math.pi
for obsno in xrange(len(self.indexed_pairs))])
rmsdexc = math.sqrt(flex.mean(excursions*excursions))
excursi.append(rmsdexc)
print "rotx %7.2f roty %7.2f degrees, RMSD excursion %7.3f degrees"%(
(0.02 * irotx),(0.02 * iroty), rmsdexc)
return excursi,rmsdpos
def refine_rotx_roty2(OO,enable_rotational_target=True):
helper = OO.per_frame_helper_factory()
helper.restart()
if enable_rotational_target:
print "Trying least squares minimization of excursions",
from scitbx.lstbx import normal_eqns_solving
iterations = normal_eqns_solving.naive_iterations(
non_linear_ls = helper,
gradient_threshold = 1.E-10)
results = helper.x
print "with %d reflections"%len(OO.parent.indexed_pairs),
print "result %6.2f degrees"%(results[1]*180./math.pi),
print "result %6.2f degrees"%(results[0]*180./math.pi)
if False: # Excursion histogram
print "The input mosaicity is %7.3f deg full width"%OO.parent.inputai.getMosaicity()
# final histogram
if OO.pvr_fix:
final = 360.* helper.fvec_callable_pvr(results)
else:
final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)
rmsdexc = math.sqrt(flex.mean(final*final))
from matplotlib import pyplot as plt
nbins = len(final)//20
n,bins,patches = plt.hist(final,
nbins, normed=0, facecolor="orange", alpha=0.75)
plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg"%rmsdexc)
plt.title("Histogram of cctbx.xfel misorientation")
plt.axis([-0.5,0.5,0,100])
plt.plot([rmsdexc],[18],"b|")
plt.show()
# Determine optimal mosaicity and domain size model (monochromatic)
if OO.pvr_fix:
final = 360.* helper.fvec_callable_pvr(results)
else:
final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)
#Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned
guard_stats = flex.max(final), flex.min(final)
if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS (guard_stats[0] > 2.0 or guard_stats[1] < -2.0):
raise Exception("Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)");
print "The mean excursion is %7.3f degrees"%(flex.mean(final))
two_thetas = helper.last_set_orientation.unit_cell().two_theta(OO.reserve_indices,OO.central_wavelength_ang,deg=True)
dspacings = helper.last_set_orientation.unit_cell().d(OO.reserve_indices)
dspace_sq = dspacings * dspacings
excursion_rad = final * math.pi/ 180.
# First -- try to get a reasonable envelope for the observed excursions.
## minimum of three regions; maximum of 50 measurements in each bin
print "fitting parameters on %d spots"%len(excursion_rad)
n_bins = min(max(3, len(excursion_rad)//25),50)
bin_sz = len(excursion_rad)//n_bins
print "nbins",n_bins,"bin_sz",bin_sz
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in xrange(0,n_bins):
subset = order[x*bin_sz:(x+1)*bin_sz]
two_thetas_env.append( flex.mean(two_thetas.select(subset)) )
dspacings_env.append( flex.mean(dspacings.select(subset)))
excursion_rads_env.append( flex.max( flex.abs( excursion_rad.select(subset))))
# Second -- parameter fit
## solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1./(2*solution[0])
print "Best LSQ fit Scheerer domain size is %9.2f ang"%(
s_ang)
tan_phi_rad = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. * s_ang)
tan_phi_deg = tan_phi_rad * 180./math.pi
k_degrees = solution[1]* 180./math.pi
print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees)
tan_outer_deg = tan_phi_deg + k_degrees
if OO.mosaic_refinement_target=="ML":
from xfel.mono_simulation.max_like import minimizer
print "input", s_ang,2. * solution[1]*180/math.pi
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = math.pow(
helper.last_set_orientation.unit_cell().volume(),
1./3.)*20 # 10-unit cell block size minimum reasonable domain
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]),
Deff = d_estimate)
print "output",1./M.x[0], M.x[1]*180./math.pi
tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi
# bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero)
tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi
if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic:
# add code here to perform polychromatic modeling.
"""
get miller indices DONE
get model-predicted mono-wavelength centroid S1 vectors
back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction
compare with spot centroid measured position
compare with locus of bodypixels
"""
print list(OO.reserve_indices)
print len(OO.reserve_indices), len(two_thetas)
positions = [
OO.ucbp3.simple_forward_calculation_spot_position(
wavelength = OO.central_wavelength_ang,
observation_no = obsno).position
for obsno in xrange(len(OO.parent.indexed_pairs))]
print len(positions)
print positions # model-predicted positions
print len(OO.parent.spots)
print OO.parent.indexed_pairs
print OO.parent.spots
print len(OO.parent.spots)
meas_spots = [OO.parent.spots[pair["spot"]] for pair in OO.parent.indexed_pairs]
# for xspot in meas_spots:
# xspot.ctr_mass_x(),xspot.ctr_mass_y()
# xspot.max_pxl_x()
# xspot.bodypixels
# xspot.ctr_mass_x()
# Do some work to calculate an rmsd
diff_vecs = flex.vec3_double()
for p,xspot in zip(positions, meas_spots):
diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0))
# could use diff_vecs.rms_length()
diff_vecs_sq = diff_vecs.dot(diff_vecs)
mean_diff_vec_sq = flex.mean(diff_vecs_sq)
rmsd = math.sqrt(mean_diff_vec_sq)
print "mean obs-pred diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd)
positions_to_fictitious = [
OO.ucbp3.simple_forward_calculation_spot_position(
wavelength = OO.central_wavelength_ang,
observation_no = obsno).position_to_fictitious
for obsno in xrange(len(OO.parent.indexed_pairs))]
# Do some work to calculate an rmsd
diff_vecs = flex.vec3_double()
for p,xspot in zip(positions_to_fictitious, meas_spots):
diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0))
rmsd = diff_vecs.rms_length()
print "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd)
"""
actually, it might be better if the entire set of experimental observations
is transformed into the ideal detector plane, for the purposes of poly_treatment.
start here. Now it would be good to actually implement probability of observing a body pixel given the model.
We have everything needed right here.
"""
if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B:
# Image plot: obs and predicted positions + bodypixels
from matplotlib import pyplot as plt
plt.plot( [p[0] for p in positions_to_fictitious], [p[1] for p in positions_to_fictitious], "r.")
plt.plot( [xspot.ctr_mass_y() for xspot in meas_spots],
[xspot.ctr_mass_x() for xspot in meas_spots], "g.")
bodypx = []
for xspot in meas_spots:
for body in xspot.bodypixels:
bodypx.append(body)
plt.plot( [b.y for b in bodypx], [b.x for b in bodypx], "b.")
plt.axes().set_aspect("equal")
plt.show()
print "MEAN excursion",flex.mean(final),
if OO.mosaic_refinement_target=="ML":
print "mosaicity deg FW=",M.x[1]*180./math.pi
else:
print
if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.
AD1TF7B_MAXDP = 1.
from matplotlib import pyplot as plt
fig = plt.figure()
plt.plot(two_thetas, final, "bo")
mean = flex.mean(final)
minplot = flex.min(two_thetas)
plt.plot([0,minplot],[mean,mean],"k-")
LR = flex.linear_regression(two_thetas, final)
#LR.show_summary()
model_y = LR.slope()*two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
print helper.last_set_orientation.unit_cell()
#for sdp,tw in zip (dspacings,two_thetas):
#print sdp,tw
if OO.mosaic_refinement_target=="ML":
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
else:
plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r|")
plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r|")
plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r-")
plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r-")
plt.plot(two_thetas, tan_phi_deg, "r.")
plt.plot(two_thetas, -tan_phi_deg, "r.")
plt.plot(two_thetas, tan_outer_deg, "g.")
plt.plot(two_thetas, -tan_outer_deg, "g.")
plt.xlim([0,AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP])
OO.parent.show_figure(plt,fig,"psi")
plt.close()
from xfel.mono_simulation.util import green_curve_area,ewald_proximal_volume
if OO.mosaic_refinement_target=="ML":
OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
OO.parent.inputai.setMosaicity(M.x[1]*180./math.pi) # full width, degrees
OO.parent.ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi)
OO.parent.ML_domain_size_ang = 1./M.x[0]
OO.parent.ewald_proximal_volume = ewald_proximal_volume(
wavelength_ang = OO.central_wavelength_ang,
resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff,
domain_size_ang = 1./M.x[0],
full_mosaicity_rad = M.x[1])
return results, helper.last_set_orientation,1./M.x[0] # full width domain size, angstroms
else:
assert OO.mosaic_refinement_target=="LSQ"
OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg)
OO.parent.inputai.setMosaicity(2*k_degrees) # full width
OO.parent.ML_half_mosaicity_deg = k_degrees
OO.parent.ML_domain_size_ang = s_ang
OO.parent.ewald_proximal_volume = ewald_proximal_volume(
wavelength_ang = OO.central_wavelength_ang,
resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff,
domain_size_ang = s_ang,
full_mosaicity_rad = 2*k_degrees*math.pi/180.)
return results, helper.last_set_orientation,s_ang # full width domain size, angstroms
def integration_concept(self,image_number=0,cb_op_to_primitive=None,verbose=False,**kwargs):
self.image_number = image_number
NEAR = 10
pxlsz = self.pixel_size
self.get_predictions_accounting_for_centering(cb_op_to_primitive,**kwargs)
FWMOSAICITY = self.inputai.getMosaicity()
DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual",0) )
refineflag = {True:0,False:1}[kwargs.get("domain_size_ang",0)==0]
self.inputpd["symmetry"].show_summary(prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f "%(refineflag,FWMOSAICITY,DOMAIN_SZ_ANG))
from annlib_ext import AnnAdaptor
self.cell = self.inputai.getOrientation().unit_cell()
query = flex.double()
for pred in self.predicted: # predicted spot coord in pixels
query.append(pred[0]/pxlsz)
query.append(pred[1]/pxlsz)
self.reserve_hkllist_for_signal_search = self.hkllist
reference = flex.double()
spots = self.get_observations_with_outlier_removal()
assert len(spots)>NEAR# Can't do spot/pred matching with too few spots
for spot in spots:
reference.append(spot.ctr_mass_x())
reference.append(spot.ctr_mass_y())
IS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
IS_adapt.query(query)
print "Calculate correction vectors for %d observations & %d predictions"%(len(spots),len(self.predicted))
indexed_pairs_provisional = []
correction_vectors_provisional = []
c_v_p_flex = flex.vec3_double()
idx_cutoff = float(min(self.mask_focus[image_number]))
if verbose:
print "idx_cutoff distance in pixels",idx_cutoff
if not self.horizons_phil.integration.enable_one_to_one_safeguard:
# legacy code, no safeguard against many-to-one predicted-to-observation mapping
for i in xrange(len(self.predicted)): # loop over predicteds
#for n in xrange(NEAR): # loop over near spotfinder spots
for n in xrange(1): # only consider the nearest spotfinder spot
Match = dict(spot=IS_adapt.nn[i*NEAR+n],pred=i)
if n==0 and math.sqrt(IS_adapt.distances[i*NEAR+n]) < idx_cutoff:
indexed_pairs_provisional.append(Match)
vector = matrix.col(
[spots[Match["spot"]].ctr_mass_x() - self.predicted[Match["pred"]][0]/pxlsz,
spots[Match["spot"]].ctr_mass_y() - self.predicted[Match["pred"]][1]/pxlsz])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0],vector[1],0.))
else:
one_to_one = {}
for i in xrange(len(self.predicted)): # loop over predicteds
annresultidx = i*NEAR
obsidx = IS_adapt.nn[annresultidx]
this_distancesq = IS_adapt.distances[annresultidx]
if not one_to_one.has_key(obsidx) or \
this_distancesq < one_to_one[obsidx]["distancesq"]:
if math.sqrt(this_distancesq) < idx_cutoff:
one_to_one[obsidx] = dict(spot=obsidx,pred=i,distancesq=this_distancesq)
for key,value in one_to_one.items():
indexed_pairs_provisional.append(value)
vector = matrix.col(
[spots[value["spot"]].ctr_mass_x() - self.predicted[value["pred"]][0]/pxlsz,
spots[value["spot"]].ctr_mass_y() - self.predicted[value["pred"]][1]/pxlsz])
correction_vectors_provisional.append(vector)
c_v_p_flex.append((vector[0],vector[1],0.))
print "... %d provisional matches"%len(correction_vectors_provisional),
print "r.m.s.d. in pixels: %5.2f"%(math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))
if self.horizons_phil.integration.enable_residual_scatter:
from matplotlib import pyplot as plt
fig = plt.figure()
for cv in correction_vectors_provisional:
plt.plot([cv[1]],[-cv[0]],"b.")
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"res")
plt.close()
if self.horizons_phil.integration.enable_residual_map:
from matplotlib import pyplot as plt
fig = plt.figure()
for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional):
plt.plot([spots[match["spot"]].ctr_mass_y()],[-spots[match["spot"]].ctr_mass_x()],"r.")
plt.plot([self.predicted[match["pred"]][1]/pxlsz],[-self.predicted[match["pred"]][0]/pxlsz],"g.")
plt.plot([spots[match["spot"]].ctr_mass_y(), spots[match["spot"]].ctr_mass_y() + 10.*cv[1]],
[-spots[match["spot"]].ctr_mass_x(), -spots[match["spot"]].ctr_mass_x() - 10.*cv[0]],'b-')
plt.xlim([0,float(self.inputpd["size2"])])
plt.ylim([-float(self.inputpd["size1"]),0])
plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))))
plt.axes().set_aspect("equal")
self.show_figure(plt,fig,"map")
plt.close()
# insert code here to remove correction length outliers...
# they are causing terrible
# problems for finding legitimate correction vectors (print out the list)
# also remove outliers for the purpose of reporting RMS
outlier_rejection = True
cache_refinement_spots = getattr(slip_callbacks.slip_callback,"requires_refinement_spots",False)
if outlier_rejection:
correction_lengths = flex.double([v.length() for v in correction_vectors_provisional])
clorder = flex.sort_permutation(correction_lengths)
sorted_cl = correction_lengths.select(clorder)
ACCEPTABLE_LIMIT = 2
limit = int(0.33 * len(sorted_cl)) # best 1/3 of data are assumed to be correctly modeled.
if (limit <= ACCEPTABLE_LIMIT):
raise Sorry("Not enough indexed spots to reject outliers; have %d need >%d" % (limit, ACCEPTABLE_LIMIT))
y_data = flex.double(len(sorted_cl))
for i in xrange(len(y_data)):
y_data[i] = float(i)/float(len(y_data))
# ideas are explained in Sauter & Poon (2010) J Appl Cryst 43, 611-616.
from rstbx.outlier_spots.fit_distribution import fit_cdf,rayleigh
fitted_rayleigh = fit_cdf(x_data = sorted_cl[0:limit],
y_data = y_data[0:limit],
distribution=rayleigh)
inv_cdf = [fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data]
#print "SORTED LIST OF ",len(sorted_cl), "with sigma",fitted_rayleigh.distribution.sigma
indexed_pairs = []
correction_vectors = []
self.correction_vectors = []
for icand in xrange(len(sorted_cl)):
# somewhat arbitrary sigma = 1.0 cutoff for outliers
if (sorted_cl[icand]-inv_cdf[icand])/fitted_rayleigh.distribution.sigma > 1.0:
break
indexed_pairs.append(indexed_pairs_provisional[clorder[icand]])
correction_vectors.append(correction_vectors_provisional[clorder[icand]])
if cache_refinement_spots:
self.spotfinder.images[self.frame_numbers[self.image_number]]["refinement_spots"].append(
spots[indexed_pairs[-1]["spot"]])
if kwargs.get("verbose_cv")==True:
print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f"%(
float(self.inputpd["size1"])/2.,float(self.inputpd["size2"])/2.,
self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz),
print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f"%(
spots[indexed_pairs[-1]["spot"]].ctr_mass_x(),
spots[indexed_pairs[-1]["spot"]].ctr_mass_y(),
self.predicted[indexed_pairs[-1]["pred"]][0]/pxlsz,
self.predicted[indexed_pairs[-1]["pred"]][1]/pxlsz),
the_hkl = self.hkllist[indexed_pairs[-1]["pred"]]
print "HKL %4d %4d %4d"%the_hkl,"%2d"%self.setting_id,
radial, azimuthal = spots[indexed_pairs[-1]["spot"]].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)
print "RADIALpx %5.3f AZIMUTpx %5.3f"%(radial,azimuthal)
# Store a list of correction vectors in self.
radial, azimuthal = spots[indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size(
self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)
self.correction_vectors.append(
dict(obscenter=(float(self.inputpd['size1']) / 2,
float(self.inputpd['size2']) / 2),
refinedcenter=(self.inputai.xbeam() / pxlsz,
self.inputai.ybeam() / pxlsz),
obsspot=(spots[indexed_pairs[-1]['spot']].ctr_mass_x(),
spots[indexed_pairs[-1]['spot']].ctr_mass_y()),
predspot=(self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz,
self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz),
hkl=(self.hkllist[indexed_pairs[-1]['pred']][0],
self.hkllist[indexed_pairs[-1]['pred']][1],
self.hkllist[indexed_pairs[-1]['pred']][2]),
setting_id=self.setting_id,
radial=radial,
azimuthal=azimuthal))
print "After outlier rejection %d indexed spotfinder spots remain."%len(indexed_pairs)
if False:
rayleigh_cdf = [
fitted_rayleigh.distribution.cdf(x=sorted_cl[c]) for c in xrange(len(sorted_cl))]
from matplotlib import pyplot as plt
plt.plot(sorted_cl,y_data,"r+")
#plt.plot(sorted_cl,rayleigh_cdf,"g.")
plt.plot(inv_cdf,y_data,"b.")
plt.show()
else:
indexed_pairs = indexed_pairs_provisional
correction_vectors = correction_vectors_provisional
########### finished with outlier rejection
self.inputpd["symmetry"].show_summary(prefix="SETTING ")
is_triclinic = (self.setting_id==1)
if is_triclinic:
self.triclinic_pairs = [ dict(pred=self.hkllist[a["pred"]],spot=a["spot"])
for a in indexed_pairs ]
if self.horizons_phil.integration.model == "user_supplied":
if kwargs.get("user-reentrant",None)==None:
from cxi_user import post_outlier_rejection
self.indexed_pairs = indexed_pairs
self.spots = spots
post_outlier_rejection(self,image_number,cb_op_to_primitive,self.horizons_phil,kwargs)
return
########### finished with user-supplied code
if self.horizons_phil.integration.spot_shape_verbose:
from rstbx.new_horizons.spot_shape import spot_shape_verbose
spot_shape_verbose(rawdata = self.imagefiles.images[self.image_number].linearintdata,
beam_center_pix = matrix.col((self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)),
indexed_pairs = indexed_pairs,
spotfinder_observations = spots,
distance_mm = self.inputai.distance(),
mm_per_pixel = pxlsz,
hkllist = self.hkllist,
unit_cell = self.cell,
wavelength_ang = self.inputai.wavelength
)
#Other checks to be implemented (future):
# spot is within active area of detector on a circular detector such as the Mar IP
# integration masks do not overlap; or deconvolute
correction_lengths=flex.double([v.length() for v in correction_vectors])
if verbose:
print "average correction %5.2f over %d vectors"%(flex.mean(correction_lengths),
len(correction_lengths)),
print "or %5.2f mm."%(pxlsz*flex.mean(correction_lengths))
self.r_residual = pxlsz*flex.mean(correction_lengths)
#assert len(indexed_pairs)>NEAR # must have enough indexed spots
if (len(indexed_pairs) <= NEAR):
raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR))
reference = flex.double()
for item in indexed_pairs:
reference.append(spots[item["spot"]].ctr_mass_x())
reference.append(spots[item["spot"]].ctr_mass_y())
PS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR)
PS_adapt.query(query)
self.BSmasks = []
#self.null_correction_mapping( predicted=self.predicted,
# correction_vectors = correction_vectors,
# IS_adapt = IS_adapt,
# spots = spots)
self.positional_correction_mapping( predicted=self.predicted,
correction_vectors = correction_vectors,
PS_adapt = PS_adapt,
IS_adapt = IS_adapt,
spots = spots)
# which spots are close enough to interfere with background?
MAXOVER=6
OS_adapt = AnnAdaptor(data=query,dim=2,k=MAXOVER) #six near nbrs
OS_adapt.query(query)
if self.mask_focus[image_number] is None:
raise Sorry("No observed/predicted spot agreement; no Spotfinder masks; skip integration")
nbr_cutoff = 2.0* max(self.mask_focus[image_number])
FRAME = int(nbr_cutoff/2)
#print "The overlap cutoff is %d pixels"%nbr_cutoff
nbr_cutoff_sq = nbr_cutoff * nbr_cutoff
#print "Optimized C++ section...",
self.set_frame(FRAME)
self.set_background_factor(kwargs["background_factor"])
self.set_nbr_cutoff_sq(nbr_cutoff_sq)
self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq)
self.set_detector_gain(self.horizons_phil.integration.detector_gain)
flex_sorted = flex.int()
for item in self.sorted:
flex_sorted.append(item[0]);flex_sorted.append(item[1]);
if self.horizons_phil.integration.mask_pixel_value is not None:
self.set_mask_pixel_val(self.horizons_phil.integration.mask_pixel_value)
image_obj = self.imagefiles.imageindex(self.frame_numbers[self.image_number])
image_obj.read()
rawdata = image_obj.linearintdata # assume image #1
if self.inputai.active_areas != None:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted,
tiles=self.inputai.active_areas.IT,
tile_id=self.inputai.active_areas.tile_id);
else:
self.detector_xy_draft = self.safe_background( rawdata=rawdata,
predicted=self.predicted,
OS_adapt=OS_adapt,
sorted=flex_sorted);
for i in xrange(len(self.predicted)): # loop over predicteds
B_S_mask = {}
keys = self.get_bsmask(i)
for k in xrange(0,len(keys),2):
B_S_mask[(keys[k],keys[k+1])]=True
self.BSmasks.append(B_S_mask)
#print "Done"
return
def __init__(self,
fmodels,
restraints_manager=None,
model=None,
is_neutron_scat_table=None,
target_weights=None,
refine_xyz=False,
refine_adp=False,
lbfgs_termination_params=None,
use_fortran=False,
verbose=0,
correct_special_position_tolerance=1.0,
iso_restraints=None,
h_params=None,
qblib_params=None,
macro_cycle=None,
u_min=adptbx.b_as_u(-5.0),
u_max=adptbx.b_as_u(999.99),
collect_monitor=True,
log=None):
timer = user_plus_sys_time()
adopt_init_args(self, locals())
self.f = None
self.xray_structure = self.fmodels.fmodel_xray().xray_structure
self.fmodels.create_target_functors()
self.fmodels.prepare_target_functors_for_minimization()
if (self.refine_adp and fmodels.fmodel_neutron() is None):
self.xray_structure.tidy_us()
self.fmodels.update_xray_structure(
xray_structure=self.xray_structure, update_f_calc=True)
self.weights = None
# QBLIB INSERT
self.qblib_params = qblib_params
if (self.qblib_params is not None and self.qblib_params.qblib):
self.macro = macro_cycle
self.qblib_cycle_count = 0
self.tmp_XYZ = None
self.XYZ_diff_curr = None
# QBLIB END
self.correct_special_position_tolerance = correct_special_position_tolerance
if (refine_xyz and target_weights is not None):
self.weights = target_weights.xyz_weights_result
elif (refine_adp and target_weights is not None):
self.weights = target_weights.adp_weights_result
else:
from phenix.refinement import weight_xray_chem
self.weights = weight_xray_chem.weights(wx=1,
wx_scale=1,
angle_x=None,
wn=1,
wn_scale=1,
angle_n=None,
w=0,
wxn=1)
if (self.collect_monitor):
self.monitor = monitor(weights=self.weights,
fmodels=fmodels,
model=model,
iso_restraints=iso_restraints,
refine_xyz=refine_xyz,
refine_adp=refine_adp,
refine_occ=False)
if (self.collect_monitor): self.monitor.collect()
self.neutron_refinement = (self.fmodels.fmodel_n is not None)
self.x = flex.double(self.xray_structure.n_parameters(), 0)
self.riding_h_manager = self.model.riding_h_manager
self._scatterers_start = self.xray_structure.scatterers()
#lbfgs_core_params = scitbx.lbfgs.core_parameters(
# stpmin = 1.e-9,
# stpmax = adptbx.b_as_u(10))
self.minimizer = scitbx.lbfgs.run(
target_evaluator=self,
termination_params=lbfgs_termination_params,
use_fortran=use_fortran,
# core_params = lbfgs_core_params,
# gradient_only=True,
exception_handling_params=scitbx.lbfgs.
exception_handling_parameters(
ignore_line_search_failed_step_at_lower_bound=True))
self.apply_shifts()
del self._scatterers_start
self.compute_target(compute_gradients=False,
u_iso_refinable_params=None)
if (self.refine_adp and self.fmodels.fmodel_neutron() is None):
self.xray_structure.tidy_us()
self.fmodels.update_xray_structure(
xray_structure=self.xray_structure, update_f_calc=True)
if (self.collect_monitor):
self.monitor.collect(iter=self.minimizer.iter(),
nfun=self.minimizer.nfun())
self.fmodels.create_target_functors()
# QBLIB INSERT
if (self.qblib_params is not None and self.qblib_params.qblib):
print('{:-^80}'.format(""), file=self.qblib_params.qblib_log)
print(file=self.qblib_params.qblib_log)
def __init__(OO,self,use_inverse_beam=False):
OO.parent = self # OO.parent is an instance of the legacy IntegrationMetaProcedure class
from xfel.mono_simulation import bandpass_gaussian
from rstbx.bandpass import parameters_bp3
#take needed parameters from parent
pxlsz = self.pixel_size # mm/pixel
detector_origin = col(( -self.inputai.xbeam(),
-self.inputai.ybeam(),
0.))
#OO.space_group = self.inputpd["symmetry"].space_group() #comment this back in as needed for refinement
indices = flex.miller_index([self.hkllist[pair["pred"]] for pair in self.indexed_pairs])
OO.reserve_indices = indices
OO.input_orientation = self.inputai.getOrientation()
OO.central_wavelength_ang = self.inputai.wavelength
incident_beam = col((0.,0.,-1.))
if use_inverse_beam: incident_beam*=-1.
parameters = parameters_bp3(
indices=indices, orientation=OO.input_orientation,
incident_beam=incident_beam,
packed_tophat=col((1.,1.,0.)),
detector_normal=col((0.,0.,-1.)),
detector_fast=col((0.,1.,0.)),detector_slow=col((1.,0.,0.)),
pixel_size=col((pxlsz,pxlsz,0)),
pixel_offset=col((0.,0.,0.0)),
distance=self.inputai.distance(),
detector_origin=detector_origin
)
OO.ucbp3 = bandpass_gaussian(parameters=parameters)
if "horizons_phil" in OO.parent.__dict__:
the_tiles = OO.parent.imagefiles.images[OO.parent.image_number
].get_tile_manager(OO.parent.horizons_phil
).effective_tiling_as_flex_int(
reapply_peripheral_margin=True,encode_inactive_as_zeroes=True)
OO.ucbp3.set_active_areas( the_tiles )
else:
OO.ucbp3.set_active_areas( [0,0,1700,1700] )
integration_signal_penetration=0.0 # easier to calculate distance derivatives
OO.ucbp3.set_sensor_model( thickness_mm = 0.5, mu_rho = 8.36644, # CS_PAD detector at 1.3 Angstrom
signal_penetration = integration_signal_penetration)
# test for horizons_phil simply skips the subpixel correction for initial labelit indexing
if "horizons_phil" in OO.parent.__dict__:
if OO.parent.horizons_phil.integration.subpixel_joint_model.translations is not None:
"Subpixel corrections: using joint-refined translation + rotation"
T = OO.parent.horizons_phil.integration.subpixel_joint_model.translations
import copy
resortedT = copy.copy(T)
for tt in xrange(0,len(T),2):
resortedT[tt] = T[tt+1]
resortedT[tt+1] = T[tt]
OO.ucbp3.set_subpixel(
translations = resortedT, rotations_deg = flex.double(
OO.parent.horizons_phil.integration.subpixel_joint_model.rotations)
)
else:
pass; "Subpixel corrections: none used"
half_mosaicity_rad = (self.inputai.getMosaicity()/2.) * math.pi/180.
OO.ucbp3.set_mosaicity(half_mosaicity_rad)
OO.ucbp3.set_bandpass(OO.central_wavelength_ang - 0.000001, OO.central_wavelength_ang + 0.000001)
OO.ucbp3.set_domain_size(280. * 17.) # for Holton psI simulation; probably doesn't detract from general case
exit()
if pixel_size_mm is None:
print "Please specify pixel size (eg. pixel_size_mm=0.079346)"
exit()
return data, hklrefin, pixel_size_mm, target_unit_cell, target_space_group, target_anomalous_flag, flag_plot, d_min, d_max, n_residues
if (__name__ == "__main__"):
cc_bin_low_thres = 0.25
beam_thres = 0.5
uc_tol = 3
#0 .read input parameters and frames (pickle files)
data, hklrefin, pixel_size_mm, target_unit_cell, \
target_space_group, target_anomalous_flag, flag_plot, d_min, d_max, n_residues = read_input(args = sys.argv[1:])
frame_files = read_pickles(data)
xbeam_set = flex.double()
ybeam_set = flex.double()
sys_abs_set = []
sys_abs_all = flex.double()
cc_bin_low_set = flex.double()
cc_bins_set = []
d_bins_set = []
oodsqr_bins_set = []
flag_good_unit_cell_set = []
print 'Summary of integration pickles:'
print '(image file, min. res., max. res, beamx, beamy, n_refl, cciso, <cciso_bin>, a, b, c, mosaicity, residual, dd_mm, wavelength, good_cell?, G, B)'
uc_a = flex.double()
uc_b = flex.double()
uc_c = flex.double()
dd_mm = flex.double()
wavelength_set = flex.double()
def __init__(pfh):
super(per_frame_helper, pfh).__init__(n_parameters=2)
pfh.x_0 = flex.double((0.,0.))
pfh.restart()
def __init__(self,
u_cart,
u_iso=None,
use_u_aniso=None,
sites_cart=None,
adp_similarity_proxies=None,
rigid_bond_proxies=None,
isotropic_adp_proxies=None,
compute_gradients=True,
gradients_aniso_cart=None,
gradients_iso=None,
disable_asu_cache=False,
normalization=False):
adopt_init_args(self, locals())
self.number_of_restraints = 0
self.residual_sum = 0
self.normalization_factor = None
if (adp_similarity_proxies is not None):
assert u_iso is not None and use_u_aniso is not None
if (rigid_bond_proxies is not None): assert sites_cart is not None
if (sites_cart is not None): assert sites_cart.size() == u_cart.size()
if (u_iso is not None): assert u_iso.size() == u_cart.size()
if (use_u_aniso is not None):
assert use_u_aniso.size() == u_cart.size()
if (compute_gradients):
if (self.gradients_aniso_cart is None):
self.gradients_aniso_cart = flex.sym_mat3_double(
sites_cart.size(), [0, 0, 0, 0, 0, 0])
else:
assert self.gradients_aniso_cart.size() == sites_cart.size()
if (u_iso is not None and self.gradients_iso is None):
self.gradients_iso = flex.double(sites_cart.size(), 0)
elif (u_iso is not None):
assert self.gradients_iso.size() == sites_cart.size()
if (adp_similarity_proxies is None):
self.n_adp_similarity_proxies = None
self.adp_similarity_residual_sum = 0
else:
self.n_adp_similarity_proxies = len(adp_similarity_proxies)
self.adp_similarity_residual_sum = adp_restraints.adp_similarity_residual_sum(
u_cart=u_cart,
u_iso=u_iso,
use_u_aniso=use_u_aniso,
proxies=adp_similarity_proxies,
gradients_aniso_cart=self.gradients_aniso_cart,
gradients_iso=self.gradients_iso)
self.number_of_restraints += 6 * self.n_adp_similarity_proxies
self.residual_sum += self.adp_similarity_residual_sum
if (rigid_bond_proxies is None):
self.n_rigid_bond_proxies = None
self.rigid_bond_residual_sum = 0
else:
self.n_rigid_bond_proxies = len(rigid_bond_proxies)
self.rigid_bond_residual_sum = adp_restraints.rigid_bond_residual_sum(
sites_cart=sites_cart,
u_cart=u_cart,
proxies=rigid_bond_proxies,
gradients_aniso_cart=self.gradients_aniso_cart)
self.number_of_restraints += self.n_rigid_bond_proxies
self.residual_sum += self.rigid_bond_residual_sum
if (isotropic_adp_proxies is None):
self.n_isotropic_adp_proxies = None
self.isotropic_adp_residual_sum = 0
else:
self.n_isotropic_adp_proxies = len(isotropic_adp_proxies)
self.isotropic_adp_residual_sum = adp_restraints.isotropic_adp_residual_sum(
u_cart=u_cart,
proxies=isotropic_adp_proxies,
gradients_aniso_cart=self.gradients_aniso_cart)
self.number_of_restraints += self.n_isotropic_adp_proxies
self.residual_sum += self.isotropic_adp_residual_sum
self.finalize_target_and_gradients()
def add_miller_array(self,
miller_array,
column_root_label,
column_types=None,
label_decorator=None):
assert column_types is None or isinstance(column_types, str)
if (label_decorator is None):
label_decorator = globals()["label_decorator"]()
default_col_types = default_column_types(miller_array=miller_array)
if (default_col_types is None):
raise RuntimeError(
"Conversion of given type of miller_array to MTZ format"
" is not supported.")
if (column_types is None):
column_types = default_col_types
elif (len(column_types) != len(default_col_types)):
raise RuntimeError(
"Invalid MTZ column_types for the given miller_array.")
self.initialize_hkl_columns()
if (not miller_array.anomalous_flag()):
if (default_col_types in ["FQ", "JQ"]):
self._add_observations(
data_label=column_root_label,
sigmas_label=label_decorator.sigmas(column_root_label),
column_types=column_types,
indices=miller_array.indices(),
data=miller_array.data(),
sigmas=miller_array.sigmas())
elif (default_col_types == "FP"):
self._add_complex(
amplitudes_label=column_root_label,
phases_label=label_decorator.phases(column_root_label),
column_types=column_types,
indices=miller_array.indices(),
data=miller_array.data())
elif (default_col_types in ["F", "J"]):
self.add_column(
label=column_root_label,
type=column_types).set_reals(
miller_indices=miller_array.indices(),
data=miller_array.data())
elif (default_col_types == "I"):
self.add_column(
label=column_root_label,
type=column_types).set_reals(
miller_indices=miller_array.indices(),
data=miller_array.data().as_double())
elif (default_col_types == "AAAA"):
mtz_reflection_indices = self.add_column(
label=label_decorator.hendrickson_lattman(column_root_label, 0),
type=column_types[0]).set_reals(
miller_indices=miller_array.indices(),
data=miller_array.data().slice(0))
for i in xrange(1,4):
self.add_column(
label=label_decorator.hendrickson_lattman(column_root_label, i),
type=column_types[i]).set_reals(
mtz_reflection_indices=mtz_reflection_indices,
data=miller_array.data().slice(i))
else:
raise RuntimeError("Fatal programming error.")
else:
asu, matches = miller_array.match_bijvoet_mates()
if (default_col_types == "FQDQY"):
_ = matches.pairs_hemisphere_selection
selpp = _("+")
selpm = _("-")
_ = matches.singles_hemisphere_selection
selsp = _("+")
selsm = _("-")
_ = asu.data()
fp = _.select(selpp)
fm = _.select(selpm)
fs = _.select(selsp)
fs.extend(_.select(selsm))
# http://www.ccp4.ac.uk/dist/html/mtzMADmod.html
f = 0.5 * (fp + fm)
d = fp - fm
_ = asu.sigmas()
sp = _.select(selpp)
sm = _.select(selpm)
ss = _.select(selsp)
ss.extend(_.select(selsm))
sd = flex.sqrt(sp**2 + sm**2)
sf = 0.5 * sd
f.extend(fs)
sf.extend(ss)
_ = asu.indices()
hd = _.select(selpp)
hf = hd.concatenate(_.select(selsp))
hf.extend(-_.select(selsm))
isym = flex.double(selpp.size(), 0) # both F+ and F-
isym.resize(selpp.size()+selsp.size(), 1) # only F+
isym.resize(hf.size(), 2) # only F-
isym.set_selected(miller_array.space_group().is_centric(hf) , 0)
label_group = [
column_root_label,
label_decorator.sigmas(column_root_label),
label_decorator.delta_anomalous(column_root_label),
label_decorator.delta_anomalous_sigmas(column_root_label),
label_decorator.delta_anomalous_isym(column_root_label)]
for i,(mi,data) in enumerate([(hf,f),(hf,sf),(hd,d),(hd,sd),(hf,isym)]):
self.add_column(
label=label_group[i],
type=column_types[i]).set_reals(miller_indices=mi, data=data)
else:
for anomalous_sign in ("+","-"):
sel = matches.pairs_hemisphere_selection(anomalous_sign)
sel.extend(matches.singles_hemisphere_selection(anomalous_sign))
if (anomalous_sign == "+"):
indices = asu.indices().select(sel)
else:
indices = -asu.indices().select(sel)
data = asu.data().select(sel)
if (default_col_types in ["GL", "KM"]):
self._add_observations(
data_label=label_decorator.anomalous(
column_root_label, anomalous_sign),
sigmas_label=label_decorator.sigmas(
column_root_label, anomalous_sign),
column_types=column_types,
indices=indices,
data=data,
sigmas=asu.sigmas().select(sel))
elif (default_col_types == "GP"):
self._add_complex(
amplitudes_label=label_decorator.anomalous(
column_root_label, anomalous_sign),
phases_label=label_decorator.phases(
column_root_label, anomalous_sign),
column_types=column_types,
indices=indices,
data=data)
elif (default_col_types in ["G", "K"]):
self.add_column(
label=label_decorator.anomalous(
column_root_label, anomalous_sign),
type=column_types).set_reals(
miller_indices=indices,
data=data)
elif (default_col_types == "I"):
self.add_column(
label=label_decorator.anomalous(
column_root_label, anomalous_sign),
type=column_types).set_reals(
miller_indices=indices,
data=data.as_double())
elif (default_col_types == "AAAA"):
mtz_reflection_indices = self.add_column(
label=label_decorator.hendrickson_lattman(
column_root_label, 0, anomalous_sign),
type=column_types[0]).set_reals(
miller_indices=indices,
data=data.slice(0))
for i in xrange(1,4):
self.add_column(
label=label_decorator.hendrickson_lattman(
column_root_label, i, anomalous_sign),
type=column_types[i]).set_reals(
mtz_reflection_indices=mtz_reflection_indices,
data=data.slice(i))
else:
raise RuntimeError("Fatal programming error.")
return self
def plot_stats(self, results, iparams):
#retrieve stats from results and plot them
if iparams.flag_plot or iparams.flag_output_verbose:
#for plotting set n_bins = 5 to avoid empty bin
n_bins_plot = 5
#get expected f^2
try:
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(iparams.n_residues)
observations_as_f = results[0].observations.as_amplitude_array(
)
binner_template_asu = observations_as_f.setup_binner(
n_bins=n_bins_plot)
wp = statistics.wilson_plot(observations_as_f,
asu_contents,
e_statistics=True)
expected_f_sq = wp.expected_f_sq
mean_stol_sq = wp.mean_stol_sq
except Exception:
expected_f_sq = flex.double([0] * n_bins_plot)
mean_stol_sq = flex.double(range(n_bins_plot))
print "Warning: Wilson plot calculation in plot stats failed."
#setup list
params_array = np.array([[pres.R_init, pres.R_final, pres.R_xy_init, pres.R_xy_final, \
pres.G, pres.B, pres.rotx*180/math.pi, pres.roty*180/math.pi, \
pres.ry, pres.rz, pres.r0, pres.re, pres.voigt_nu, \
pres.uc_params[0], pres.uc_params[1], pres.uc_params[2], \
pres.uc_params[3], pres.uc_params[4], pres.uc_params[5], \
pres.CC_final, pres.pickle_filename] for pres in results])
params = ['Rinit','Rfinal','Rxyinit', 'Rxyfinal', \
'G','B','rot_x','rot_y','gamma_y','gamma_z','gamma_0','gamma_e','voigtnu' , \
'a','b','c','alpha','beta','gamma','CC','Filename']
#keep parameter history if verbose is selected
if iparams.flag_output_verbose:
fileseq_list = flex.int()
for file_in in os.listdir(iparams.run_no):
if file_in.endswith('.paramhist'):
file_split = file_in.split('.')
fileseq_list.append(int(file_split[0]))
if len(fileseq_list) == 0:
new_fileseq = 0
else:
new_fileseq = flex.max(fileseq_list) + 1
newfile_name = str(new_fileseq) + '.paramhist'
txt_out_verbose = '\n'.join([' '.join(p) for p in params_array])
f = open(iparams.run_no + '/' + newfile_name, 'w')
f.write(txt_out_verbose)
f.close()
#plotting
if iparams.flag_plot:
n_rows = 3
n_cols = int(math.ceil(len(params) / n_rows))
num_bins = 10
for i in xrange(len(params) - 1):
tmp_params = params_array[:, i].astype(np.float)
plt.subplot(n_rows, n_cols, i + 1)
plt.hist(tmp_params,
num_bins,
normed=0,
facecolor='green',
alpha=0.5)
plt.ylabel('Frequencies')
plt.title(params[i] + '\nmu %5.1f med %5.1f sigma %5.1f' %
(np.mean(tmp_params), np.median(tmp_params),
np.std(tmp_params)))
plt.show()
def __init__(self,
pdb_hierarchy,
crystal_symmetry,
angular_difference_threshold_deg=5.,
sequence_identity_threshold=90.,
quiet=False):
h = pdb_hierarchy
superposition_threshold = 2 * sequence_identity_threshold - 100.
n_atoms_all = h.atoms_size()
s_str = "altloc ' ' and (protein or nucleotide)"
h = h.select(h.atom_selection_cache().selection(s_str))
h1 = iotbx.pdb.hierarchy.root()
h1.append_model(h.models()[0].detached_copy())
unit_cell = crystal_symmetry.unit_cell()
result = {}
if not quiet:
print("Find groups of chains related by translational NCS")
# double loop over chains to find matching pairs related by pure translation
for c1 in h1.chains():
c1.parent().remove_chain(c1)
nchains = len(h1.models()[0].chains())
if ([c1.is_protein(), c1.is_na()].count(True) == 0): continue
r1 = list(c1.residues())
c1_seq = "".join(c1.as_sequence())
sc_1_tmp = c1.atoms().extract_xyz()
h1_p1 = h1.expand_to_p1(crystal_symmetry=crystal_symmetry)
for (ii, c2) in enumerate(h1_p1.chains()):
orig_c2 = h1.models()[0].chains()[ii % nchains]
r2 = list(c2.residues())
c2_seq = "".join(c2.as_sequence())
sites_cart_1, sites_cart_2 = None, None
sc_2_tmp = c2.atoms().extract_xyz()
# chains are identical
if (c1_seq == c2_seq and sc_1_tmp.size() == sc_2_tmp.size()):
sites_cart_1 = sc_1_tmp
sites_cart_2 = sc_2_tmp
p_identity = 100.
# chains are not identical, do alignment
else:
align_obj = mmtbx.alignment.align(seq_a=c1_seq,
seq_b=c2_seq)
alignment = align_obj.extract_alignment()
matches = alignment.matches()
equal = matches.count("|")
total = len(alignment.a) - alignment.a.count("-")
p_identity = 100. * equal / max(1, total)
if (p_identity > superposition_threshold):
sites_cart_1 = flex.vec3_double()
sites_cart_2 = flex.vec3_double()
for i1, i2, match in zip(alignment.i_seqs_a,
alignment.i_seqs_b, matches):
if (i1 is not None and i2 is not None
and match == "|"):
r1i, r2i = r1[i1], r2[i2]
assert r1i.resname == r2i.resname, [
r1i.resname, r2i.resname, i1, i2
]
for a1 in r1i.atoms():
for a2 in r2i.atoms():
if (a1.name == a2.name):
sites_cart_1.append(a1.xyz)
sites_cart_2.append(a2.xyz)
break
# superpose two sequence-aligned chains
if ([sites_cart_1, sites_cart_2].count(None) == 0):
lsq_fit_obj = superpose.least_squares_fit(
reference_sites=sites_cart_1, other_sites=sites_cart_2)
angle = lsq_fit_obj.r.rotation_angle()
t_frac = unit_cell.fractionalize(
(sites_cart_1 - sites_cart_2).mean())
t_frac = [math.modf(t)[0]
for t in t_frac] # put into [-1,1]
radius = flex.sum(
flex.sqrt((sites_cart_1 - sites_cart_1.mean()
).dot())) / sites_cart_1.size() * 4. / 3.
fracscat = min(c1.atoms_size(),
c2.atoms_size()) / n_atoms_all
result.setdefault(frozenset([c1, orig_c2]), []).append([
p_identity,
[lsq_fit_obj.r, t_frac, angle, radius, fracscat]
])
else:
result.setdefault(frozenset([c1, orig_c2]),
[]).append([p_identity, None])
# Build graph
g = graph.adjacency_list()
vertex_handle = {}
for key in result:
seqid = result[key][0][0]
sup = min(result[key],
key=lambda s: 0 if s[1] is None else s[1][2])[1]
result[key] = [seqid, sup]
if ((seqid > sequence_identity_threshold)
and (sup[2] < angular_difference_threshold_deg)):
(c1, c2) = key
if (c1 not in vertex_handle):
vertex_handle[c1] = g.add_vertex(label=c1)
if (c2 not in vertex_handle):
vertex_handle[c2] = g.add_vertex(label=c2)
g.add_edge(vertex1=vertex_handle[c1],
vertex2=vertex_handle[c2])
# Do connected component analysis and compose final tNCS pairs object
components = connected_component_algorithm.connected_components(g)
import itertools
self.ncs_pairs = []
self.tncsresults = [0, "", [], 0.0]
for (i, group) in enumerate(components):
chains = [g.vertex_label(vertex=v) for v in group]
fracscats = []
radii = []
for pair in itertools.combinations(chains, 2):
sup = result[frozenset(pair)][1]
fracscats.append(sup[-1])
radii.append(sup[-2])
fs = sum(fracscats) / len(fracscats)
self.tncsresults[3] = fs # store fracscat in array
rad = sum(radii) / len(radii)
#import code, traceback; code.interact(local=locals(), banner="".join( traceback.format_stack(limit=10) ) )
maxorder = 1
vectors = []
previous_id = next(itertools.combinations(chains, 2))[0].id
for pair in itertools.combinations(chains, 2):
sup = result[frozenset(pair)][1]
ncs_pair = ext.pair(
r=sup[0],
t=sup[1],
radius=rad,
radius_estimate=rad,
fracscat=fs,
rho_mn=flex.double(
), # rho_mn undefined, needs to be set later
id=i)
self.ncs_pairs.append(ncs_pair)
# show tNCS pairs in group
fmt = "group %d chains %s <> %s angle: %4.2f trans.vect.: (%s) fracscat: %5.3f"
t = ",".join([("%6.3f" % t_).strip() for t_ in sup[1]]).strip()
if not quiet:
print(fmt % (i, pair[0].id, pair[1].id, sup[2], t, fs))
if pair[0].id == previous_id:
maxorder += 1
orthoxyz = unit_cell.orthogonalize(sup[1])
vectors.append((sup[1], orthoxyz, sup[2]))
else:
previous_id = pair[0].id
maxorder = 1
vectors = []
if maxorder > self.tncsresults[0]:
self.tncsresults[0] = maxorder
self.tncsresults[1] = previous_id
self.tncsresults[2] = vectors
if not quiet:
print("Largest TNCS order, peptide chain, fracvector, orthvector, angle, fracscat = ", \
str(self.tncsresults))
def exercise_miller_arrays_as_cif_block():
from iotbx.cif import reader
cif_model = reader(input_string=cif_miller_array,
builder=cif.builders.cif_model_builder()).model()
ma_builder = cif.builders.miller_array_builder(cif_model['global'])
ma1 = ma_builder.arrays()['_refln_F_squared_meas']
mas_as_cif_block = cif.miller_arrays_as_cif_block(ma1,
array_type='meas',
format="corecif")
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.complex_double([1 - 1j] * ma1.size())),
array_type='calc')
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.complex_double([1 - 2j] * ma1.size())),
column_names=['_refln_A_calc', '_refln_B_calc'])
for key in ('_refln_F_squared_meas', '_refln_F_squared_sigma',
'_refln_F_calc', '_refln_phase_calc', '_refln_A_calc',
'_refln_A_calc'):
assert (key in mas_as_cif_block.cif_block.keys()), key
#
mas_as_cif_block = cif.miller_arrays_as_cif_block(ma1,
array_type='meas',
format="mmcif")
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.complex_double([1 - 1j] * ma1.size())),
array_type='calc')
for key in ('_refln.F_squared_meas', '_refln.F_squared_sigma',
'_refln.F_calc', '_refln.phase_calc',
'_space_group_symop.operation_xyz', '_cell.length_a',
'_refln.index_h'):
assert key in mas_as_cif_block.cif_block.keys()
#
mas_as_cif_block = cif.miller_arrays_as_cif_block(
ma1,
column_names=[
'_diffrn_refln_intensity_net', '_diffrn_refln_intensity_sigma'
],
miller_index_prefix='_diffrn_refln')
mas_as_cif_block.add_miller_array(
ma1.array(data=flex.std_string(ma1.size(), 'om')),
column_name='_diffrn_refln_intensity_u')
for key in ('_diffrn_refln_intensity_net', '_diffrn_refln_intensity_sigma',
'_diffrn_refln_intensity_u'):
assert key in list(mas_as_cif_block.cif_block.keys())
#
try:
reader(input_string=cif_global)
except CifParserError as e:
pass
else:
raise Exception_expected
cif_model = reader(input_string=cif_global, strict=False).model()
assert not show_diff(
str(cif_model), """\
data_1
_c 3
_d 4
""")
# exercise adding miller arrays with non-matching indices
cs = crystal.symmetry(unit_cell=uctbx.unit_cell((10, 10, 10, 90, 90, 90)),
space_group_info=sgtbx.space_group_info(symbol="P1"))
mi = flex.miller_index(((1, 0, 0), (1, 2, 3), (2, 3, 4)))
ms1 = miller.set(cs, mi)
ma1 = miller.array(ms1, data=flex.double((1, 2, 3)))
mas_as_cif_block = cif.miller_arrays_as_cif_block(
ma1, column_name="_refln.F_meas_au")
ms2 = miller.set(cs, mi[:2])
ma2 = miller.array(ms2, data=flex.complex_double([1 - 2j] * ms2.size()))
mas_as_cif_block.add_miller_array(ma2,
column_names=("_refln.F_calc_au",
"_refln.phase_calc")),
ms3 = miller.set(cs, flex.miller_index(((1, 0, 0), (5, 6, 7), (2, 3, 4))))
ma3 = miller.array(ms3, data=flex.double((4, 5, 6)))
mas_as_cif_block.add_miller_array(ma3, column_name="_refln.F_squared_meas")
ms4 = miller.set(
cs,
flex.miller_index(
((1, 2, 3), (5, 6, 7), (1, 1, 1), (1, 0, 0), (2, 3, 4))))
ma4 = ms4.d_spacings()
mas_as_cif_block.add_miller_array(ma4, column_name="_refln.d_spacing")
# extract arrays from cif block and make sure we get back what we started with
arrays = cif.builders.miller_array_builder(
mas_as_cif_block.cif_block).arrays()
recycled_arrays = (arrays['_refln.F_meas_au'], arrays['_refln.F_calc_au'],
arrays['_refln.F_squared_meas'],
arrays['_refln.d_spacing'])
for orig, recycled in zip((ma1, ma2, ma3, ma4), recycled_arrays):
assert orig.size() == recycled.size()
recycled = recycled.customized_copy(
anomalous_flag=orig.anomalous_flag())
orig, recycled = orig.common_sets(recycled)
assert orig.indices().all_eq(recycled.indices())
assert approx_equal(orig.data(), recycled.data(), eps=1e-5)
#
cif_model = reader(input_string=r3adrsf,
builder=cif.builders.cif_model_builder()).model()
cs = cif.builders.crystal_symmetry_builder(
cif_model["r3adrsf"]).crystal_symmetry
ma_builder = cif.builders.miller_array_builder(
cif_model['r3adrAsf'],
base_array_info=miller.array_info(crystal_symmetry_from_file=cs))
miller_arrays = list(ma_builder.arrays().values())
assert len(miller_arrays) == 4
mas_as_cif_block = cif.miller_arrays_as_cif_block(
miller_arrays[0].map_to_asu(),
column_names=miller_arrays[0].info().labels,
format="corecif")
for array in miller_arrays[1:]:
labels = array.info().labels
if len(labels) > 1:
for label in labels:
if label.startswith("wavelength_id"):
labels.remove(label)
mas_as_cif_block.add_miller_array(array=array.map_to_asu(),
column_names=array.info().labels)
s = StringIO()
print(mas_as_cif_block.refln_loop, file=s)
assert not show_diff(
s.getvalue(), """\
loop_
_refln_index_h
_refln_index_k
_refln_index_l
_refln.crystal_id
_refln.scale_group_code
_refln.wavelength_id
_refln.pdbx_I_plus
_refln.pdbx_I_plus_sigma
_refln.pdbx_I_minus
_refln.pdbx_I_minus_sigma
-87 5 46 1 1 3 40.2 40.4 6.7 63.9
-87 5 45 1 1 3 47.8 29.7 35.1 30.5
-87 5 44 1 1 3 18.1 33.2 0.5 34.6
-87 5 43 1 1 3 6.1 45.4 12.9 51.6
-87 5 42 1 1 3 -6.6 45.6 -15.5 55.8
-87 7 37 1 1 3 6.3 43.4 ? ?
-87 7 36 1 1 3 -67.2 55.4 ? ?
-88 2 44 1 1 3 0 -1 35 38.5
-88 2 43 1 1 3 0 -1 57.4 41.5
-88 4 45 1 1 3 -1 46.1 -9.1 45.6
-88 4 44 1 1 3 -19.8 49.2 0.3 34.7
-88 6 44 1 1 3 -1.8 34.8 ? ?
""")
def prepare_output(self, results, iparams, avg_mode):
if avg_mode == 'average':
cc_thres = 0
else:
cc_thres = iparams.frame_accept_min_cc
std_filter = iparams.sigma_rejection
if iparams.flag_weak_anomalous:
if avg_mode == 'final':
target_anomalous_flag = iparams.target_anomalous_flag
else:
target_anomalous_flag = False
else:
target_anomalous_flag = iparams.target_anomalous_flag
pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters(
results)
G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean
G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med
G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std
#prepare data for merging
miller_indices_all = flex.miller_index()
miller_indices_ori_all = flex.miller_index()
I_all = flex.double()
sigI_all = flex.double()
G_all = flex.double()
B_all = flex.double()
p_all = flex.double()
rx_all = flex.double()
rs_all = flex.double()
rh_all = flex.double()
SE_all = flex.double()
sin_sq_all = flex.double()
wavelength_all = flex.double()
detector_distance_set = flex.double()
R_init_all = flex.double()
R_final_all = flex.double()
R_xy_init_all = flex.double()
R_xy_final_all = flex.double()
pickle_filename_all = flex.std_string()
filtered_results = []
cn_good_frame, cn_bad_frame_SE, cn_bad_frame_uc, cn_bad_frame_cc, cn_bad_frame_G, cn_bad_frame_re = (
0, 0, 0, 0, 0, 0)
crystal_orientation_dict = {}
for pres in results:
if pres is not None:
pickle_filepath = pres.pickle_filename.split('/')
img_filename = pickle_filepath[len(pickle_filepath) - 1]
flag_pres_ok = True
#check SE, CC, UC, G, B, gamma_e
if math.isnan(pres.G):
flag_pres_ok = False
if math.isnan(pres.SE) or np.isinf(pres.SE):
flag_pres_ok = False
if flag_pres_ok and SE_std > 0:
if abs(pres.SE - SE_med) / SE_std > std_filter:
flag_pres_ok = False
cn_bad_frame_SE += 1
if flag_pres_ok and pres.CC_final < cc_thres:
flag_pres_ok = False
cn_bad_frame_cc += 1
if flag_pres_ok:
if G_std > 0:
if abs(pres.G - G_med) / G_std > std_filter:
flag_pres_ok = False
cn_bad_frame_G += 1
if flag_pres_ok:
if re_std > 0:
if abs(pres.re - re_med) / re_std > std_filter:
flag_pres_ok = False
cn_bad_frame_re += 1
if flag_pres_ok and not good_unit_cell(
pres.uc_params, iparams, iparams.merge.uc_tolerance):
flag_pres_ok = False
cn_bad_frame_uc += 1
data_size = pres.observations.size()
if flag_pres_ok:
cn_good_frame += 1
filtered_results.append(pres)
R_init_all.append(pres.R_init)
R_final_all.append(pres.R_final)
R_xy_init_all.append(pres.R_xy_init)
R_xy_final_all.append(pres.R_xy_final)
miller_indices_all.extend(pres.observations.indices())
miller_indices_ori_all.extend(
pres.observations_original.indices())
I_all.extend(pres.observations.data())
sigI_all.extend(pres.observations.sigmas())
G_all.extend(flex.double([pres.G] * data_size))
B_all.extend(flex.double([pres.B] * data_size))
p_all.extend(pres.partiality)
rs_all.extend(pres.rs_set)
rh_all.extend(pres.rh_set)
sin_sq_all.extend(
pres.observations.two_theta(wavelength=pres.wavelength)
.sin_theta_over_lambda_sq().data())
SE_all.extend(flex.double([pres.SE] * data_size))
wavelength_all.extend(
flex.double([pres.wavelength] * data_size))
detector_distance_set.append(pres.detector_distance_mm)
pickle_filename_all.extend(
flex.std_string([pres.pickle_filename] * data_size))
crystal_orientation_dict[
pres.pickle_filename] = pres.crystal_orientation
#plot stats
self.plot_stats(filtered_results, iparams)
#write out updated crystal orientation as a pickle file
if not iparams.flag_hush:
pickle.dump(crystal_orientation_dict,
open(iparams.run_no + '/' + "crystal.o", "wb"),
pickle.HIGHEST_PROTOCOL)
#calculate average unit cell
uc_mean, uc_med, uc_std = self.calc_mean_unit_cell(filtered_results)
unit_cell_mean = unit_cell(tuple(uc_mean))
#recalculate stats for pr parameters
pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters(
filtered_results)
G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean
G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med
G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std
#from all observations merge them
crystal_symmetry = crystal.symmetry(
unit_cell=tuple(uc_mean),
space_group_symbol=iparams.target_space_group)
miller_set_all = miller.set(crystal_symmetry=crystal_symmetry,
indices=miller_indices_all,
anomalous_flag=target_anomalous_flag)
miller_array_all = miller_set_all.array(
data=I_all, sigmas=sigI_all).set_observation_type_xray_intensity()
#sort reflections according to asymmetric-unit symmetry hkl
perm = miller_array_all.sort_permutation(by_value="packed_indices")
miller_indices_all_sort = miller_array_all.indices().select(perm)
miller_indices_ori_all_sort = miller_indices_ori_all.select(perm)
I_obs_all_sort = miller_array_all.data().select(perm)
sigI_obs_all_sort = miller_array_all.sigmas().select(perm)
G_all_sort = G_all.select(perm)
B_all_sort = B_all.select(perm)
p_all_sort = p_all.select(perm)
rs_all_sort = rs_all.select(perm)
wavelength_all_sort = wavelength_all.select(perm)
sin_sq_all_sort = sin_sq_all.select(perm)
SE_all_sort = SE_all.select(perm)
pickle_filename_all_sort = pickle_filename_all.select(perm)
miller_array_uniq = miller_array_all.merge_equivalents().array(
).complete_array(d_min=iparams.merge.d_min, d_max=iparams.merge.d_max)
matches_uniq = miller.match_multi_indices(
miller_indices_unique=miller_array_uniq.indices(),
miller_indices=miller_indices_all_sort)
pair_0 = flex.int([pair[0] for pair in matches_uniq.pairs()])
pair_1 = flex.int([pair[1] for pair in matches_uniq.pairs()])
group_id_list = flex.int(
[pair_0[pair_1[i]] for i in range(len(matches_uniq.pairs()))])
tally = Counter()
for elem in group_id_list:
tally[elem] += 1
cn_group = len(tally)
#preparte txt out stat
txt_out = 'Summary of refinement and merging\n'
txt_out += ' No. good frames: %12.0f\n' % (cn_good_frame)
txt_out += ' No. bad cc frames: %12.0f\n' % (cn_bad_frame_cc)
txt_out += ' No. bad G frames) : %12.0f\n' % (cn_bad_frame_G)
txt_out += ' No. bad unit cell frames: %12.0f\n' % (cn_bad_frame_uc)
txt_out += ' No. bad gamma_e frames: %12.0f\n' % (cn_bad_frame_re)
txt_out += ' No. bad SE: %12.0f\n' % (cn_bad_frame_SE)
txt_out += ' No. observations: %12.0f\n' % (
len(I_obs_all_sort))
txt_out += 'Mean target value (BEFORE: Mean Median (Std.))\n'
txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_init_all), np.median(R_init_all), np.std(R_init_all))
txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_xy_init_all), np.median(R_xy_init_all),
np.std(R_xy_init_all))
txt_out += 'Mean target value (AFTER: Mean Median (Std.))\n'
txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_final_all), np.median(R_final_all), np.std(R_final_all))
txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_xy_final_all), np.median(R_xy_final_all),
np.std(R_xy_final_all))
txt_out += ' SE: %12.2f %12.2f (%9.2f)\n' % (
SE_mean, SE_med, SE_std)
txt_out += ' G: %12.3e %12.3e (%9.2e)\n' % (
G_mean, G_med, G_std)
txt_out += ' B: %12.2f %12.2f (%9.2f)\n' % (
B_mean, B_med, B_std)
txt_out += ' Rot.x: %12.2f %12.2f (%9.2f)\n' % (
rotx_mean * 180 / math.pi, rotx_med * 180 / math.pi,
rotx_std * 180 / math.pi)
txt_out += ' Rot.y: %12.2f %12.2f (%9.2f)\n' % (
roty_mean * 180 / math.pi, roty_med * 180 / math.pi,
roty_std * 180 / math.pi)
txt_out += ' gamma_y: %12.5f %12.5f (%9.5f)\n' % (
ry_mean, ry_med, ry_std)
txt_out += ' gamma_z: %12.5f %12.5f (%9.5f)\n' % (
rz_mean, rz_med, rz_std)
txt_out += ' gamma_0: %12.5f %12.5f (%9.5f)\n' % (
r0_mean, r0_med, r0_std)
txt_out += ' gamma_e: %12.5f %12.5f (%9.5f)\n' % (
re_mean, re_med, re_std)
txt_out += ' voigt_nu: %12.5f %12.5f (%9.5f)\n' % (
voigt_nu_mean, voigt_nu_med, voigt_nu_std)
txt_out += ' unit cell\n'
txt_out += ' a: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[0], uc_med[0], uc_std[0])
txt_out += ' b: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[1], uc_med[1], uc_std[1])
txt_out += ' c: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[2], uc_med[2], uc_std[2])
txt_out += ' alpha: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[3], uc_med[3], uc_std[3])
txt_out += ' beta: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[4], uc_med[4], uc_std[4])
txt_out += ' gamma: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[5], uc_med[5], uc_std[5])
txt_out += 'Parmeters from integration (not-refined)\n'
txt_out += ' Wavelength: %12.5f %12.5f (%9.5f)\n' % (
np.mean(wavelength_all), np.median(wavelength_all),
np.std(wavelength_all))
txt_out += ' Detector distance: %12.5f %12.5f (%9.5f)\n' % (
np.mean(detector_distance_set), np.median(detector_distance_set),
np.std(detector_distance_set))
txt_out += '* (standard deviation)\n'
#write out stat. pickle
if not iparams.flag_hush:
stat_dict = {"n_frames_good": [cn_good_frame], \
"n_frames_bad_cc": [cn_bad_frame_cc], \
"n_frames_bad_G": [cn_bad_frame_G], \
"n_frames_bad_uc": [cn_bad_frame_uc], \
"n_frames_bad_gamma_e": [cn_bad_frame_re], \
"n_frames_bad_SE": [cn_bad_frame_SE], \
"n_observations": [len(I_obs_all_sort)], \
"R_start": [np.mean(R_init_all)], \
"R_end": [np.mean(R_final_all)], \
"R_xy_start": [np.mean(R_xy_init_all)], \
"R_xy_end": [np.mean(R_xy_final_all)], \
"mean_gamma_y": [ry_mean], \
"std_gamma_y": [ry_std], \
"mean_gamma_z": [rz_mean], \
"std_gamma_z": [rz_std], \
"mean_gamma_0": [r0_mean], \
"std_gamma_0": [r0_std], \
"mean_gamma_e": [re_mean], \
"std_gamma_e": [re_std], \
"mean_voigt_nu": [voigt_nu_mean], \
"std_voigt_nu": [voigt_nu_std], \
"mean_a": [uc_mean[0]], \
"std_a": [uc_std[0]], \
"mean_b": [uc_mean[1]], \
"std_b": [uc_std[1]], \
"mean_c": [uc_mean[2]], \
"std_c": [uc_std[2]], \
"mean_alpha": [uc_mean[3]], \
"std_alpha": [uc_std[3]], \
"mean_beta": [uc_mean[4]], \
"std_beta": [uc_std[4]], \
"mean_gamma": [uc_mean[5]], \
"std_gamma": [uc_std[5]]}
self.write_stat_pickle(iparams, stat_dict)
return cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, \
I_obs_all_sort, sigI_obs_all_sort,G_all_sort, B_all_sort, \
p_all_sort, rs_all_sort, wavelength_all_sort, sin_sq_all_sort, SE_all_sort, uc_mean, \
np.mean(wavelength_all), pickle_filename_all_sort, txt_out
def french_wilson_scale(miller_array,
params=None,
sigma_iobs_rejection_criterion=None,
merge=False,
min_bin_size=40,
max_bins=60,
log=None):
from cctbx.array_family import flex
if not miller_array.is_xray_intensity_array():
raise Sorry("Input array appears to be amplitudes. " +
"This method is only appropriate for input intensities.")
if miller_array.unit_cell() is None:
raise Sorry(
"No unit cell information found. Please supply unit cell data.")
if miller_array.crystal_symmetry() is None:
raise Sorry("No crystal symmetry information found. Please supply " +
"crystal symmetry data.")
if miller_array.sigmas() is None:
raise Sorry(
"Input array does not contain sigma values. " +
"This method requires input intensities with associated sigmas.")
if (not miller_array.is_unique_set_under_symmetry()):
if (merge):
miller_array = miller_array.merge_equivalents().array()
else:
raise Sorry("Unmerged data not allowed - please merge " +
"symmetry-equivalent reflections first.")
if (miller_array.data().all_eq(miller_array.data()[0])):
# XXX some Scalepack files (and possibly others) crash the routine if this
# check is not performed. presumably an HKL2000 bug?
raise Sorry((
"The input intensities have uniform values (%g); this is probably "
+ "a bug in one of the data processing and/or conversion programs."
) % miller_array.data()[0])
# Phil defaults are set in master_phil above - they should be kept in sync with the
# default arguments for this function
if params and params.max_bins:
max_bins = params.max_bins
if params and params.min_bin_size:
min_bin_size = params.min_bin_size
if log == None:
log = sys.stdout
if (sigma_iobs_rejection_criterion is None):
sigma_iobs_rejection_criterion = -4.0
elif (sigma_iobs_rejection_criterion == 0.0):
libtbx.warn(
"For French and Wilson scaling, sigma_iobs_rejection_criterion " +
"must be a value between -4.0 and -1.0, or None. " +
"Setting sigma_iobs_rejection_criteriont to -4.0.")
sigma_iobs_rejection_criterion = -4.0
elif ((sigma_iobs_rejection_criterion < -4.0)
or (sigma_iobs_rejection_criterion > -1.0)):
raise Sorry(
"For French and Wilson scaling, sigma_iobs_rejection_criterion " +
"must be a value between -4.0 and -1.0, or None.")
rejected = []
make_sub_header("Scaling input intensities via French-Wilson Method",
out=log)
print("Trying %d bins..." % max_bins, file=log)
try:
f_w_binning(miller_array=miller_array,
max_bins=max_bins,
min_bin_size=min_bin_size,
log=log)
except ValueError:
try:
miller_array.setup_binner_counting_sorted(reflections_per_bin=5)
except AssertionError:
print(
"Too few reflections for accurate binning.\n"
"** Skipping French-Wilson scaling **",
file=log)
return None
print("Number of bins = %d" % miller_array.binner().n_bins_used(),
file=log)
new_I = flex.double()
new_sigma_I = flex.double()
new_F = flex.double()
new_sigma_F = flex.double()
new_indices = flex.miller_index()
bin_mean_intensities = miller_array.mean(use_binning=True).data
d_mean_intensities = \
calculate_mean_intensities(miller_array=miller_array, log=log)
assert len(d_mean_intensities) == miller_array.data().size()
for i_bin in miller_array.binner().range_all():
sel = miller_array.binner().selection(i_bin)
bin = miller_array.select(sel)
if bin.size() > 0:
#bin_mean_intensity = bin_mean_intensities[i_bin]
cen = bin.select_centric()
acen = bin.select_acentric()
for I, sigma_I, index in zip(cen.data(), cen.sigmas(),
cen.indices()):
mean_intensity = d_mean_intensities[index]
if (mean_intensity == 0):
# XXX is this the appropriate way to handle this?
rejected.append((index, I, sigma_I, mean_intensity))
elif (sigma_I <= 0):
if I <= 0 or sigma_I < 0:
rejected.append((index, I, sigma_I, mean_intensity))
continue
else:
J = I
sigma_J = sigma_I
F = math.sqrt(I)
sigma_F = sigma_I
else:
J, sigma_J, F, sigma_F = fw_centric(
I=I,
sigma_I=sigma_I,
mean_intensity=mean_intensity,
sigma_iobs_rejection_criterion=\
sigma_iobs_rejection_criterion)
if J >= 0:
assert sigma_J >= 0 and F >= 0 and sigma_F >= 0
new_I.append(J)
new_indices.append(index)
new_sigma_I.append(sigma_J)
new_F.append(F)
new_sigma_F.append(sigma_F)
else:
rejected.append((index, I, sigma_I, mean_intensity))
for I, sigma_I, index in zip(acen.data(), acen.sigmas(),
acen.indices()):
mean_intensity = d_mean_intensities[index]
if (mean_intensity == 0):
rejected.append((index, I, sigma_I, mean_intensity))
elif (sigma_I <= 0):
if I <= 0 or sigma_I < 0:
rejected.append((index, I, sigma_I, mean_intensity))
continue
else:
J = I
sigma_J = sigma_I
F = math.sqrt(I)
sigma_F = sigma_I
else:
J, sigma_J, F, sigma_F = fw_acentric(
I=I,
sigma_I=sigma_I,
mean_intensity=mean_intensity,
sigma_iobs_rejection_criterion=\
sigma_iobs_rejection_criterion)
if J >= 0:
assert sigma_J >= 0 and F >= 0 and sigma_F >= 0
new_I.append(J)
new_indices.append(index)
new_sigma_I.append(sigma_J)
new_F.append(F)
new_sigma_F.append(sigma_F)
else:
rejected.append((index, I, sigma_I, mean_intensity))
f_obs = miller_array.customized_copy(indices=new_indices,
data=new_F,
sigmas=new_sigma_F)
f_obs.set_observation_type_xray_amplitude()
show_rejected_summary(rejected=rejected, log=log)
return f_obs
def exercise_mmcif_structure_factors():
miller_arrays = cif.reader(input_string=r3adrsf).as_miller_arrays()
assert len(miller_arrays) == 16
hl_coeffs = find_miller_array_from_labels(
miller_arrays, ','.join([
'scale_group_code=1', 'crystal_id=2', 'wavelength_id=3',
'_refln.pdbx_HL_A_iso', '_refln.pdbx_HL_B_iso',
'_refln.pdbx_HL_C_iso', '_refln.pdbx_HL_D_iso'
]))
assert hl_coeffs.is_hendrickson_lattman_array()
assert hl_coeffs.size() == 2
mas_as_cif_block = cif.miller_arrays_as_cif_block(
hl_coeffs,
column_names=('_refln.pdbx_HL_A_iso', '_refln.pdbx_HL_B_iso',
'_refln.pdbx_HL_C_iso', '_refln.pdbx_HL_D_iso'))
abcd = []
for key in ('_refln.pdbx_HL_A_iso', '_refln.pdbx_HL_B_iso',
'_refln.pdbx_HL_C_iso', '_refln.pdbx_HL_D_iso'):
assert key in list(mas_as_cif_block.cif_block.keys())
abcd.append(flex.double(mas_as_cif_block.cif_block[key]))
hl_coeffs_from_cif_block = flex.hendrickson_lattman(*abcd)
assert approx_equal(hl_coeffs.data(), hl_coeffs_from_cif_block)
f_meas_au = find_miller_array_from_labels(
miller_arrays, ','.join([
'scale_group_code=1', 'crystal_id=1', 'wavelength_id=1',
'_refln.F_meas_au', '_refln.F_meas_sigma_au'
]))
assert f_meas_au.is_xray_amplitude_array()
assert f_meas_au.size() == 5
assert f_meas_au.sigmas() is not None
assert f_meas_au.space_group_info().symbol_and_number(
) == 'C 1 2 1 (No. 5)'
assert approx_equal(f_meas_au.unit_cell().parameters(),
(163.97, 45.23, 110.89, 90.0, 131.64, 90.0))
pdbx_I_plus_minus = find_miller_array_from_labels(
miller_arrays, ','.join([
'_refln.pdbx_I_plus', '_refln.pdbx_I_plus_sigma',
'_refln.pdbx_I_minus', '_refln.pdbx_I_minus_sigma'
]))
assert pdbx_I_plus_minus.is_xray_intensity_array()
assert pdbx_I_plus_minus.anomalous_flag()
assert pdbx_I_plus_minus.size() == 21
assert pdbx_I_plus_minus.unit_cell() is None # no symmetry information in
assert pdbx_I_plus_minus.space_group() is None # this CIF block
#
miller_arrays = cif.reader(input_string=r3ad7sf).as_miller_arrays()
assert len(miller_arrays) == 11
f_calc = find_miller_array_from_labels(
miller_arrays, ','.join([
'crystal_id=2', 'wavelength_id=1', '_refln.F_calc',
'_refln.phase_calc'
]))
assert f_calc.is_complex_array()
assert f_calc.size() == 4
#
miller_arrays = cif.reader(
input_string=integer_observations).as_miller_arrays()
assert len(miller_arrays) == 2
assert isinstance(miller_arrays[0].data(), flex.double)
assert isinstance(miller_arrays[0].sigmas(), flex.double)
#
miller_arrays = cif.reader(input_string=r3v56sf).as_miller_arrays()
assert len(miller_arrays) == 2
for ma in miller_arrays:
assert ma.is_complex_array()
assert miller_arrays[0].info().labels == [
'r3v56sf', '_refln.pdbx_DELFWT', '_refln.pdbx_DELPHWT'
]
assert miller_arrays[1].info().labels == [
'r3v56sf', '_refln.pdbx_FWT', '_refln.pdbx_PHWT'
]
def __init__(self, **kwargs):
group_args.__init__(self, **kwargs)
# require Dij, d_c
P = Profiler("2. calculate rho density")
print("finished Dij, now calculating rho_i, the density")
from xfel.clustering import Rodriguez_Laio_clustering_2014
# alternative clustering algorithms: see http://scikit-learn.org/stable/modules/clustering.html
# also see https://cran.r-project.org/web/packages/dbscan/vignettes/hdbscan.html
# see also https://en.wikipedia.org/wiki/Hausdorff_dimension
R = Rodriguez_Laio_clustering_2014(distance_matrix=self.Dij,
d_c=self.d_c)
self.rho = rho = R.get_rho()
ave_rho = flex.mean(rho.as_double())
NN = self.Dij.focus()[0]
print("The average rho_i is %5.2f, or %4.1f%%" %
(ave_rho, 100 * ave_rho / NN))
i_max = flex.max_index(rho)
P = Profiler("3.transition")
print("the index with the highest density is %d" % (i_max))
delta_i_max = flex.max(
flex.double([self.Dij[i_max, j] for j in range(NN)]))
print("delta_i_max", delta_i_max)
rho_order = flex.sort_permutation(rho, reverse=True)
rho_order_list = list(rho_order)
P = Profiler("4. delta")
self.delta = delta = R.get_delta(rho_order=rho_order,
delta_i_max=delta_i_max)
P = Profiler("5. find cluster maxima")
#---- Now hunting for clusters ---Lot's of room for improvement (or simplification) here!!!
cluster_id = flex.int(NN, -1) # default -1 means no cluster
delta_order = flex.sort_permutation(delta, reverse=True)
N_CLUST = 10 # maximum of 10 points to be considered as possible clusters
#MAX_PERCENTILE_DELTA = 0.99 # cluster centers have to be in the top 10% percentile delta
MAX_PERCENTILE_RHO = 0.99 # cluster centers have to be in the top 75% percentile rho
n_cluster = 0
#max_n_delta = min(N_CLUST, int(MAX_PERCENTILE_DELTA*NN))
for ic in range(NN):
# test the density, rho
item_idx = delta_order[ic]
if delta[item_idx] > 100:
print("A: iteration", ic, "delta", delta[item_idx],
delta[item_idx] < 0.25 * delta[delta_order[0]])
if delta[item_idx] < 0.25 * delta[
delta_order[0]]: # too low (another heuristic!)
continue
item_rho_order = rho_order_list.index(item_idx)
if delta[item_idx] > 100:
print("B: iteration", ic, item_rho_order, item_rho_order / NN,
MAX_PERCENTILE_RHO)
if item_rho_order / NN < MAX_PERCENTILE_RHO:
cluster_id[item_idx] = n_cluster
print(ic, item_idx, item_rho_order, cluster_id[item_idx])
n_cluster += 1
print("Found %d clusters" % n_cluster)
for x in range(NN):
if cluster_id[x] >= 0:
print("XC", x, cluster_id[x], rho[x], delta[x])
self.cluster_id_maxima = cluster_id.deep_copy()
P = Profiler("6. assign all points")
R.cluster_assignment(rho_order, cluster_id)
self.cluster_id_full = cluster_id.deep_copy()
# assign the halos
P = Profiler("7. assign halos")
halo = flex.bool(NN, False)
border = R.get_border(cluster_id=cluster_id)
for ic in range(n_cluster
): #loop thru all border regions; find highest density
print("cluster", ic, "in border", border.count(True))
this_border = (cluster_id == ic) & (border == True)
print(len(this_border), this_border.count(True))
if this_border.count(True) > 0:
highest_density = flex.max(rho.select(this_border))
halo_selection = (rho < highest_density) & (this_border
== True)
if halo_selection.count(True) > 0:
cluster_id.set_selected(halo_selection, -1)
core_selection = (cluster_id == ic) & ~halo_selection
highest_density = flex.max(rho.select(core_selection))
too_sparse = core_selection & (
rho.as_double() < highest_density / 10.
) # another heuristic
if too_sparse.count(True) > 0:
cluster_id.set_selected(too_sparse, -1)
self.cluster_id_final = cluster_id.deep_copy()
print("%d in the excluded halo" % ((cluster_id == -1).count(True)))
def combine_pre_merge(self, result, iparams):
mi_all = flex.miller_index()
mio_all = flex.miller_index()
I_all = flex.double()
sigI_all = flex.double()
G_all = flex.double()
B_all = flex.double()
p_all = flex.double()
rs_all = flex.double()
wavelength_all = flex.double()
sin_all = flex.double()
SE_all = flex.double()
uc_mean_set = []
wavelength_mean_set = []
pickle_filename_all = flex.std_string()
for res in result:
for prep_output in res:
_, _, mi, mio, I, sigI, G, B, p, rs, wavelength, sin, SE, uc_mean, wavelength_mean, pickle_filename_set, txt_out = prep_output
mi_all.extend(mi)
mio_all.extend(mio)
I_all.extend(I)
sigI_all.extend(sigI)
G_all.extend(G)
B_all.extend(B)
p_all.extend(p)
rs_all.extend(rs)
wavelength_all.extend(wavelength)
sin_all.extend(sin)
SE_all.extend(SE)
uc_mean_set.extend(uc_mean)
wavelength_mean_set.append(wavelength_mean)
pickle_filename_all.extend(pickle_filename_set)
uc_mean = np.mean(np.array(uc_mean_set).reshape(-1, 6), axis=0)
wavelength_mean = np.mean(wavelength_mean_set)
ms_template = crystal.symmetry(
unit_cell=tuple(uc_mean),
space_group_symbol=iparams.target_space_group).build_miller_set(
anomalous_flag=iparams.target_anomalous_flag,
d_min=iparams.merge.d_min)
ma_all = ms_template.array().customized_copy(indices=mi_all,
data=I_all,
sigmas=sigI_all)
#sort reflections according to asymmetric-unit symmetry hkl
perm = ma_all.sort_permutation(by_value="packed_indices")
mi_all_sort = mi_all.select(perm)
mio_all_sort = mio_all.select(perm)
I_all_sort = I_all.select(perm)
sigI_all_sort = sigI_all.select(perm)
G_all_sort = G_all.select(perm)
B_all_sort = B_all.select(perm)
p_all_sort = p_all.select(perm)
rs_all_sort = rs_all.select(perm)
wavelength_all_sort = wavelength_all.select(perm)
sin_all_sort = sin_all.select(perm)
SE_all_sort = SE_all.select(perm)
pickle_filename_all_sort = pickle_filename_all.select(perm)
ma_uniq = ma_all.merge_equivalents().array().complete_array(
d_min=iparams.merge.d_min, d_max=iparams.merge.d_max)
matches_uniq = miller.match_multi_indices(
miller_indices_unique=ma_uniq.indices(),
miller_indices=mi_all_sort)
pair_0 = flex.int([pair[0] for pair in matches_uniq.pairs()])
pair_1 = flex.int([pair[1] for pair in matches_uniq.pairs()])
group_id_list = flex.int(
[pair_0[pair_1[i]] for i in range(len(matches_uniq.pairs()))])
tally = Counter()
for elem in group_id_list:
tally[elem] += 1
cn_group = len(tally)
return cn_group, group_id_list, mi_all_sort, mio_all_sort, \
I_all_sort, sigI_all_sort, G_all_sort, B_all_sort, \
p_all_sort, rs_all_sort, wavelength_all_sort, sin_all_sort, SE_all_sort, uc_mean, \
wavelength_mean, pickle_filename_all_sort, ""
def add_cells_and_files(self, cells, symm_str):
self.cells = cells
# Table
table_str = ""
for idx, xac in enumerate(cells):
cell = cells[xac]
table_str += "<tr>\n"
table_str += " <td>%.4d</td><td>%s</td>" % (idx+1, xac) # idx, file
table_str += "".join(map(lambda x: "<td>%.2f</td>"%x, cell))
table_str += "\n</tr>\n"
# Hist
cellconstr = CellConstraints(sgtbx.space_group_info(symm_str).group())
show_flags = (True, not cellconstr.is_b_equal_a(), not cellconstr.is_c_equal_a_b(),
not cellconstr.is_angle_constrained("alpha"),
not cellconstr.is_angle_constrained("beta"),
not cellconstr.is_angle_constrained("gamma"))
names = ("a", "b", "c", "α", "β", "γ")
hist_str = ""
label1 = ""
for i, (name, show) in enumerate(zip(names, show_flags)):
tmp = ""
if i in (0,3): tmp += "<tr>"
if show: tmp += "<th>%s</th>" % name
if i in (2,5): tmp += "</tr>"
if i < 3: hist_str += tmp
else: label1 += tmp
hist_str += "\n<tr>\n"
for idx, (name, show) in enumerate(zip(names, show_flags)):
if idx==3: hist_str += "</tr>" + label1 + "<tr>"
if not show: continue
vals = flex.double(map(lambda x: x[idx], cells.values()))
if len(vals) == 0: continue
nslots = max(30, int((max(vals) - min(vals)) / 0.5))
hist = flex.histogram(vals, n_slots=nslots)
x_vals = map(lambda i: hist.data_min() + hist.slot_width() * (i+.5), xrange(len(hist.slots())))
y_vals = hist.slots()
hist_str += """
<td>
<div id="chartdiv_cell%(idx)d" style="width: 500px; height: 400px;"></div>
<script>
var chart_cell%(idx)d = AmCharts.makeChart("chartdiv_cell%(idx)d", {
"type": "serial",
"theme": "none",
"legend": {
"useGraphSettings": true,
"markerSize":12,
"valueWidth":0,
"verticalGap":0
},
"dataProvider": [%(data)s],
"valueAxes": [{
"minorGridAlpha": 0.08,
"minorGridEnabled": true,
"position": "top",
"axisAlpha":0
}],
"graphs": [{
"balloonText": "[[category]]: [[value]]",
"title": "%(name)s",
"type": "column",
"fillAlphas": 0.8,
"valueField": "yval"
}],
"rotate": false,
"categoryField": "xval",
"categoryAxis": {
"gridPosition": "start",
"title": ""
}
});
</script>
</td>
""" % dict(idx=idx, name=name,
data=",".join(map(lambda x: '{"xval":%.2f,"yval":%d}'%x, zip(x_vals,y_vals)))
)
hist_str += "</tr>"
self.html_inputfiles = """
<h2>Input files</h2>
%d files for merging in %s symmetry
<h3>Unit cell histogram</h3>
<table>
%s
</table>
<h3>Files</h3>
<a href="#" onClick="toggle_show('div-input-files'); return false;">Show/Hide</a>
<div id="div-input-files" style="display:none;">
<table class="cells">
<tr>
<th>idx</th> <th>file</th> <th>a</th> <th>b</th> <th>c</th> <th>α</th> <th>β</th> <th>γ</th>
</tr>
%s
</table>
</div>
""" % (len(cells), symm_str, hist_str, table_str)
self.write_html()
def __init__(O,
sites_cart,
density_map,
gradients_method,
weight_map=None,
unit_cell=None,
selection_variable=None,
selection_variable_real_space=None,
geometry_restraints_manager=None,
energies_sites_flags=None,
real_space_target_weight=1,
real_space_gradients_delta=None,
local_standard_deviations_radius=None,
weight_map_scale_factor=None,
lbfgs_termination_params=None,
lbfgs_exception_handling_params=None,
states_collector=None):
assert [unit_cell, geometry_restraints_manager].count(None) == 1
assert real_space_gradients_delta is not None
if (unit_cell is None):
unit_cell = geometry_restraints_manager.crystal_symmetry.unit_cell(
)
if (selection_variable_real_space is not None):
assert selection_variable_real_space.size() == sites_cart.size()
else:
selection_variable_real_space = flex.bool(sites_cart.size(), True)
O.gradients_method = gradients_method
O.x_previous = None
O.states_collector = states_collector
O.density_map = density_map
O.weight_map = weight_map
O.unit_cell = unit_cell
O.sites_cart = sites_cart
O.geometry_restraints_manager = geometry_restraints_manager
O.energies_sites_flags = energies_sites_flags
O.real_space_target_weight = real_space_target_weight
O.real_space_gradients_delta = real_space_gradients_delta
O.local_standard_deviations_radius = local_standard_deviations_radius
O.selection_variable_real_space = selection_variable_real_space
if (O.local_standard_deviations_radius is None):
O.site_radii = None
else:
O.site_radii = flex.double(O.sites_cart.size(),
O.local_standard_deviations_radius)
O.weight_map_scale_factor = weight_map_scale_factor
O.selection_variable = selection_variable
if (O.selection_variable is None):
O.sites_cart = sites_cart
O.x = sites_cart.as_double()
else:
O.sites_cart = sites_cart.deep_copy()
O.x = sites_cart.select(O.selection_variable).as_double()
O.number_of_function_evaluations = -1
O.f_start, O.g_start = O.compute_functional_and_gradients()
O.minimizer = scitbx.lbfgs.run(
target_evaluator=O,
termination_params=lbfgs_termination_params,
exception_handling_params=lbfgs_exception_handling_params)
O.f_final, O.g_final = O.compute_functional_and_gradients()
del O.x
del O.site_radii
def fit_side_chain(self, clusters):
rotamer_iterator = \
mmtbx.refinement.real_space.fit_residue.get_rotamer_iterator(
mon_lib_srv = self.mon_lib_srv,
residue = self.residue)
if(rotamer_iterator is None): return
selection_clash = self.co.clash_eval_selection
selection_rsr = self.co.rsr_eval_selection
if(self.target_map is not None):
start_target_value = self.get_target_value(
sites_cart = self.residue.atoms().extract_xyz(),
selection = selection_rsr)
sites_cart_start = self.residue.atoms().extract_xyz()
sites_cart_first_rotamer = list(rotamer_iterator)[0][1]
# From this point on the coordinates in residue are to initial rotamer!
self.residue.atoms().set_xyz(sites_cart_first_rotamer)
axes = []
atr = []
for i, angle in enumerate(self.chi_angles[0]):
cl = clusters[i]
axes.append(flex.size_t(cl.axis))
atr.append(flex.size_t(cl.atoms_to_rotate))
#
if(self.target_map is not None and self.xyzrad_bumpers is not None):
# Get reference map values
ref_map_vals = flex.double()
for a in self.residue.atoms():
key = "%s_%s_%s"%(
a.parent().parent().parent().id, a.parent().resname,
a.name.strip())
ref_map_vals.append(self.cmv[key])
# Get radii
radii = mmtbx.refinement.real_space.get_radii(
residue = self.residue, vdw_radii = self.vdw_radii)
# Exclude rotatable H from clash calculation
tmp = flex.size_t()
for i in selection_clash:
if(self.rotatable_hd[self.residue.atoms()[i].i_seq]): continue
tmp.append(i)
selection_clash = tmp[:]
# Ad hoc: S or SE have larger peaks!
if(self.residue.resname in ["MET","MSE"]): scale=100
else: scale=3
moving = ext.moving(
sites_cart = self.residue.atoms().extract_xyz(),
sites_cart_start = sites_cart_start,
radii = radii,
weights = self.weights,
bonded_pairs = self.pairs,
ref_map_max = ref_map_vals * scale,
ref_map_min = ref_map_vals / 10)
#
ro = ext.fit(
fixed = self.xyzrad_bumpers,
axes = axes,
rotatable_points_indices = atr,
angles_array = self.chi_angles,
density_map = self.target_map,
moving = moving,
unit_cell = self.unit_cell,
selection_clash = selection_clash,
selection_rsr = selection_rsr, # select atoms to compute map target
sin_table = self.sin_cos_table.sin_table,
cos_table = self.sin_cos_table.cos_table,
step = self.sin_cos_table.step,
n = self.sin_cos_table.n)
elif(self.target_map is not None and self.xyzrad_bumpers is None):
ro = ext.fit(
target_value = start_target_value,
axes = axes,
rotatable_points_indices = atr,
angles_array = self.chi_angles,
density_map = self.target_map,
all_points = self.residue.atoms().extract_xyz(),
unit_cell = self.unit_cell,
selection = selection_rsr,
sin_table = self.sin_cos_table.sin_table,
cos_table = self.sin_cos_table.cos_table,
step = self.sin_cos_table.step,
n = self.sin_cos_table.n)
else:
ro = ext.fit(
sites_cart_start = sites_cart_start.deep_copy(),
axes = axes,
rotatable_points_indices = atr,
angles_array = self.chi_angles,
all_points = self.residue.atoms().extract_xyz(),
sin_table = self.sin_cos_table.sin_table,
cos_table = self.sin_cos_table.cos_table,
step = self.sin_cos_table.step,
n = self.sin_cos_table.n)
sites_cart_result = ro.result()
if(sites_cart_result.size()>0):
dist = None
if(self.accept_only_if_max_shift_is_smaller_than is not None):
dist = flex.max(flex.sqrt((sites_cart_start - sites_cart_result).dot()))
if(dist is None):
self.residue.atoms().set_xyz(sites_cart_result)
else:
if(dist is not None and
dist < self.accept_only_if_max_shift_is_smaller_than):
self.residue.atoms().set_xyz(sites_cart_result)
else:
self.residue.atoms().set_xyz(sites_cart_start)
else:
self.residue.atoms().set_xyz(sites_cart_start)
if(self.m): self.m.add(residue = self.residue, state = "fitting")
# # tune up
if(self.target_map is not None):
tune_up(
target_map = self.target_map,
residue = self.residue,
mon_lib_srv = self.mon_lib_srv,
rotamer_manager = self.rotamer_manager.rotamer_evaluator,
unit_cell = self.unit_cell,
monitor = self.m,
torsion_search_start = -30,
torsion_search_stop = 30,
torsion_search_step = 1)
def run_detail(show_plot, save_plot):
P = Profiler("0. Read data")
import sys
file_name = sys.argv[1]
from xfel.clustering.singleframe import CellOnlyFrame
from cctbx import crystal
cells = []
for line in open(file_name, "r"):
tokens = line.strip().split()
unit_cell = tuple(float(x) for x in tokens[0:6])
space_group_symbol = tokens[6]
crystal_symmetry = crystal.symmetry(
unit_cell=unit_cell, space_group_symbol=space_group_symbol)
cells.append(CellOnlyFrame(crystal_symmetry, path=None))
MM = [c.mm for c in cells] # get all metrical matrices
MM_double = flex.double()
for i in range(len(MM)):
Tup = MM[i]
for j in range(6):
MM_double.append(Tup[j])
print(("There are %d cells" % (len(MM))))
coord_x = flex.double([c.uc[0] for c in cells])
coord_y = flex.double([c.uc[1] for c in cells])
if show_plot or save_plot:
import matplotlib
if not show_plot:
# http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear
matplotlib.use('Agg') # use a non-interactive backend
from matplotlib import pyplot as plt
plt.plot([c.uc[0] for c in cells], [c.uc[1] for c in cells],
"k.",
markersize=3.)
plt.axes().set_aspect("equal")
if save_plot:
plt.savefig(plot_name,
size_inches=(10, 10),
dpi=300,
bbox_inches='tight')
if show_plot:
plt.show()
print("Now constructing a Dij matrix.")
P = Profiler("1. compute Dij matrix")
NN = len(MM)
import omptbx
omptbx.omp_set_num_threads(64)
from cctbx.uctbx.determine_unit_cell import NCDist_matrix, NCDist_flatten
#Dij = NCDist_matrix(MM_double) # double loop, less efficient evaluation of upper triangle
Dij = NCDist_flatten(MM_double) # loop is flattened
#from cctbx.uctbx.determine_unit_cell import NCDist # can this be refactored with MPI?
#Dij = flex.double(flex.grid(NN,NN))
#for i in xrange(NN):
# for j in xrange(i+1,NN):
# Dij[i,j] = NCDist(MM[i], MM[j])
del P
d_c = 10 # the distance cutoff, such that average item neighbors 1-2% of all items
CM = clustering_manager(Dij=Dij, d_c=d_c)
# Summarize the results here
n_cluster = 1 + flex.max(CM.cluster_id_final)
print(len(cells), "have been analyzed")
print(("# ------------ %d CLUSTERS ----------------" % (n_cluster)))
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
print("Cluster %d. Central unit cell: item %d" % (i, item))
cells[item].crystal_symmetry.show_summary()
print("Cluster has %d items, or %d after trimming borders" %
((CM.cluster_id_full == i).count(True),
(CM.cluster_id_final == i).count(True)))
print()
appcolors = [
'b', 'r', '#ff7f0e', '#2ca02c', '#9467bd', '#8c564b', '#e377c2',
'#7f7f7f', '#bcbd22', '#17becf'
]
if show_plot:
#Decision graph
from matplotlib import pyplot as plt
plt.plot(CM.rho, CM.delta, "r.", markersize=3.)
for x in range(NN):
if CM.cluster_id_maxima[x] >= 0:
plt.plot([CM.rho[x]], [CM.delta[x]], "ro")
plt.show()
#No-halo plot
from matplotlib import pyplot as plt
colors = [appcolors[i % 10] for i in CM.cluster_id_full]
plt.scatter(coord_x,
coord_y,
marker='o',
color=colors,
linewidths=0.4,
edgecolor='k')
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
plt.plot([cells[item].uc[0]], [cells[item].uc[1]], 'y.')
plt.axes().set_aspect("equal")
plt.show()
#Final plot
halo = (CM.cluster_id_final == -1)
core = ~halo
plt.plot(coord_x.select(halo), coord_y.select(halo), "k.")
colors = [appcolors[i % 10] for i in CM.cluster_id_final.select(core)]
plt.scatter(coord_x.select(core),
coord_y.select(core),
marker="o",
color=colors,
linewidths=0.4,
edgecolor='k')
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
plt.plot([cells[item].uc[0]], [cells[item].uc[1]], 'y.')
plt.axes().set_aspect("equal")
plt.show()
def exercise_Compute_phifom_from_abcd_interface(args):
miller_arrays = reflection_file_converter.run(
args=args, simply_return_all_miller_arrays=True)
complex_input = None
for miller_array in miller_arrays:
if (miller_array.is_complex_array()):
complex_input = miller_array
break
if (complex_input is not None):
print("complex_input.info():", complex_input.info())
fom_input = None
for miller_array in miller_arrays:
if (miller_array.is_real_array()
and miller_array.info().lower().find("fom")):
fom_input = miller_array
break
if (fom_input is not None):
print("fom_input.info():", fom_input.info())
for miller_array in miller_arrays:
if (isinstance(miller_array.data(), flex.hendrickson_lattman)):
print("Hendrickson-Lattman coefficients:", miller_array.info())
n_bad_figures_of_merit = 0
n_bad_phases = 0
if (miller_array.anomalous_flag()):
print("Clipper cannot currently handle anomalous arrays" \
+ " with Hendrickson-Lattman coefficients.")
continue
phi_fom = clipper.Compute_phifom_from_abcd_interface(
unit_cell=miller_array.unit_cell(),
space_group=miller_array.space_group(),
miller_indices=miller_array.indices(),
phase_probabilities=miller_array.data())
phi_clipper = phi_fom.centroid_phases() * 180 / math.pi
fom_clipper = phi_fom.figures_of_merit()
fom_deltas = fom_input.data() - fom_clipper
if (fom_input is not None):
perm = flex.sort_permutation(flex.abs(fom_deltas),
reverse=True)
fom_deltas_sorted = fom_deltas.select(perm)
fom_clipper_sorted = fom_clipper.select(perm)
fom_input_sorted = fom_input.data().select(perm)
pp_sorted = miller_array.data().select(perm)
indices_sorted = miller_array.indices().select(perm)
centric_flags_sorted = fom_input.centric_flags().data().select(
perm)
print("FOM comparison")
for f, i, c, d, p, ix in zip(centric_flags_sorted,
fom_input_sorted,
fom_clipper_sorted,
fom_deltas_sorted, pp_sorted,
indices_sorted):
print(f, "%.3f %.3f %.3f" % (i, c, d), end=' ')
if (abs(d) > 0.01):
print("LOOK", "%.2f %.2f %.2f %.2f" % p, ix, end=' ')
n_bad_figures_of_merit += 1
print()
print()
if (complex_input is not None):
phi_input = complex_input.phases(deg=True).data()
phi_deltas = flex.double()
for phi1, phi2 in zip(phi_input, phi_clipper):
phi_deltas.append(
scitbx.math.signed_phase_error(phi1, phi2, deg=True))
perm = flex.sort_permutation(flex.abs(phi_deltas),
reverse=True)
phi_deltas_sorted = phi_deltas.select(perm)
phi_clipper_sorted = phi_clipper.select(perm)
phi_input_sorted = phi_input.select(perm)
pp_sorted = miller_array.data().select(perm)
indices_sorted = miller_array.indices().select(perm)
centric_flags_sorted = complex_input.centric_flags().data(
).select(perm)
fom_clipper_sorted = fom_clipper.select(perm)
print("PHI comparison")
for f, i, c, d, m, p, ix in zip(centric_flags_sorted,
phi_input_sorted,
phi_clipper_sorted,
phi_deltas_sorted,
fom_clipper_sorted, pp_sorted,
indices_sorted):
if (m < 0.01): continue
print(f, "%.3f %.3f %.3f" % (i, c, d), end=' ')
if (abs(d) > 3 and (max(p) < 1000 or abs(d) > 10)):
print("LOOK",
"%.2f %.2f %.2f %.2f" % p,
"fom=%.3f" % m,
ix,
end=' ')
n_bad_phases += 1
print()
print()
assert n_bad_figures_of_merit == 0
assert n_bad_phases == 0
def __init__(self,
residue,
mon_lib_srv,
rotamer_manager,
sin_cos_table,
cmv,
unit_cell,
rotatable_hd=None,
vdw_radii=None,
xyzrad_bumpers=None,
target_map=None,
target_map_for_cb=None,
backbone_sample=False,
accept_only_if_max_shift_is_smaller_than=None,
log=None):
adopt_init_args(self, locals())
if(self.log is None): self.log = sys.stdout
self.co = mmtbx.refinement.real_space.aa_residue_axes_and_clusters(
residue = self.residue,
mon_lib_srv = self.mon_lib_srv,
backbone_sample = True,
log = self.log)
self.m = None
if(self.target_map is not None and len(self.co.clusters)>0):
# Set weights
AN = {"S":16, "O":8, "N":7, "C":6, "SE":34, "H":1, "D":5}
#AN = {"S":1, "O":1, "N":1, "C":1, "SE":1, "H":1}
self.weights = flex.double()
for atom in self.residue.atoms():
self.weights.append(AN[atom.element.strip().upper()])
# Bonded pairs
exclude = ["C","N","O","CA"]
reference = exclude + ["CB"]
atoms = self.residue.atoms()
self.pairs = []
for i, ai in enumerate(atoms):
if(ai.name.strip() in reference):
mv = self.target_map.eight_point_interpolation(
self.unit_cell.fractionalize(ai.xyz))
if(ai.name.strip() in exclude): continue
if(ai.element.strip().upper() in ["H","S","SE"]): continue
for j, aj in enumerate(atoms):
if i==j: continue
if(aj.name.strip() in exclude): continue
if(aj.element.strip().upper() in ["H","S","SE"]): continue
d = ai.distance(aj)
if d < 1.6:
pair = [i,j]
pair.sort()
if(not pair in self.pairs):
self.pairs.append(pair)
# Set monitor
id_str=""
if(self.residue.parent() is not None and
self.residue.parent().parent() is not None):
id_str+="chain: %s"%(self.residue.parent().parent().id)
id_str+=" residue: %s %s"%(self.residue.resname, self.residue.resseq.strip())
if(len(self.co.clusters)>1):
msel = flex.size_t(flatten(self.co.clusters[1:][0].vector))
else:
msel = flex.size_t()
self.m = monitor(
id_str = id_str,
selection = msel,
map_data = self.target_map,
unit_cell = self.unit_cell,
weights = self.weights,
pairs = self.pairs,
cmv = self.cmv,
rotamer_evaluator = self.rotamer_manager.rotamer_evaluator,
log = self.log)
self.m.add(residue = self.residue, state = "start")
if(self.target_map is None):
assert not backbone_sample
# Actual calculations
self.chi_angles = self.rotamer_manager.get_chi_angles(
resname = self.residue.resname)
if(len(self.co.clusters)>0):
if(backbone_sample):
self.fit_c_beta(c_beta_rotation_cluster = self.co.clusters[0])
self.fit_side_chain(clusters = self.co.clusters[1:])
if(self.m is not None):
self.m.finalize(residue = self.residue)
