def plot_positions(values, positions, file_name, cmap=pyplot.cm.Reds,
vmin=None, vmax=None, invalid='white'):
values = values.as_double()
assert positions.size() >= values.size()
positions = positions[:values.size()]
if vmin is None:
vmin = flex.min(values)
if vmax is None:
vmax = flex.max(values)
x, y = positions.parts()
dx = flex.abs(x[1:] - x[:-1])
dy = flex.abs(y[1:] - y[:-1])
dx = dx.select(dx > 0)
dy = dy.select(dy > 0)
scale = 1/flex.min(dx)
#print scale
x = (x * scale).iround()
y = (y * scale).iround()
from libtbx.math_utils import iceil
z = flex.double(flex.grid(iceil(flex.max(y))+1, iceil(flex.max(x))+1), -2)
#print z.all()
for x_, y_, z_ in zip(x, y, values):
z[y_, x_] = z_
plot_grid(z.as_1d(), z.all(), file_name, cmap=cmap, vmin=vmin, vmax=vmax,
invalid=invalid)
return
def test_grid_step(n_sites = 50,
volume_per_atom = 50,
d_min = 2.0):
grid_step = (0.2,0.4,0.6,0.7,0.9,1.0)
for step in grid_step:
symmetry = crystal.symmetry(space_group_symbol="P1")
structure = random_structure.xray_structure(space_group_info = symmetry.space_group_info(),
elements=["C"]*n_sites,
volume_per_atom=volume_per_atom,
random_u_iso=False)
fc = structure.structure_factors(d_min = d_min,
anomalous_flag=False,
algorithm="fft").f_calc()
manager = max_like_non_uniform.ordered_solvent_distribution(
structure = structure,
fo = fc,
grid_step = step)
f_water_dist = manager.fcalc_from_distribution()
### check phase compatibility with the symmetry:
centrics = f_water_dist.select_centric()
if(centrics.indices().size() > 0):
ideal = centrics.phase_transfer(centrics)
assert flex.max(flex.abs(ideal.data() - centrics.data())) < 1.e-6
###
#print "max = ", flex.max( flex.abs( f_water_dist.data() ) )
#print "min = ", flex.min( flex.abs( f_water_dist.data() ) )
#print "ave = ", flex.mean( flex.abs( f_water_dist.data() ) )
assert flex.max( flex.abs( f_water_dist.data() ) ) < 1.0
def __init__(self, intensities, dose, n_bins=8,
range_min=None, range_max=None, range_width=1):
self.intensities = intensities
self.dose = dose
self.n_bins = n_bins
self.range_min = range_min
self.range_max = range_max
self.range_width = range_width
assert self.range_width > 0
if self.range_min is None:
self.range_min = flex.min(self.dose) - self.range_width
if self.range_max is None:
self.range_max = flex.max(self.dose)
self.n_steps = 2 + int((self.range_max - self.range_min) - self.range_width)
sel = (self.dose.as_double() <= self.range_max) & (self.dose.as_double() >= self.range_min)
self.dose = self.dose.select(sel)
self.intensities = self.intensities.select(sel)
self.d_star_sq = self.intensities.d_star_sq().data()
self.binner = self.intensities.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(self.d_star_sq)-flex.min(self.d_star_sq)+1e-8)/self.n_bins)
self.observations = unmerged_observations(self.intensities)
self._calc_completeness_vs_dose()
self._calc_rcp_scp()
self._calc_rd()
def linear_regression_test(d_analytical, d_numerical, test_hard=True,
slope_tolerance=1.e-3,
correlation_min=0.999,
verbose=0):
if (type(d_analytical) != type(flex.double())):
d_analytical = flex_tuple_as_flex_double(d_analytical)
if (type(d_numerical) != type(flex.double())):
d_numerical = flex_tuple_as_flex_double(d_numerical)
if (0 or verbose):
print "analytical:", tuple(d_analytical)
print "numerical: ", tuple(d_numerical)
if ( flex.max(flex.abs(d_analytical)) == 0
and flex.max(flex.abs(d_numerical)) == 0):
return
regr = flex.linear_regression(d_analytical, d_numerical)
corr = flex.linear_correlation(d_analytical, d_numerical).coefficient()
assert regr.is_well_defined()
if (abs(regr.slope() - 1) > slope_tolerance or corr < correlation_min):
print "Error: finite difference mismatch:"
print "slope:", regr.slope()
print "correlation:", corr
if (0 or verbose):
for a, n in zip(d_analytical, d_numerical):
print a, n
assert not test_hard
def miller_array_export_as_shelx_hklf(
miller_array,
file_object=None,
normalise_if_format_overflow=False):
"""\
If the maximum data value does not fit into the f8.2/f8.0 format:
normalise_if_format_overflow=False: RuntimeError is thrown
normalise_if_format_overflow=True: data is normalised to the largest
number to fit f8.2/f8.0 format
"""
assert miller_array.is_real_array()
if (file_object is None): file_object = sys.stdout
def raise_f8_overflow(v):
raise RuntimeError("SHELX HKL file F8.2/F8.0 format overflow: %.8g" % v)
data = miller_array.data()
sigmas = miller_array.sigmas()
assert data is not None
min_val = flex.min(data)
max_val = flex.max(data)
if (sigmas is not None):
min_val = min(min_val, flex.min(sigmas))
max_val = max(max_val, flex.max(sigmas))
min_sc = 1
max_sc = 1
if (min_val < 0):
s = "%8.0f" % min_val
if (len(s) > 8):
if (not normalise_if_format_overflow):
raise_f8_overflow(min_val)
min_sc = -9999999. / min_val
if (max_val > 0):
s = "%8.0f" % max_val
if (len(s) > 8):
if (not normalise_if_format_overflow):
raise_f8_overflow(max_val)
max_sc = 99999999. / max_val
scale = min(min_sc, max_sc)
sigmas = miller_array.sigmas()
s = 0.01
for i,h in enumerate(miller_array.indices()):
if (sigmas is not None): s = sigmas[i]
def fmt_3i4(h):
result = "%4d%4d%4d" % h
if (len(result) != 12):
raise RuntimeError(
"SHELXL HKL file 3I4 format overflow: %s" % result)
return result
def fmt_f8(v):
result = "%8.2f" % v
if (len(result) != 8):
result = "%8.1f" % v
if (len(result) != 8):
result = "%8.0f" % v
assert len(result) == 8
return result
line = fmt_3i4(h) + fmt_f8(data[i]*scale) + fmt_f8(s*scale)
print >> file_object, line
print >> file_object, " 0 0 0 0.00 0.00"
def verify_miller_arrays(a1, a2, eps=1.0e-5):
v = a2.adopt_set(a1)
if a1.is_bool_array():
if a2.is_integer_array():
assert flex.max(flex.abs(a1.data().as_int() - v.data())) == 0
else:
assert flex.max(flex.abs(a1.data().as_double() - v.data())) < eps
elif a1.is_hendrickson_lattman_array():
for i in xrange(4):
assert flex.max(flex.abs(a1.data().slice(i) - v.data().slice(i))) < eps
else:
assert flex.max(flex.abs(a1.data() - v.data())) < eps
if v.sigmas() is not None:
assert flex.max(flex.abs(a1.sigmas() - v.sigmas())) < eps
def minimize_kb(self, use_curvatures_options,
set_use_scale_options=[True, False], n_cycles=5):
#print "minimize_kb, r:", self.kbu.r_factor()
for use_curvatures in use_curvatures_options*n_cycles:
start_r = self.kbu.r_factor()
save_k_sols = self.kbu.k_sols()
save_b_sols = self.kbu.b_sols()
#self.set_use_scale(value = random.choice(set_use_scale_options))
self.set_use_scale(value = True)
m = self.minimize_kb_once(use_curvatures=use_curvatures)
r = self.kbu.r_factor()
if(r>start_r and r>1.e-2 and (flex.min(self.kbu.k_sols())<0 or
flex.max(self.kbu.k_sols())>1 or flex.min(self.kbu.b_sols())<0 or
flex.max(self.kbu.k_sols())>100.)):
self.kbu.update(k_sols = save_k_sols, b_sols = save_b_sols)
def exercise_xray_structure(use_u_aniso, verbose=0):
structure = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info("P 31"),
elements=["N", "C", "C", "O", "Si"] * 2,
volume_per_atom=500,
min_distance=2.0,
general_positions_only=False,
random_u_iso=True,
use_u_aniso=use_u_aniso,
)
f_abs = abs(structure.structure_factors(anomalous_flag=False, d_min=2, algorithm="direct").f_calc())
for resname in (None, "res"):
for fractional_coordinates in (False, True):
pdb_file = structure.as_pdb_file(
remark="Title", remarks=["Any", "Thing"], fractional_coordinates=fractional_coordinates, resname=resname
)
if 0 or verbose:
sys.stdout.write(pdb_file)
structure_read = iotbx.pdb.input(
source_info=None, lines=flex.std_string(pdb_file.splitlines())
).xray_structure_simple(fractional_coordinates=fractional_coordinates, use_scale_matrix_if_available=False)
f_read = abs(
f_abs.structure_factors_from_scatterers(xray_structure=structure_read, algorithm="direct").f_calc()
)
regression = flex.linear_regression(f_abs.data(), f_read.data())
assert regression.is_well_defined()
if 0 or verbose:
regression.show_summary()
assert approx_equal(regression.slope(), 1, eps=1.0e-2)
assert approx_equal(regression.y_intercept(), 0, eps=flex.max(f_abs.data()) * 0.01)
def show_literature_fits(label, n_terms, null_fit, n_points, e_other=None):
for lib in [xray_scattering.wk1995,
xray_scattering.it1992,
xray_scattering.two_gaussian_agarwal_isaacs,
xray_scattering.two_gaussian_agarwal_1978,
xray_scattering.one_gaussian_agarwal_1978]:
if (lib == xray_scattering.wk1995):
try:
lib_gaussian = xray_scattering.wk1995(label, True).fetch()
lib_source = "WK1995"
except Exception:
lib_gaussian = None
elif (lib == xray_scattering.it1992):
try:
lib_gaussian = xray_scattering.it1992(label, True).fetch()
lib_source = "IT1992"
except Exception:
lib_gaussian = None
elif (lib.table.has_key(label)):
lib_gaussian = lib.table[label]
lib_source = lib.source_short
else:
lib_gaussian = None
if (lib_gaussian is not None):
gaussian_fit = scitbx.math.gaussian.fit(
null_fit.table_x()[:n_points],
null_fit.table_y()[:n_points],
null_fit.table_sigmas()[:n_points],
lib_gaussian)
e = flex.max(gaussian_fit.significant_relative_errors())
show_fit_summary(lib_source, label, gaussian_fit, e,
e_other, lib_gaussian.n_terms())
def tst_hes_ls_f_wt(self,h=0.0000001):
hes_anal = self.ls_f_wt.hessian_as_packed_u()
hes_anal=hes_anal.matrix_packed_u_as_symmetric()
grads = self.ls_f_wt.get_gradient()
self.ls_f_wt.set_p_scale(self.p_scale+h)
tmp = self.ls_f_wt.get_gradient()
tmp = list( (grads-tmp)/-h )
tmp_hess=[]
tmp_hess.append( tmp )
self.ls_f_wt.set_p_scale(self.p_scale)
for ii in range(6):
u_tmp=list(flex.double(self.u).deep_copy())
u_tmp[ii]+=h
self.ls_f_wt.set_u_rwgk(u_tmp)
tmp = self.ls_f_wt.get_gradient()
tmp = (grads - tmp)/-h
tmp_hess.append( list(tmp) )
self.ls_f_wt.set_u_rwgk(self.u)
f = max(1, flex.max(flex.abs(hes_anal)))
count=0
for ii in range(7):
for jj in range(7):
assert approx_equal(tmp_hess[ii][jj]/f, hes_anal[count]/f)
count+=1
def gradients(self, xray_structure, force_update_mask=False):
factor = 1.0
sites_cart = xray_structure.sites_cart()
if(self.fmodel is not None):
max_shift = flex.max(flex.sqrt((self.sites_cart - sites_cart).dot()))
if(max_shift > self.update_gradient_threshold):
self.fmodel.update_xray_structure(
xray_structure = xray_structure,
update_f_calc = True,
update_f_mask = False)
self.gx = flex.vec3_double(self.x_target_functor(compute_gradients=True).\
gradients_wrt_atomic_parameters(site=True).packed())
self.sites_cart = sites_cart
if(self.restraints_manager is not None):
c = self.restraints_manager.energies_sites(sites_cart = sites_cart,
compute_gradients=True)
self.gc = c.gradients
factor *= self.wc
if(c.normalization_factor is not None): factor *= c.normalization_factor
result = None
if(self.wx is not None):
result = self.wx * self.gx
if(self.wc is not None):
gcw = self.wc * self.gc
if(result is None): result = gcw
else: result = result + gcw
if(factor != 1.0): result *= 1.0 / factor
#print "norms:", self.gc.norm(), self.gx.norm(), result.norm()
return result
def plot_overall_completeness(completeness):
completeness_range = xrange(-1, flex.max(completeness) + 1)
completeness_counts = [completeness.count(n) for n in completeness_range]
from matplotlib import pyplot as plt
plt.plot(completeness_range, completeness_counts, "r+")
plt.show()
def __init__(self, f_obs, ncs_pairs, reflections_per_bin):
adopt_init_args(self, locals())
# Create bins
f_obs.setup_binner(reflections_per_bin = reflections_per_bin)
self.binner = f_obs.binner()
n_bins = self.binner.n_bins_used()
self.n_bins = n_bins
self.SigmaN = None
self.update_SigmaN()
#
self.rbin = flex.int(f_obs.data().size(), -1)
for i_bin in self.binner.range_used():
for i_seq in self.binner.array_indices(i_bin):
self.rbin[i_seq] = i_bin-1 # i_bin starts with 1, not 0 !
assert flex.min(self.rbin)==0
assert flex.max(self.rbin)==n_bins-1
# Extract symmetry matrices
self.sym_matrices = []
for m_as_string in f_obs.space_group().smx():
o = sgtbx.rt_mx(symbol=str(m_as_string), t_den=f_obs.space_group().t_den())
m_as_double = o.r().as_double()
self.sym_matrices.append(m_as_double)
self.gradient_evaluator = None
self.target_and_grads = ext.tncs_eps_factor_refinery(
tncs_pairs = self.ncs_pairs,
f_obs = self.f_obs.data(),
sigma_f_obs = self.f_obs.sigmas(),
rbin = self.rbin,
SigmaN = self.SigmaN,
space_group = self.f_obs.space_group(),
miller_indices = self.f_obs.indices(),
fractionalization_matrix = self.f_obs.unit_cell().fractionalization_matrix(),
sym_matrices = self.sym_matrices)
self.update()
def target_and_gradients(self, xray_structure, to_compute_weight=False):
if(to_compute_weight):
xrs = xray_structure.deep_copy_scatterers()
# This may be useful to explore:
#xrs.shake_adp_if_all_equal(b_iso_tolerance = 1.e-3)
#xrs.shake_adp(spread=10, keep_anisotropic= False)
else:
xrs = xray_structure
if(self.refine_adp): params = xrs.extract_u_iso_or_u_equiv()
if(self.refine_occ): params = xrs.scatterers().extract_occupancies()
if(to_compute_weight):
pmin = flex.min(params)
pmax = flex.max(params)
if(abs(pmin-pmax)/abs(pmin+pmax)*2*100<1.e-3):
pmean = flex.mean(params)
n_par = params.size()
params = flex.double()
for i in xrange(n_par):
params.append(pmean + 0.1 * pmean * random.choice([-1,0,1]))
return crystal.adp_iso_local_sphere_restraints_energies(
pair_sym_table = self.pair_sym_table,
orthogonalization_matrix = self.orthogonalization_matrix,
sites_frac = self.sites_frac,
u_isos = params,
selection = self.selection,
use_u_iso = self.selection,
grad_u_iso = self.selection,
sphere_radius = self.sphere_radius,
distance_power = 2,
average_power = 1,
min_u_sum = 1.e-6,
compute_gradients = True,
collect = False)
def exercise_1(grid_step,
radius,
shell,
a,
b,
buffer_layer,
site_cart):
xray_structure = xray_structure_of_one_atom(site_cart = site_cart,
buffer_layer = buffer_layer,
a = a,
b = b)
sampled_density = mmtbx.real_space.sampled_model_density(
xray_structure = xray_structure,
grid_step = grid_step)
site_frac = xray_structure.unit_cell().fractionalization_matrix()*site_cart
assert approx_equal(flex.max(sampled_density.data()),
sampled_density.data().value_at_closest_grid_point(site_frac[0]))
around_atom_obj = mmtbx.real_space.around_atom(
unit_cell = xray_structure.unit_cell(),
map_data = sampled_density.data(),
radius = radius,
shell = shell,
site_frac = list(xray_structure.sites_frac())[0])
data_exact = around_atom_obj.data()
dist_exact = around_atom_obj.distances()
approx_obj = maptbx.one_gaussian_peak_approximation(
data_at_grid_points = data_exact,
distances = dist_exact,
use_weights = False,
optimize_cutoff_radius = True)
assert approx_equal(approx_obj.a_reciprocal_space(), 6.0, 1.e-3)
assert approx_equal(approx_obj.b_reciprocal_space(), 3.0, 1.e-3)
assert approx_obj.gof() < 0.3
assert approx_obj.cutoff_radius() < radius
def run_fast_nv1995(f_obs, f_calc_fixed, f_calc_p1,
symmetry_flags, gridding, grid_tags, verbose):
if (f_calc_fixed is None):
f_part = flex.complex_double()
else:
f_part = f_calc_fixed.data()
assert f_obs.anomalous_flag() == f_calc_p1.anomalous_flag()
fast_nv1995 = translation_search.fast_nv1995(
gridding=gridding,
space_group=f_obs.space_group(),
anomalous_flag=f_obs.anomalous_flag(),
miller_indices_f_obs=f_obs.indices(),
f_obs=f_obs.data(),
f_part=f_part,
miller_indices_p1_f_calc=f_calc_p1.indices(),
p1_f_calc=f_calc_p1.data())
assert fast_nv1995.target_map().all() == gridding
map_stats = maptbx.statistics(fast_nv1995.target_map())
if (0 or verbose):
map_stats.show_summary()
grid_tags.build(f_obs.space_group_info().type(), symmetry_flags)
assert grid_tags.n_grid_misses() == 0
assert grid_tags.verify(fast_nv1995.target_map())
peak_list = maptbx.peak_list(
data=fast_nv1995.target_map(),
tags=grid_tags.tag_array(),
peak_search_level=1,
max_peaks=10,
interpolate=True)
if (0 or verbose):
print "gridding:", gridding
for i,site in enumerate(peak_list.sites()):
print "(%.4f,%.4f,%.4f)" % site, "%.6g" % peak_list.heights()[i]
assert approx_equal(map_stats.max(), flex.max(peak_list.grid_heights()))
return peak_list
def quick_test(file_name): from libtbx.utils import user_plus_sys_time t = user_plus_sys_time() s = reader(file_name) print "Time read:", t.delta() s.show_summary() print tuple(s.original_indices[:3]) print tuple(s.unique_indices[:3]) print tuple(s.batch_numbers[:3]) print tuple(s.centric_tags[:3]) print tuple(s.spindle_flags[:3]) print tuple(s.asymmetric_unit_indices[:3]) print tuple(s.i_obs[:3]) print tuple(s.sigmas[:3]) print tuple(s.original_indices[-3:]) print tuple(s.unique_indices[-3:]) print tuple(s.batch_numbers[-3:]) print tuple(s.centric_tags[-3:]) print tuple(s.spindle_flags[-3:]) print tuple(s.asymmetric_unit_indices[-3:]) print tuple(s.i_obs[-3:]) print tuple(s.sigmas[-3:]) m = s.as_miller_array(merge_equivalents=False).merge_equivalents() print "min redundancies:", flex.min(m.redundancies().data()) print "max redundancies:", flex.max(m.redundancies().data()) print "mean redundancies:", flex.mean(m.redundancies().data().as_double()) s.as_miller_arrays()[0].show_summary() print
def show_plot(widegrid,excursi):
excursi.reshape(flex.grid(widegrid, widegrid))
plot_max = flex.max(excursi)
idx_max = flex.max_index(excursi)
def igrid(x): return x - (widegrid//2)
idxs = [igrid(i)*plot_px_sz for i in xrange(widegrid)]
from matplotlib import pyplot as plt
plt.figure()
CS = plt.contour([igrid(i)*plot_px_sz for i in xrange(widegrid)],
[igrid(i)*plot_px_sz for i in xrange(widegrid)], excursi.as_numpy_array())
plt.clabel(CS, inline=1, fontsize=10, fmt="%6.3f")
plt.title("Wide scope search for detector origin offset")
plt.scatter([0.0],[0.0],color='g',marker='o')
plt.scatter([new_offset[0]] , [new_offset[1]],color='r',marker='*')
plt.scatter([idxs[idx_max%widegrid]] , [idxs[idx_max//widegrid]],color='k',marker='s')
plt.axes().set_aspect("equal")
plt.xlabel("offset (mm) along beamr1 vector")
plt.ylabel("offset (mm) along beamr2 vector")
plt.show()
#changing value
trial_origin_offset = (idxs[idx_max%widegrid])*beamr1 + (idxs[idx_max//widegrid])*beamr2
return trial_origin_offset
def main():
parser = OptionParser(usage="usage: python %prog [options] file_name ...")
parser.add_option("-c", "--cutoff", type="float", default=6.05, metavar="FLOAT", help="maximum sin(theta)/lambda")
(options, args) = parser.parse_args()
if len(args) < 1:
parser.print_help()
return
cutoff = options.cutoff
for file_name in args:
tab = read_table(file_name)
if tab.element == "Es":
continue
wk = xray_scattering.wk1995(tab.element, True).fetch()
sel = tab.x <= cutoff
tab_x = tab.x.select(sel)
tab_y = tab.y.select(sel)
sigmas = flex.double(tab_x.size(), 0.0005)
wky = wk.at_x(tab_x)
errors_abs = flex.abs(wky - tab_y)
fit = scitbx.math.gaussian.fit(tab_x, tab_y, sigmas, wk)
errors_rel = fit.significant_relative_errors(1.0e-6)
print tab.element, tab.atomic_number,
print "max error < %.1fA-1 abs, rel: %7.4f %7.4f" % (cutoff, flex.max(errors_abs), flex.max(errors_rel))
for x, y, f, ea, er in zip(tab_x, tab_y, wky, errors_abs, errors_rel):
print "%7.4f %7.4f %7.4f %7.4f %7.4f" % (x, y, f, ea, er)
print
def resolution_rmerge(self, limit = None, log = None):
'''Compute a resolution limit where either rmerge = 1.0 (limit if
set) or the full extent of the data. N.B. this fit is only meaningful
for positive values.'''
if limit is None:
limit = self._params.rmerge
rmerge_s = flex.double(
[b.r_merge for b in self._merging_statistics.bins]).reversed()
s_s = flex.double(
[1/b.d_min**2 for b in self._merging_statistics.bins]).reversed()
sel = rmerge_s > 0
rmerge_s = rmerge_s.select(sel)
s_s = s_s.select(sel)
if limit == 0.0:
return 1.0 / math.sqrt(flex.max(s_s))
if limit > flex.max(rmerge_s):
return 1.0 / math.sqrt(flex.max(s_s))
rmerge_f = log_inv_fit(s_s, rmerge_s, 6)
if log:
fout = open(log, 'w')
for j, s in enumerate(s_s):
d = 1.0 / math.sqrt(s)
o = rmerge_s[j]
m = rmerge_f[j]
fout.write('%f %f %f %f\n' % (s, d, o, m))
fout.close()
try:
r_rmerge = 1.0 / math.sqrt(interpolate_value(s_s, rmerge_f, limit))
except:
r_rmerge = 1.0 / math.sqrt(flex.max(s_s))
if self._params.plot:
plot = resolution_plot(ylabel='Rmerge')
plot.plot(s_s, rmerge_f, label='fit')
plot.plot(s_s, rmerge_s, label='Rmerge')
plot.plot_resolution_limit(r_rmerge)
plot.savefig('rmerge.png')
return r_rmerge
def show_xray_structure_statistics(xray_structure, atom_selections, hd_sel = None):
result = group_args(
all = None,
macromolecule = None,
sidechain = None,
solvent = None,
ligand = None,
backbone = None)
if(hd_sel is not None):
xray_structure = xray_structure.select(~hd_sel)
for key in atom_selections.__dict__.keys():
value = atom_selections.__dict__[key]
if(value.count(True) > 0):
if(hd_sel is not None):
value = value.select(~hd_sel)
xrs = xray_structure.select(value)
atom_counts = xrs.scattering_types_counts_and_occupancy_sums()
atom_counts_strs = []
for ac in atom_counts:
atom_counts_strs.append("%s:%s:%s"%(ac.scattering_type,str(ac.count),
str("%10.2f"%ac.occupancy_sum).strip()))
atom_counts_str = " ".join(atom_counts_strs)
b_isos = xrs.extract_u_iso_or_u_equiv()
n_aniso = xrs.use_u_aniso().count(True)
n_not_positive_definite = xrs.is_positive_definite_u().count(False)
b_mean = format_value("%-6.1f",adptbx.u_as_b(flex.mean(b_isos)))
b_min = format_value("%-6.1f",adptbx.u_as_b(flex.min(b_isos)))
b_max = format_value("%-6.1f",adptbx.u_as_b(flex.max(b_isos)))
n_atoms = format_value("%-8d",xrs.scatterers().size()).strip()
n_npd = format_value("%-8s",n_not_positive_definite).strip()
occ = xrs.scatterers().extract_occupancies()
o_mean = format_value("%-6.2f",flex.mean(occ)).strip()
o_min = format_value("%-6.2f",flex.min(occ)).strip()
o_max = format_value("%-6.2f",flex.max(occ)).strip()
tmp_result = group_args(
n_atoms = n_atoms,
atom_counts_str = atom_counts_str,
b_min = b_min,
b_max = b_max,
b_mean = b_mean,
o_min = o_min,
o_max = o_max,
o_mean = o_mean,
n_aniso = n_aniso,
n_npd = n_npd)
setattr(result,key,tmp_result)
return result
def get_pseudo_curvs(): ag_max = flex.max(flex.abs(inp_info.grads)) assert ag_max != 0 dests = (-inp_info.grads/ag_max) * (limits/2) assert flex.abs(dests).all_le(limits/2*(1+1e-6)) assert (dests > 0).all_eq(inp_info.grads < 0) O.pseudo_curvs_i_info = inp_i_info return dests
def __init__(self, unmerged_intensities, batches): self.unmerged_intensities = unmerged_intensities self.batches = batches self.minb = flex.min(self.batches.data()) self.maxb = flex.max(self.batches.data()) n_scale_factors = self.maxb-self.minb + 1 self.x = flex.double(n_scale_factors, 1) scitbx.lbfgs.run(target_evaluator=self)
def exercise_3(grid_step,
radius,
shell,
a,
b,
d_min,
site_cart,
buffer_layer):
xray_structure = xray_structure_of_one_atom(site_cart = site_cart,
buffer_layer = buffer_layer,
a = a,
b = b)
miller_set = miller.build_set(
crystal_symmetry = xray_structure.crystal_symmetry(),
anomalous_flag = False,
d_min = d_min,
d_max = None)
f_calc = miller_set.structure_factors_from_scatterers(
xray_structure = xray_structure,
algorithm = "direct",
cos_sin_table = False,
exp_table_one_over_step_size = False).f_calc()
fft_map = f_calc.fft_map(grid_step = grid_step, symmetry_flags = None)
fft_map = fft_map.apply_volume_scaling()
site_frac = xray_structure.sites_frac()
r = abs(abs(flex.max(fft_map.real_map_unpadded()))-\
abs(fft_map.real_map_unpadded().value_at_closest_grid_point(site_frac[0])))/\
abs(flex.max(fft_map.real_map_unpadded())) * 100.0
assert approx_equal(r, 0.0)
around_atom_obj_ = mmtbx.real_space.around_atom(
unit_cell = xray_structure.unit_cell(),
map_data = fft_map.real_map_unpadded(),
radius = radius,
shell = shell,
site_frac = list(xray_structure.sites_frac())[0])
data = around_atom_obj_.data()
dist = around_atom_obj_.distances()
approx_obj_ = maptbx.one_gaussian_peak_approximation(
data_at_grid_points = data,
distances = dist,
use_weights = False,
optimize_cutoff_radius = True)
assert approx_equal(approx_obj_.a_reciprocal_space(), 6.0, 0.1)
assert approx_equal(approx_obj_.b_reciprocal_space(), 3.0, 0.01)
assert approx_obj_.gof() < 0.6
assert approx_obj_.cutoff_radius() < radius
def test_4():
symmetry = crystal.symmetry(unit_cell = (15.67, 25.37, 35.68, 90, 90, 90),
space_group_symbol = "P 21 21 21")
structure = xray.structure(crystal_symmetry = symmetry)
ma = structure.structure_factors(d_min = 1.5,
anomalous_flag = False).f_calc()
mi = ma.indices()
# ================================================================= TEST-1
alpha = flex.double(mi.size())
beta = flex.double(mi.size())
d_obs = flex.double(mi.size())
d_calc = flex.double(mi.size())
# define test set reflections
flags=flex.int(beta.size(), 0)
k=0
for i in xrange(flags.size()):
k=k+1
if (k !=10):
flags[i]=0
else:
k=0
flags[i]=1
for i in range(1,mi.size()+1):
d_obs [i-1] = i*1.5
d_calc[i-1] = i*1.0
beta [i-1] = i*500.0
alpha [i-1] = float(i) / float(i + 1)
obj = max_lik.fom_and_phase_error(
f_obs = d_obs,
f_model = d_calc,
alpha = alpha,
beta = beta,
epsilons = ma.epsilons().data().as_double(),
centric_flags = ma.centric_flags().data())
per = obj.phase_error()
fom = obj.fom()
assert approx_equal(flex.max(per) , 89.9325000127 , 1.e-4)
assert approx_equal(flex.min(per) , 5.37565067746e-05 , 1.e-4)
assert approx_equal(flex.mean(per), 20.7942460698 , 1.e-4)
assert approx_equal(flex.max(fom) , 0.999999402705 , 1.e-4)
assert approx_equal(flex.min(fom) , 0.000749999859375 , 1.e-4)
assert approx_equal(flex.mean(fom), 0.858269037582 , 1.e-4)
def reset_max_error(itvc_entry, fit):
sel = international_tables_stols <= fit.stol + 1.e-6
gaussian_fit = scitbx.math.gaussian.fit(
international_tables_stols.select(sel),
itvc_entry.table_y.select(sel),
itvc_entry.table_sigmas.select(sel),
fit)
fit.max_error = flex.max(gaussian_fit.significant_relative_errors())
def anisotropic_correction(cache_0,
p_scale,
u_star,
b_add=None,
must_be_greater_than=0.):
## Make sure that u_star is not rwgk scaled, i.e. like you get it from
## the ml_absolute_scale_aniso routine (!which is !!NOT!! scaled!)
work_array = None
try:
work_array = cache_0.input.select( cache_0.input.data() > must_be_greater_than)
except KeyboardInterrupt: raise
except Exception: pass
if work_array is None:
work_array = cache_0.select( cache_0.data() > must_be_greater_than)
change_back_to_intensity=False
if work_array.is_xray_intensity_array():
work_array = work_array.f_sq_as_f()
change_back_to_intensity=True
assert not work_array.is_xray_intensity_array()
if b_add is not None:
u_star_add = adptbx.b_iso_as_u_star( work_array.unit_cell(),
b_add )
u_star = u_star+u_star_add
corrected_amplitudes = scaling.ml_normalise_aniso( work_array.indices(),
work_array.data(),
p_scale,
work_array.unit_cell(),
u_star )
if work_array.sigmas() is not None:
corrected_sigmas = scaling.ml_normalise_aniso( work_array.indices(),
work_array.sigmas(),
p_scale,
work_array.unit_cell(),
u_star )
else:
corrected_sigmas = None
work_array = work_array.customized_copy(
data = corrected_amplitudes,
sigmas = corrected_sigmas ).set_observation_type(work_array)
if change_back_to_intensity:
# XXX check for floating-point overflows (which trigger the Boost trap
# and crash the interpreter). The only known case is 2q8o:IOBS2,SIGIOBS2
# which is missing nearly all acentric hkls but it clearly points to a bug
# in this routine when dealing with pathological data.
f_max = flex.max(work_array.data())
if (not f_max < math.sqrt(sys.float_info.max)) :
raise OverflowError("Amplitudes will exceed floating point limit if "+
"converted to intensities (max F = %e)." % f_max)
work_array = work_array.f_as_f_sq()
return work_array
def accelerations(self):
self.stereochemistry_residuals = self.restraints_manager.energies_sites(
sites_cart=self.structure.sites_cart(), compute_gradients=True
)
# Harmonic restraints
if self.er_data is not None:
if self.er_data.er_harmonic_restraints_info is not None:
harmonic_grads = self.restraints_manager.geometry.ta_harmonic_restraints(
sites_cart=self.structure.sites_cart(),
ta_harmonic_restraint_info=self.er_data.er_harmonic_restraints_info,
weight=self.er_data.er_harmonic_restraints_weight,
slack=self.er_data.er_harmonic_restraints_slack,
)
assert self.stereochemistry_residuals.gradients.size() == harmonic_grads.size()
self.stereochemistry_residuals.gradients += harmonic_grads
result = self.stereochemistry_residuals.gradients
d_max = None
if self.xray_structure_last_updated is not None and self.shift_update > 0:
array_of_distances_between_each_atom = flex.sqrt(
self.structure.difference_vectors_cart(self.xray_structure_last_updated).dot()
)
d_max = flex.max(array_of_distances_between_each_atom)
if self.fmodel is not None:
if d_max is not None:
if d_max > self.shift_update:
self.xray_structure_last_updated = self.structure.deep_copy_scatterers()
self.xray_gradient = self.xray_grads()
else:
self.xray_gradient = self.xray_grads()
result = (
self.xray_gradient * self.xray_target_weight
+ self.stereochemistry_residuals.gradients * self.chem_target_weight
)
factor = 1.0
if self.chem_target_weight is not None:
factor *= self.chem_target_weight
if self.stereochemistry_residuals.normalization_factor is not None:
factor *= self.stereochemistry_residuals.normalization_factor
if factor != 1.0:
result *= 1.0 / factor
# Store RMS non-solvent atom gradients for Xray and Geo
if self.er_data is not None:
self.wc = self.chem_target_weight / factor
self.wx = self.xray_target_weight / factor
self.gg = self.stereochemistry_residuals.gradients * self.wc
self.xg = self.xray_gradient * self.wx
gg_pro = self.gg.select(~self.er_data.solvent_sel)
xg_pro = self.xg.select(~self.er_data.solvent_sel)
self.er_data.geo_grad_rms += (flex.mean_sq(gg_pro.as_double()) ** 0.5) / self.n_steps
self.er_data.xray_grad_rms += (flex.mean_sq(xg_pro.as_double()) ** 0.5) / self.n_steps
return result
def exercise_2():
symmetry = crystal.symmetry(
unit_cell=(5.67, 10.37, 10.37, 90, 135.49, 90),
space_group_symbol="C2")
structure = xray.structure(crystal_symmetry=symmetry)
atmrad = flex.double()
xyzf = flex.vec3_double()
for k in xrange(100):
scatterer = xray.scatterer(
site = ((1.+k*abs(math.sin(k)))/1000.0,
(1.+k*abs(math.cos(k)))/1000.0,
(1.+ k)/1000.0),
scattering_type = "C")
structure.add_scatterer(scatterer)
atmrad.append(van_der_waals_radii.vdw.table[scatterer.element_symbol()])
xyzf.append(scatterer.site)
miller_set = miller.build_set(
crystal_symmetry=structure,
d_min=1.0,
anomalous_flag=False)
step = 0.5
crystal_gridding = maptbx.crystal_gridding(
unit_cell=structure.unit_cell(),
step=step)
nxyz = crystal_gridding.n_real()
shrink_truncation_radius = 1.0
solvent_radius = 1.0
m1 = around_atoms(
structure.unit_cell(),
structure.space_group().order_z(),
structure.sites_frac(),
atmrad,
nxyz,
solvent_radius,
shrink_truncation_radius)
assert m1.solvent_radius == 1
assert m1.shrink_truncation_radius == 1
assert flex.max(m1.data) == 1
assert flex.min(m1.data) == 0
assert m1.data.size() == m1.data.count(1) + m1.data.count(0)
m2 = mmtbx.masks.bulk_solvent(
xray_structure=structure,
gridding_n_real=nxyz,
ignore_zero_occupancy_atoms = False,
solvent_radius=solvent_radius,
shrink_truncation_radius=shrink_truncation_radius)
assert m2.data.all_eq(m1.data)
m3 = mmtbx.masks.bulk_solvent(
xray_structure=structure,
grid_step=step,
ignore_zero_occupancy_atoms = False,
solvent_radius=solvent_radius,
shrink_truncation_radius=shrink_truncation_radius)
assert m3.data.all_eq(m1.data)
f_mask2 = m2.structure_factors(miller_set=miller_set)
f_mask3 = m3.structure_factors(miller_set=miller_set)
assert approx_equal(f_mask2.data(), f_mask3.data())
assert approx_equal(flex.sum(flex.abs(f_mask3.data())), 1095.17999134)
def fit_side_chain(self, clusters):
rotamer_iterator = \
mmtbx.refinement.real_space.fit_residue.get_rotamer_iterator(
mon_lib_srv = self.mon_lib_srv,
residue = self.residue)
if(rotamer_iterator is None): return
selection = flex.size_t(flatten(clusters[0].vector))
if(self.target_map is not None):
start_target_value = self.get_target_value(
sites_cart = self.residue.atoms().extract_xyz(),
selection = selection)
sites_cart_start = self.residue.atoms().extract_xyz()
sites_cart_first_rotamer = list(rotamer_iterator)[0][1]
self.residue.atoms().set_xyz(sites_cart_first_rotamer)
axes = []
atr = []
for i, angle in enumerate(self.chi_angles[0]):
cl = clusters[i]
axes.append(flex.size_t(cl.axis))
atr.append(flex.size_t(cl.atoms_to_rotate))
if(self.target_map is not None):
ro = ext.fit(
target_value = start_target_value,
axes = axes,
rotatable_points_indices = atr,
angles_array = self.chi_angles,
density_map = self.target_map,
all_points = self.residue.atoms().extract_xyz(),
unit_cell = self.unit_cell,
selection = selection,
sin_table = self.sin_cos_table.sin_table,
cos_table = self.sin_cos_table.cos_table,
step = self.sin_cos_table.step,
n = self.sin_cos_table.n)
else:
ro = ext.fit(
sites_cart_start = sites_cart_start.deep_copy(),
axes = axes,
rotatable_points_indices = atr,
angles_array = self.chi_angles,
all_points = self.residue.atoms().extract_xyz(),
sin_table = self.sin_cos_table.sin_table,
cos_table = self.sin_cos_table.cos_table,
step = self.sin_cos_table.step,
n = self.sin_cos_table.n)
sites_cart_result = ro.result()
if(sites_cart_result.size()>0):
dist = None
if(self.accept_only_if_max_shift_is_smaller_than is not None):
dist = flex.max(flex.sqrt((sites_cart_start - sites_cart_result).dot()))
if(dist is None):
self.residue.atoms().set_xyz(sites_cart_result)
else:
if(dist is not None and
dist < self.accept_only_if_max_shift_is_smaller_than):
self.residue.atoms().set_xyz(sites_cart_result)
else:
self.residue.atoms().set_xyz(sites_cart_start)
def test_2():
n_sites = 1000
d_min = 2.0
volume_per_atom = 50
fraction_missing = (0.0, )
scale = 5.0
# create dummy model
space_group_info = sgtbx.space_group_info("P212121")
structure = random_structure.xray_structure(
space_group_info=space_group_info,
elements=["N"] * (n_sites),
volume_per_atom=volume_per_atom,
random_u_iso=False)
structure.scattering_type_registry(table="wk1995")
f_calc = structure.structure_factors(d_min=d_min,
anomalous_flag=False,
algorithm="direct").f_calc()
f_obs = abs(f_calc)
for fm in fraction_missing:
# partial model
n_keep = int(round(structure.scatterers().size() * (1 - fm)))
partial_structure = xray.structure(special_position_settings=structure)
partial_structure.add_scatterers(structure.scatterers()[:n_keep])
# fcalc (partial model), fobs (fcalc full model)
f_calc_partial = partial_structure.structure_factors(
d_min=d_min, anomalous_flag=False, algorithm="direct").f_calc()
f_calc = abs(f_calc_partial)
# define test set reflections
flags = flex.bool(f_calc_partial.indices().size(), False)
k = 0
for i in xrange(f_calc_partial.indices().size()):
k = k + 1
if (k != 10):
flags[i] = False
else:
k = 0
flags[i] = True
# *********************************************************TEST = 1
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=f_obs,
f_calc=f_calc,
free_reflections_per_bin=f_obs.data().size(),
flags=flags,
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=f_obs,
f_calc=f_calc,
free_reflections_per_bin=f_obs.data().size(),
flags=flags,
interpolation=True,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
# *********************************************************TEST = 2
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flags,
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flags,
interpolation=True,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
# *********************************************************TEST = 3
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flex.bool(f_obs.data().size(), True),
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flex.bool(f_obs.data().size(), True),
interpolation=True,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
# *********************************************************TEST = 4
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flex.bool(f_obs.data().size(), False),
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flex.bool(f_obs.data().size(), False),
interpolation=True,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
def fit_side_chain(self, clusters):
rotamer_iterator = \
mmtbx.refinement.real_space.fit_residue.get_rotamer_iterator(
mon_lib_srv = self.mon_lib_srv,
residue = self.residue)
if(rotamer_iterator is None): return
selection_clash = self.co.clash_eval_selection
selection_rsr = self.co.rsr_eval_selection
if(self.target_map is not None):
start_target_value = self.get_target_value(
sites_cart = self.residue.atoms().extract_xyz(),
selection = selection_rsr)
sites_cart_start = self.residue.atoms().extract_xyz()
sites_cart_first_rotamer = list(rotamer_iterator)[0][1]
# From this point on the coordinates in residue are to initial rotamer!
self.residue.atoms().set_xyz(sites_cart_first_rotamer)
axes = []
atr = []
for i, angle in enumerate(self.chi_angles[0]):
cl = clusters[i]
axes.append(flex.size_t(cl.axis))
atr.append(flex.size_t(cl.atoms_to_rotate))
#
if(self.target_map is not None and self.xyzrad_bumpers is not None):
# Get reference map values
ref_map_vals = flex.double()
for a in self.residue.atoms():
key = "%s_%s_%s"%(
a.parent().parent().parent().id, a.parent().resname,
a.name.strip())
ref_map_vals.append(self.cmv[key])
# Get radii
radii = mmtbx.refinement.real_space.get_radii(
residue = self.residue, vdw_radii = self.vdw_radii)
# Exclude rotatable H from clash calculation
tmp = flex.size_t()
for i in selection_clash:
if(self.rotatable_hd[self.residue.atoms()[i].i_seq]): continue
tmp.append(i)
selection_clash = tmp[:]
# Ad hoc: S or SE have larger peaks!
if(self.residue.resname in ["MET","MSE"]): scale=100
else: scale=3
moving = ext.moving(
sites_cart = self.residue.atoms().extract_xyz(),
sites_cart_start = sites_cart_start,
radii = radii,
weights = self.weights,
bonded_pairs = self.pairs,
ref_map_max = ref_map_vals * scale,
ref_map_min = ref_map_vals / 10)
#
ro = ext.fit(
fixed = self.xyzrad_bumpers,
axes = axes,
rotatable_points_indices = atr,
angles_array = self.chi_angles,
density_map = self.target_map,
moving = moving,
unit_cell = self.unit_cell,
selection_clash = selection_clash,
selection_rsr = selection_rsr, # select atoms to compute map target
sin_table = self.sin_cos_table.sin_table,
cos_table = self.sin_cos_table.cos_table,
step = self.sin_cos_table.step,
n = self.sin_cos_table.n)
elif(self.target_map is not None and self.xyzrad_bumpers is None):
ro = ext.fit(
target_value = start_target_value,
axes = axes,
rotatable_points_indices = atr,
angles_array = self.chi_angles,
density_map = self.target_map,
all_points = self.residue.atoms().extract_xyz(),
unit_cell = self.unit_cell,
selection = selection_rsr,
sin_table = self.sin_cos_table.sin_table,
cos_table = self.sin_cos_table.cos_table,
step = self.sin_cos_table.step,
n = self.sin_cos_table.n)
else:
ro = ext.fit(
sites_cart_start = sites_cart_start.deep_copy(),
axes = axes,
rotatable_points_indices = atr,
angles_array = self.chi_angles,
all_points = self.residue.atoms().extract_xyz(),
sin_table = self.sin_cos_table.sin_table,
cos_table = self.sin_cos_table.cos_table,
step = self.sin_cos_table.step,
n = self.sin_cos_table.n)
sites_cart_result = ro.result()
if(sites_cart_result.size()>0):
dist = None
if(self.accept_only_if_max_shift_is_smaller_than is not None):
dist = flex.max(flex.sqrt((sites_cart_start - sites_cart_result).dot()))
if(dist is None):
self.residue.atoms().set_xyz(sites_cart_result)
else:
if(dist is not None and
dist < self.accept_only_if_max_shift_is_smaller_than):
self.residue.atoms().set_xyz(sites_cart_result)
else:
self.residue.atoms().set_xyz(sites_cart_start)
else:
self.residue.atoms().set_xyz(sites_cart_start)
if(self.m): self.m.add(residue = self.residue, state = "fitting")
# # tune up
if(self.target_map is not None):
tune_up(
target_map = self.target_map,
residue = self.residue,
mon_lib_srv = self.mon_lib_srv,
rotamer_manager = self.rotamer_manager.rotamer_evaluator,
unit_cell = self.unit_cell,
monitor = self.m,
torsion_search_start = -30,
torsion_search_stop = 30,
torsion_search_step = 1)
def _map_mmm_str(self):
return " ".join([("%8.3f" % m).strip() for m in [
flex.min(self.map_data),
flex.max(self.map_data),
flex.mean(self.map_data)
]])
def exercise_3():
#test torsion restraints
for use_reference in ['True', 'False', 'top_out', 'None']:
pdb_inp = iotbx.pdb.input(lines=flex.std_string(
pdb_str_2.splitlines()),
source_info=None)
model = manager(model_input=pdb_inp, log=null_out())
grm = model.get_restraints_manager().geometry
xrs2 = model.get_xray_structure()
awl2 = model.get_hierarchy().atoms_with_labels()
pdb2 = model.get_hierarchy()
pdb_inp3 = iotbx.pdb.input(source_info=None, lines=pdb_str_3)
xrs3 = pdb_inp3.xray_structure_simple()
ph3 = pdb_inp3.construct_hierarchy()
ph3.atoms().reset_i_seq()
awl3 = ph3.atoms_with_labels()
sites_cart_reference = flex.vec3_double()
selection = flex.size_t()
min_selection = flex.size_t()
reference_names = [
"N", "CA", "CB", "CG", "CD", "NE", "CZ", "NH1", "NH2"
]
minimize_names = ["CG", "CD", "NE", "CZ", "NH1", "NH2"]
for a2, a3 in zip(tuple(awl2), tuple(awl3)):
assert a2.resname == a3.resname
assert a2.name == a3.name
assert a2.i_seq == a3.i_seq
if (a2.resname == "ARG" and a2.name.strip() in reference_names):
selection.append(a2.i_seq)
sites_cart_reference.append(a3.xyz)
if a2.name.strip() in minimize_names:
min_selection.append(a2.i_seq)
assert selection.size() == len(reference_names)
selection_bool = flex.bool(xrs2.scatterers().size(), min_selection)
if (use_reference == 'True'):
grm.add_chi_torsion_restraints_in_place(
pdb_hierarchy=pdb2,
sites_cart=sites_cart_reference,
selection=selection,
sigma=2.5)
elif (use_reference == 'top_out'):
grm.add_chi_torsion_restraints_in_place(
pdb_hierarchy=pdb2,
sites_cart=sites_cart_reference,
selection=selection,
sigma=2.5,
limit=180.0,
top_out_potential=True)
elif (use_reference == 'None'):
grm.add_chi_torsion_restraints_in_place(
pdb_hierarchy=pdb2,
sites_cart=sites_cart_reference,
selection=selection,
sigma=2.5)
grm.remove_chi_torsion_restraints_in_place(selection=selection)
d1 = flex.mean(
flex.sqrt((xrs2.sites_cart().select(min_selection) -
xrs3.sites_cart().select(min_selection)).dot()))
print "distance start (use_reference: %s): %6.4f" % (
str(use_reference), d1)
assert d1 > 4.0
assert approx_equal(
flex.max(
flex.sqrt((xrs2.sites_cart().select(~selection_bool) -
xrs3.sites_cart().select(~selection_bool)).dot())),
0)
from cctbx import geometry_restraints
import mmtbx.refinement.geometry_minimization
import scitbx.lbfgs
grf = geometry_restraints.flags.flags(default=True)
grf.nonbonded = False
sites_cart = xrs2.sites_cart()
minimized = mmtbx.refinement.geometry_minimization.lbfgs(
sites_cart=sites_cart,
correct_special_position_tolerance=1.0,
geometry_restraints_manager=grm,
sites_cart_selection=flex.bool(sites_cart.size(), min_selection),
geometry_restraints_flags=grf,
lbfgs_termination_params=scitbx.lbfgs.termination_parameters(
max_iterations=5000))
xrs2.set_sites_cart(sites_cart=sites_cart)
d2 = flex.mean(
flex.sqrt((xrs2.sites_cart().select(min_selection) -
xrs3.sites_cart().select(min_selection)).dot()))
print "distance final (use_reference: %s): %6.4f" % (
str(use_reference), d2)
if (use_reference in ['True', 'top_out']): assert d2 < 0.02, d2
else: assert d2 > 4.0, d2
assert approx_equal(
flex.max(
flex.sqrt((xrs2.sites_cart().select(~selection_bool) -
xrs3.sites_cart().select(~selection_bool)).dot())),
0)
#test torsion manipulation
grm.remove_chi_torsion_restraints_in_place()
grm.remove_chi_torsion_restraints_in_place()
sites_cart_reference = []
selections_reference = []
for model in pdb2.models():
for chain in model.chains():
for residue in chain.residues():
sites_cart_reference.append(residue.atoms().extract_xyz())
selections_reference.append(residue.atoms().extract_i_seq())
#one residue at a time (effectively chi angles only)
for sites_cart, selection in zip(sites_cart_reference,
selections_reference):
grm.add_chi_torsion_restraints_in_place(pdb_hierarchy=pdb2,
sites_cart=sites_cart,
selection=selection)
assert grm.get_n_chi_torsion_proixes() == 6
grm.remove_chi_torsion_restraints_in_place()
#all sites at once, chi angles only
sites_cart = xrs2.sites_cart()
grm.add_chi_torsion_restraints_in_place(pdb_hierarchy=pdb2,
sites_cart=sites_cart,
selection=None,
chi_angles_only=True)
assert grm.get_n_chi_torsion_proixes() == 6
#all sites at once, all torsions
grm.add_chi_torsion_restraints_in_place(pdb_hierarchy=pdb2,
sites_cart=sites_cart,
selection=None,
chi_angles_only=False)
# grm.get_chi_torsion_proxies().show_sorted(
# by_value='residual',
# sites_cart=sites_cart,
# site_labels=[atom.id_str() for atom in pdb2.atoms()])
assert grm.get_n_chi_torsion_proixes(
) == 12, grm.get_n_chi_torsion_proixes()
def __init__(self,datadir,work_params,plot=False,esd_plot=False,half_data_flag=0):
casetag = work_params.output.prefix
# read the ground truth values back in
from six.moves import cPickle as pickle
# it is assumed (for now) that the reference millers contain a complete asymmetric unit
# of indices, within the (d_max,d_min) region of interest and possibly outside the region.
reference_millers = pickle.load(open(os.path.join(datadir,casetag+"_miller.pickle"),"rb"))
experiment_manager = read_experiments(work_params)
obs = pickle.load(open(os.path.join(datadir,casetag+"_observation.pickle"),"rb"))
print "Read in %d observations"%(len(obs["observed_intensity"]))
reference_millers.show_summary(prefix="Miller index file ")
print len(obs["frame_lookup"]),len(obs["observed_intensity"]), flex.max(obs['miller_lookup']),flex.max(obs['frame_lookup'])
max_frameno = flex.max(obs["frame_lookup"])
from iotbx import mtz
mtz_object = mtz.object(file_name=work_params.scaling.mtz_file)
#for array in mtz_object.as_miller_arrays():
# this_label = array.info().label_string()
# print this_label, array.observation_type()
I_sim = mtz_object.as_miller_arrays()[0].as_intensity_array()
I_sim.show_summary()
MODEL_REINDEX_OP = work_params.model_reindex_op
I_sim = I_sim.change_basis(MODEL_REINDEX_OP).map_to_asu()
#match up isomorphous (the simulated fake F's) with experimental unique set
matches = miller.match_multi_indices(
miller_indices_unique=reference_millers.indices(),
miller_indices=I_sim.indices())
print "original unique",len(reference_millers.indices())
print "isomorphous set",len(I_sim.indices())
print "pairs",len(matches.pairs())
iso_data = flex.double(len(reference_millers.indices()))
for pair in matches.pairs():
iso_data[pair[0]] = I_sim.data()[pair[1]]
reference_data = miller.array(miller_set = reference_millers,
data = iso_data)
reference_data.set_observation_type_xray_intensity()
FOBS = prepare_observations_for_scaling(work_params,obs=obs,
reference_intensities=reference_data,
files = experiment_manager.get_files(),
half_data_flag=half_data_flag)
I,I_visited,G,G_visited = I_and_G_base_estimate(FOBS,params=work_params)
print "I length",len(I), "G length",len(G), "(Reference set; entire asymmetric unit)"
assert len(reference_data.data()) == len(I)
#presumably these assertions fail when half data are taken for CC1/2 or d_min is cut
model_I = reference_data.data()[0:len(I)]
T = Timer("%d frames"%(len(G), ))
mapper = mapper_factory(xscale6e)
minimizer = mapper(I,G,I_visited,G_visited,FOBS,params=work_params,
experiments=experiment_manager.get_experiments())
del T
minimizer.show_summary()
Fit = minimizer.e_unpack()
Gstats=flex.mean_and_variance(Fit["G"].select(G_visited==1))
print "G mean and standard deviation:",Gstats.mean(),Gstats.unweighted_sample_standard_deviation()
if "Bfactor" in work_params.levmar.parameter_flags:
Bstats=flex.mean_and_variance(Fit["B"].select(G_visited==1))
print "B mean and standard deviation:",Bstats.mean(),Bstats.unweighted_sample_standard_deviation()
show_correlation(Fit["I"],model_I,I_visited,"Correlation of I:")
Fit_stddev = minimizer.e_unpack_stddev()
# XXX FIXME known bug: the length of Fit["G"] could be smaller than the length of experiment_manager.get_files()
# Not sure if this has any operational drawbacks. It's a result of half-dataset selection.
if plot:
plot_it(Fit["I"], model_I, mode="I")
if "Rxy" in work_params.levmar.parameter_flags:
show_histogram(Fit["Ax"],"Histogram of x rotation (degrees)")
show_histogram(Fit["Ay"],"Histogram of y rotation (degrees)")
print
if esd_plot:
minimizer.esd_plot()
from cctbx.examples.merging.show_results import show_overall_observations
table1,self.n_bins,self.d_min = show_overall_observations(
Fit["I"],Fit_stddev["I"],I_visited,
reference_data,FOBS,title="Statistics for all reflections",
work_params = work_params)
self.FSIM=FOBS
self.ordered_intensities=reference_data
self.reference_millers=reference_millers
self.Fit_I=Fit["I"]
self.Fit_I_stddev=Fit_stddev["I"]
self.I_visited=I_visited
self.Fit = Fit
self.experiments = experiment_manager
def calculate_fsc(
si=None,
f_array=None, # just used for binner
map_coeffs=None,
model_map_coeffs=None,
external_map_coeffs=None,
first_half_map_coeffs=None,
second_half_map_coeffs=None,
resolution=None,
fraction_complete=None,
min_fraction_complete=None,
is_model_based=None,
cc_cut=None,
scale_using_last=None,
max_cc_for_rescale=None,
pseudo_likelihood=False,
skip_scale_factor=False,
equalize_power=False,
verbose=None,
out=sys.stdout):
# calculate anticipated fall-off of model data with resolution
if si.rmsd is None and is_model_based:
si.rmsd = resolution * si.rmsd_resolution_factor
print("Setting rmsd to %5.1f A based on resolution of %5.1f A" %
(si.rmsd, resolution),
file=out)
# get f and model_f vs resolution and FSC vs resolution and apply
# If external_map_coeffs then simply scale f to external_map_coeffs
# scale to f_array and return sharpened map
dsd = f_array.d_spacings().data()
from cctbx.maptbx.segment_and_split_map import map_coeffs_to_fp
if is_model_based:
mc1 = map_coeffs
mc2 = model_map_coeffs
fo_map = map_coeffs # scale map_coeffs to model_map_coeffs*FSC
fc_map = model_map_coeffs
b_eff = get_b_eff(si=si, out=out)
elif external_map_coeffs:
mc1 = map_coeffs
mc2 = external_map_coeffs
fo_map = map_coeffs # scale map_coeffs to external_map_coeffs
fc_map = external_map_coeffs
b_eff = None
else: # half_dataset
mc1 = first_half_map_coeffs
mc2 = second_half_map_coeffs
fo_map = map_coeffs # scale map_coeffs to cc*
fc_map = model_map_coeffs
b_eff = None
ratio_list = flex.double()
target_sthol2 = flex.double()
cc_list = flex.double()
sthol_list = flex.double()
d_min_list = flex.double()
rms_fo_list = flex.double()
rms_fc_list = flex.double()
max_possible_cc = None
n_list = flex.double()
for i_bin in f_array.binner().range_used():
sel = f_array.binner().selection(i_bin)
d = dsd.select(sel)
if d.size() < 1:
raise Sorry("Please reduce number of bins (no data in bin " +
"%s) from current value of %s" %
(i_bin, f_array.binner().n_bins_used()))
d_min = flex.min(d)
d_max = flex.max(d)
d_avg = flex.mean(d)
n = d.size()
m1 = mc1.select(sel)
m2 = mc2.select(sel)
cc = m1.map_correlation(other=m2)
if external_map_coeffs:
cc = 1.
if fo_map:
fo = fo_map.select(sel)
f_array_fo = map_coeffs_to_fp(fo)
rms_fo = f_array_fo.data().norm()
else:
rms_fo = 1.
if fc_map:
fc = fc_map.select(sel)
f_array_fc = map_coeffs_to_fp(fc)
rms_fc = f_array_fc.data().norm()
else:
rms_fc = 1.
sthol2 = 0.25 / d_avg**2
ratio_list.append(max(1.e-10, rms_fc) / max(1.e-10, rms_fo))
target_sthol2.append(sthol2)
if cc is None: cc = 0.
cc_list.append(cc)
sthol_list.append(1 / d_avg)
d_min_list.append(d_min)
rms_fo_list.append(rms_fo)
rms_fc_list.append(rms_fc)
n_list.append(m1.size())
if b_eff is not None:
max_cc_estimate = cc * math.exp(min(20., sthol2 * b_eff))
else:
max_cc_estimate = cc
max_cc_estimate = max(0., min(1., max_cc_estimate))
if max_possible_cc is None or (max_cc_estimate > 0
and max_cc_estimate > max_possible_cc):
max_possible_cc = max_cc_estimate
if verbose:
print("d_min: %5.1f FC: %7.1f FOBS: %7.1f CC: %5.2f" %
(d_avg, rms_fc, rms_fo, cc),
file=out)
if scale_using_last: # rescale to give final value average==0
cc_list, baseline = rescale_cc_list(
cc_list=cc_list,
scale_using_last=scale_using_last,
max_cc_for_rescale=max_cc_for_rescale)
if baseline is None: # don't use it
scale_using_last = None
original_cc_list = deepcopy(cc_list)
if is_model_based: # jut smooth cc if nec
fitted_cc = get_fitted_cc(cc_list=cc_list,
sthol_list=sthol_list,
cc_cut=cc_cut,
scale_using_last=scale_using_last)
cc_list = fitted_cc
text = " FIT "
else:
cc_list = estimate_cc_star(cc_list=cc_list,
sthol_list=sthol_list,
cc_cut=cc_cut,
scale_using_last=scale_using_last)
text = " CC* "
if not max_possible_cc:
max_possible_cc = 0.01
if si.target_scale_factors: # not using these
max_possible_cc = 1.
fraction_complete = 1.
elif (not is_model_based):
max_possible_cc = 1.
fraction_complete = 1.
else:
# Define overall CC based on model completeness (CC=sqrt(fraction_complete))
if fraction_complete is None:
fraction_complete = max_possible_cc**2
print(
"Estimated fraction complete is %5.2f based on low_res CC of %5.2f"
% (fraction_complete, max_possible_cc),
file=out)
else:
print("Using fraction complete value of %5.2f " %
(fraction_complete),
file=out)
max_possible_cc = fraction_complete**0.5
target_scale_factors = flex.double()
sum_w = 0.
sum_w_scale = 0.
for i_bin in f_array.binner().range_used():
index = i_bin - 1
ratio = ratio_list[index]
cc = cc_list[index]
sthol2 = target_sthol2[index]
d_min = d_min_list[index]
corrected_cc = max(0.00001, min(1., cc / max_possible_cc))
if (not is_model_based): # cc is already cc*
scale_on_fo = ratio * corrected_cc
elif b_eff is not None:
if pseudo_likelihood:
scale_on_fo = (cc / max(0.001, 1 - cc**2))
else: # usual
scale_on_fo = ratio * min(
1.,
max(0.00001, corrected_cc) *
math.exp(min(20., sthol2 * b_eff)))
else:
scale_on_fo = ratio * min(1., max(0.00001, corrected_cc))
w = n_list[index] * (rms_fo_list[index])**2
sum_w += w
sum_w_scale += w * scale_on_fo**2
target_scale_factors.append(scale_on_fo)
if not pseudo_likelihood and not skip_scale_factor: # normalize
avg_scale_on_fo = (sum_w_scale / sum_w)**0.5
if equalize_power:
scale_factor = 1 / avg_scale_on_fo
else: # usual
scale_factor = 1. / target_scale_factors.min_max_mean().max
target_scale_factors=\
target_scale_factors*scale_factor
print("Scale factor A: %.5f" % (scale_factor), file=out)
if fraction_complete < min_fraction_complete:
print("\nFraction complete (%5.2f) is less than minimum (%5.2f)..." %
(fraction_complete, min_fraction_complete) +
"\nSkipping scaling",
file=out)
target_scale_factors = flex.double(target_scale_factors.size() *
(1.0, ))
print("\nAverage CC: %.3f" % (cc_list.min_max_mean().mean), file=out)
print("\nScale factors vs resolution:", file=out)
print("Note 1: CC* estimated from sqrt(2*CC/(1+CC))", file=out)
print("Note 2: CC estimated by fitting (smoothing) for values < %s" %
(cc_cut),
file=out)
print("Note 3: Scale = A CC* rmsFc/rmsFo (A is normalization)", file=out)
print(" d_min rmsFo rmsFc CC %s Scale" % (text),
file=out)
for sthol2, scale, rms_fo, cc, rms_fc, orig_cc in zip(
target_sthol2, target_scale_factors, rms_fo_list, cc_list,
rms_fc_list, original_cc_list):
print("%7.2f %9.1f %9.1f %7.3f %7.3f %5.2f" %
(0.5 / sthol2**0.5, rms_fo, rms_fc, orig_cc, cc, scale),
file=out)
si.target_scale_factors = target_scale_factors
si.target_sthol2 = target_sthol2
si.d_min_list = d_min_list
si.cc_list = cc_list
return si
def exercise_negative_parameters(verbose=0):
structure_default = xray.structure(
crystal_symmetry=crystal.symmetry(unit_cell=((10, 13, 17, 75, 80, 85)),
space_group_symbol="P 1"),
scatterers=flex.xray_scatterer(
[xray.scatterer(label="C", site=(0, 0, 0), u=0.25)]))
negative_gaussian = eltbx.xray_scattering.gaussian((1, 2), (2, 3), -4)
for i_trial in xrange(7):
structure = structure_default.deep_copy_scatterers()
scatterer = structure.scatterers()[0]
if (i_trial == 1):
scatterer.occupancy *= -1
elif (i_trial == 2):
structure.scattering_type_registry(
custom_dict={"C": negative_gaussian})
elif (i_trial == 3):
scatterer.u_iso *= -1
elif (i_trial == 4):
u_cart = adptbx.random_u_cart(u_scale=1, u_min=-1.1)
assert max(adptbx.eigenvalues(u_cart)) < 0
u_star = adptbx.u_cart_as_u_star(structure.unit_cell(), u_cart)
scatterer.u_star = u_star
scatterer.flags.set_use_u_aniso_only()
elif (i_trial == 5):
scatterer.fp = -10
elif (i_trial == 6):
scatterer.fp = -3
f_direct = structure.structure_factors(d_min=1,
algorithm="direct",
cos_sin_table=False).f_calc()
f_fft = structure.structure_factors(d_min=1,
algorithm="fft",
quality_factor=1.e8,
wing_cutoff=1.e-10).f_calc()
if (i_trial == 2):
assert negative_gaussian.at_d_star_sq(
f_fft.d_star_sq().data()).all_lt(0)
if (i_trial in [5, 6]):
f = structure.scattering_type_registry().gaussian_not_optional(
scattering_type="C").at_d_star_sq(f_fft.d_star_sq().data())
if (i_trial == 5):
assert flex.max(f) + scatterer.fp < 0
else:
assert flex.max(f) + scatterer.fp > 0
assert flex.min(f) + scatterer.fp < 0
cc = flex.linear_correlation(abs(f_direct).data(),
abs(f_fft).data()).coefficient()
if (cc < 0.999):
raise AssertionError("i_trial=%d, correlation=%.6g" %
(i_trial, cc))
elif (0 or verbose):
print "correlation=%.6g" % cc
#
# very simple test of gradient calculations with negative parameters
structure_factor_gradients = \
cctbx.xray.structure_factors.gradients(
miller_set=f_direct,
cos_sin_table=False)
target_functor = xray.target_functors.intensity_correlation(
f_obs=abs(f_direct))
target_result = target_functor(f_fft, True)
xray.set_scatterer_grad_flags(scatterers=structure.scatterers(),
site=True,
u_iso=True,
u_aniso=True,
occupancy=True,
fp=True,
fdp=True)
for algorithm in ["direct", "fft"]:
grads = structure_factor_gradients(
xray_structure=structure,
u_iso_refinable_params=None,
miller_set=f_direct,
d_target_d_f_calc=target_result.derivatives(),
n_parameters=structure.n_parameters(),
algorithm=algorithm).packed()
def run():
import sys
reflection_file_name = sys.argv[1]
import iotbx.cif
miller_arrays = iotbx.cif.reader(
file_path=reflection_file_name).as_miller_arrays()
for miller_array in miller_arrays:
s = str(miller_array.info())
if '_meas' in s:
if miller_array.is_xray_intensity_array():
break
elif miller_array.is_xray_amplitude_array():
break
if not ('_meas' in str(miller_array.info()) and
(miller_array.is_xray_amplitude_array()
or miller_array.is_xray_intensity_array())):
print("Sorry: CIF does not contain an appropriate miller array")
return
miller_array.show_comprehensive_summary()
print()
if (miller_array.is_xray_intensity_array()):
print("Converting intensities to amplitudes.")
miller_array = miller_array.as_amplitude_array()
print()
miller_array.setup_binner(auto_binning=True)
miller_array.binner().show_summary()
print()
all_e_values = miller_array.quasi_normalize_structure_factors().sort(
by_value="data")
large_e_values = all_e_values.select(all_e_values.data() > 1.2)
print("number of large_e_values:", large_e_values.size())
print()
from cctbx import dmtbx
triplets = dmtbx.triplet_generator(large_e_values)
from cctbx.array_family import flex
print("triplets per reflection: min,max,mean: %d, %d, %.2f" %
(flex.min(triplets.n_relations()), flex.max(triplets.n_relations()),
flex.mean(triplets.n_relations().as_double())))
print("total number of triplets:", flex.sum(triplets.n_relations()))
print()
input_phases = large_e_values \
.random_phases_compatible_with_phase_restrictions()
tangent_formula_phases = input_phases.data()
for i in range(10):
tangent_formula_phases = triplets.apply_tangent_formula(
amplitudes=large_e_values.data(),
phases_rad=tangent_formula_phases,
selection_fixed=None,
use_fixed_only=False,
reuse_results=True)
e_map_coeff = large_e_values.phase_transfer(
phase_source=tangent_formula_phases)
from cctbx import maptbx
e_map = e_map_coeff.fft_map(symmetry_flags=maptbx.use_space_group_symmetry)
e_map.apply_sigma_scaling()
e_map.statistics().show_summary(prefix="e_map ")
print()
peak_search = e_map.peak_search(parameters=maptbx.peak_search_parameters(
min_distance_sym_equiv=1.2))
peaks = peak_search.all(max_clusters=10)
print("e_map peak list")
print(" fractional coordinates peak height")
for site, height in zip(peaks.sites(), peaks.heights()):
print(" (%9.6f, %9.6f, %9.6f)" % site, "%10.3f" % height)
print()
if (len(sys.argv) > 2):
coordinate_file_name = sys.argv[2]
xray_structure = iotbx.cif.reader(
file_path=coordinate_file_name).build_crystal_structures(
data_block_name="I")
xray_structure.show_summary().show_scatterers()
print()
f_calc = abs(
miller_array.structure_factors_from_scatterers(
xray_structure=xray_structure, algorithm="direct").f_calc())
correlation = flex.linear_correlation(f_calc.data(),
miller_array.data())
assert correlation.is_well_defined()
print("correlation of f_obs and f_calc: %.4f" %
correlation.coefficient())
print()
reference_model = xray_structure.as_emma_model()
assert reference_model.unit_cell().is_similar_to(e_map.unit_cell())
assert reference_model.space_group() == e_map.space_group()
from cctbx import euclidean_model_matching as emma
peak_model = emma.model(special_position_settings=reference_model)
for i, site in enumerate(peaks.sites()):
peak_model.add_position(
emma.position(label="peak%02d" % i, site=site))
matches = emma.model_matches(
model1=reference_model,
model2=peak_model,
tolerance=1.,
models_are_diffraction_index_equivalent=True)
for match in matches.refined_matches:
match.show()
def exercise_1():
#------------------------------------------------------------------------TEST-1
if (1):
r = (1.67, 0.68, 0.0, 0.5, -0.31, -0.64, -1.63)
d = 2.0 * van_der_waals_radii.vdw.table["C"]
a = (d / 2.) * (4. / 3. * math.pi * 2.)**(1. / 3.)
for x in r:
for y in r:
for z in r:
structure, xyzf, atmrad = structure_init(
(x, y, z), "P1", (a, a, a, 90, 90, 90))
step = 0.1
crystal_gridding = maptbx.crystal_gridding(
unit_cell=structure.unit_cell(), step=step)
shrink_truncation_radius = 0.0
solvent_radius = 0.0
m = around_atoms(structure.unit_cell(),
structure.space_group().order_z(), xyzf,
atmrad, crystal_gridding.n_real(),
solvent_radius, shrink_truncation_radius)
assert m.solvent_radius == 0
assert m.shrink_truncation_radius == 0
assert approx_equal(1. / m.accessible_surface_fraction,
2.0, 1e-2)
assert approx_equal(1. / m.contact_surface_fraction, 2.0,
1e-2)
assert flex.max(m.data) == 1
assert flex.min(m.data) == 0
assert m.data.size() == m.data.count(1) + m.data.count(0)
#------------------------------------------------------------------------TEST-2
if (1):
r = (-2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0)
asf = []
csf = []
for x in r:
for y in r:
for z in r:
structure, xyzf, atmrad = structure_init(
(x, y, z), "C2", (5.0, 5.5, 7.0, 90, 80, 90))
step = 0.25
crystal_gridding = maptbx.crystal_gridding(
unit_cell=structure.unit_cell(), step=step)
shrink_truncation_radius = 0.0
solvent_radius = 0.0
m = around_atoms(structure.unit_cell(),
structure.space_group().order_z(), xyzf,
atmrad, crystal_gridding.n_real(),
solvent_radius, shrink_truncation_radius)
asf.append(m.accessible_surface_fraction)
csf.append(m.contact_surface_fraction)
assert m.accessible_surface_fraction == m.contact_surface_fraction
assert approx_equal(min(asf), max(asf))
#------------------------------------------------------------------------TEST-3
if (1):
r = (1.673, 0.685, -0.315, -0.649, -1.637)
asf = []
csf = []
for x in r:
for y in r:
for z in r:
structure, xyzf, atmrad = structure_init(
(x, y, z), "P1", (5.0, 5.5, 6.0, 70, 80, 100))
step = 0.1
crystal_gridding = maptbx.crystal_gridding(
unit_cell=structure.unit_cell(), step=step)
shrink_truncation_radius = 0.0
solvent_radius = 0.0
m = around_atoms(structure.unit_cell(),
structure.space_group().order_z(), xyzf,
atmrad, crystal_gridding.n_real(),
solvent_radius, shrink_truncation_radius)
asf.append(m.accessible_surface_fraction)
csf.append(m.contact_surface_fraction)
assert m.accessible_surface_fraction == m.contact_surface_fraction
assert approx_equal(min(asf), max(asf), 1e-3)
def fit_side_chain(self, clusters):
rotamer_iterator = \
mmtbx.refinement.real_space.fit_residue.get_rotamer_iterator(
mon_lib_srv = self.mon_lib_srv,
residue = self.residue)
if (rotamer_iterator is None): return
#selection_rsr = flex.size_t(flatten(clusters[0].vector))
selection_clash = self.co.clash_eval_selection
selection_rsr = self.co.rsr_eval_selection
if (self.target_map is not None):
start_target_value = self.get_target_value(
sites_cart=self.residue.atoms().extract_xyz(),
selection=selection_rsr)
sites_cart_start = self.residue.atoms().extract_xyz()
sites_cart_first_rotamer = list(rotamer_iterator)[0][1]
self.residue.atoms().set_xyz(sites_cart_first_rotamer)
axes = []
atr = []
for i, angle in enumerate(self.chi_angles[0]):
cl = clusters[i]
axes.append(flex.size_t(cl.axis))
atr.append(flex.size_t(cl.atoms_to_rotate))
sites = self.residue.atoms().extract_xyz()
if (self.target_map is not None and self.xyzrad_bumpers is not None):
# Get vdW radii
radii = flex.double()
atom_names = []
for a in self.residue.atoms():
atom_names.append(a.name.strip())
converter = iotbx.pdb.residue_name_plus_atom_names_interpreter(
residue_name=self.residue.resname, atom_names=atom_names)
mon_lib_names = converter.atom_name_interpretation.mon_lib_names()
for n in mon_lib_names:
try:
radii.append(self.vdw_radii[n.strip()] - 0.25)
except KeyError:
radii.append(1.5) # XXX U, Uranium, OXT are problems!
#
xyzrad_residue = ext.xyzrad(sites_cart=sites, radii=radii)
#
ro = ext.fit(target_value=start_target_value,
xyzrad_bumpers=self.xyzrad_bumpers,
axes=axes,
rotatable_points_indices=atr,
angles_array=self.chi_angles,
density_map=self.target_map,
all_points=xyzrad_residue,
unit_cell=self.unit_cell,
selection_clash=selection_clash,
selection_rsr=selection_rsr,
sin_table=self.sin_cos_table.sin_table,
cos_table=self.sin_cos_table.cos_table,
step=self.sin_cos_table.step,
n=self.sin_cos_table.n)
elif (self.target_map is not None and self.xyzrad_bumpers is None):
ro = ext.fit(target_value=start_target_value,
axes=axes,
rotatable_points_indices=atr,
angles_array=self.chi_angles,
density_map=self.target_map,
all_points=sites,
unit_cell=self.unit_cell,
selection=selection_rsr,
sin_table=self.sin_cos_table.sin_table,
cos_table=self.sin_cos_table.cos_table,
step=self.sin_cos_table.step,
n=self.sin_cos_table.n)
else:
ro = ext.fit(sites_cart_start=sites_cart_start.deep_copy(),
axes=axes,
rotatable_points_indices=atr,
angles_array=self.chi_angles,
all_points=self.residue.atoms().extract_xyz(),
sin_table=self.sin_cos_table.sin_table,
cos_table=self.sin_cos_table.cos_table,
step=self.sin_cos_table.step,
n=self.sin_cos_table.n)
sites_cart_result = ro.result()
if (sites_cart_result.size() > 0):
dist = None
if (self.accept_only_if_max_shift_is_smaller_than is not None):
dist = flex.max(
flex.sqrt((sites_cart_start - sites_cart_result).dot()))
if (dist is None):
self.residue.atoms().set_xyz(sites_cart_result)
else:
if (dist is not None and
dist < self.accept_only_if_max_shift_is_smaller_than):
self.residue.atoms().set_xyz(sites_cart_result)
else:
self.residue.atoms().set_xyz(sites_cart_start)
else:
self.residue.atoms().set_xyz(sites_cart_start)
def select_best_by_cc_mproc(shot_no, cryst_id, iparams):
if iparams.iotacc.set_id is not None:
data_dir = iparams.data[0] + "/" + iparams.iotacc.set_id
else:
data_dir = iparams.data[0]
cryst_id_shot_no = (cryst_id + "_0_" +
str(shot_no).zfill(iparams.iotacc.LEN_SHOT_SEQ))
# cryst_id_shot_no = cryst_id+str(shot_no).zfill(LEN_SHOT_SEQ)
cc_list = flex.double()
n_refl_list = flex.int()
pickle_filename_list = []
flag_success = False
d_min, d_max = (iparams.iotacc.d_min, iparams.iotacc.d_max)
txt_out = ("ID".center(3) + "shot".center(6) + "res.".center(7) +
"n_refl".center(7) + "cc_pt".center(7) + "cc_fl".center(7) +
"n_ref".center(7) + "cc_pt".center(7) + "cc_fl".center(7) +
"n_iso".center(7) + "a".center(7) + "b".center(7) +
"c".center(7) + "alp".center(7) + "gam".center(7) +
"bet".center(7) + "file name".center(20) + "\n")
for tmp_shot_dir in os.listdir(data_dir + "/" + cryst_id):
if tmp_shot_dir.find(cryst_id_shot_no) > 0:
data = data_dir + "/" + cryst_id + "/" + tmp_shot_dir
# data = data_dir+'/'+tmp_shot_dir
frame_files = []
if os.path.isdir(data):
frame_files = read_pickles(data)
if len(frame_files) == 0:
print(data, " - no pickle file found.")
else:
for pickle_filename in frame_files:
pickle_filename_split = pickle_filename.split("/")
pickle_filename_only = pickle_filename_split[
len(pickle_filename_split) - 1]
pickle_filename_only_split = pickle_filename_only.split(
"_")
observations_Eoc_corrected, observations_original = get_Eoc_corrected_observations(
pickle_filename, iparams)
# check unitcell
uc_params = observations_Eoc_corrected.unit_cell(
).parameters()
cc_eoc, n_refl_eoc, cc_ori, n_refl_ori, cc_iso_eoc, n_refl_iso_eoc, cc_iso_ori, n_refl_iso_ori = (
0,
0,
0,
0,
0,
0,
0,
0,
)
a, b, c, alpha, beta, gamma = uc_params
if good_unit_cell(
uc_params,
iparams.target_unit_cell.parameters(),
iparams.iotacc.uc_tolerance,
):
try:
miller_set = symmetry(
unit_cell=observations_Eoc_corrected.unit_cell(
).parameters(),
space_group_symbol=str(
iparams.target_space_group),
).build_miller_set(
anomalous_flag=iparams.target_anomalous_flag,
d_min=d_min,
)
observations_Eoc_corrected = observations_Eoc_corrected.customized_copy(
anomalous_flag=iparams.target_anomalous_flag,
crystal_symmetry=miller_set.crystal_symmetry(),
)
miller_set = symmetry(
unit_cell=observations_original.unit_cell().
parameters(),
space_group_symbol=str(
iparams.target_space_group),
).build_miller_set(
anomalous_flag=iparams.target_anomalous_flag,
d_min=d_min,
)
observations_original = observations_original.customized_copy(
anomalous_flag=iparams.target_anomalous_flag,
crystal_symmetry=miller_set.crystal_symmetry(),
)
# convert to asymmetric unit index
observations_Eoc_corrected_asu = (
observations_Eoc_corrected.map_to_asu())
observations_original_asu = (
observations_original.map_to_asu())
# calculate CCori CCEoc
miller_array_ref = get_observations_ref(
iparams.hklrefin)
cc_eoc, n_refl_eoc = calc_cc(
miller_array_ref,
observations_Eoc_corrected_asu)
cc_ori, n_refl_ori = calc_cc(
miller_array_ref, observations_original_asu)
miller_array_iso = get_observations_ref(
iparams.hklisoin)
cc_iso_eoc, n_refl_iso_eoc = calc_cc(
miller_array_iso,
observations_Eoc_corrected_asu)
cc_iso_ori, n_refl_iso_ori = calc_cc(
miller_array_iso, observations_original_asu)
except Exception:
dummy = 0
# print cryst_id, shot_no, sel_type.ljust(15,' '), 'mismatch spacegroup %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f'%(a,b,c,alpha,beta,gamma)
if cc_eoc > 0:
pickle_filename_list.append(pickle_filename)
cc_list.append(cc_eoc)
n_refl_list.append(n_refl_eoc)
txt_out += (
cryst_id + " " +
str(shot_no).zfill(iparams.iotacc.LEN_SHOT_SEQ) +
" %6.2f %6.0f %6.2f %6.2f %6.0f %6.2f %6.2f %6.0f %6.2f %6.2f %6.2f %6.2f %6.2f %6.2f"
% (
observations_original.d_min(),
len(observations_original.data()),
cc_ori,
cc_eoc,
n_refl_eoc,
cc_iso_ori,
cc_iso_eoc,
n_refl_iso_eoc,
a,
b,
c,
alpha,
beta,
gamma,
) + " " + pickle_filename_only + "\n")
# select and copy best_by_cc shot
if len(pickle_filename_list) > 0:
# sort by cc and select best n results
cc_thres = flex.max(cc_list) - (flex.max(cc_list) *
iparams.iotacc.percent_top_cc / 100)
i_sel = cc_list > cc_thres
cc_list_sel = cc_list.select(i_sel)
n_refl_list_sel = n_refl_list.select(i_sel)
i_seq_sel = flex.int(range(len(cc_list))).select(i_sel)
pickle_filename_list_sel = [
pickle_filename_list[ii_sel] for ii_sel in i_seq_sel
]
# sort by n_refl and select the first
i_sorted = flex.sort_permutation(n_refl_list_sel, reverse=True)
n_refl_sel = n_refl_list_sel[i_sorted[0]]
cc_sel = cc_list_sel[i_sorted[0]]
pickle_filename_sel = pickle_filename_list_sel[i_sorted[0]]
pickle_filename_sel_split = pickle_filename_sel.split("/")
pickle_filename_sel_only = pickle_filename_sel_split[
len(pickle_filename_sel_split) - 1]
pickle_filename_out = (iparams.run_no + "/" + iparams.iotacc.set_id +
"_" + pickle_filename_sel_only)
# shutil.copyfile(pickle_filename_sel, pickle_filename_out)
txt_out += ("Select --> CC_thres=%6.2f Best CC=%6.2f N_refl=%4.0f " %
(cc_thres, cc_sel, n_refl_sel) + pickle_filename_out +
"\n")
flag_success = True
else:
txt_out += "No image with CC > 0\n"
if flag_success:
return pickle_filename_sel, txt_out
else:
return None
def plot_stats(self, results, iparams):
#retrieve stats from results and plot them
if iparams.flag_plot or iparams.flag_output_verbose:
#for plotting set n_bins = 5 to avoid empty bin
n_bins_plot = 5
#get expected f^2
try:
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(iparams.n_residues)
observations_as_f = results[0].observations.as_amplitude_array(
)
binner_template_asu = observations_as_f.setup_binner(
n_bins=n_bins_plot)
wp = statistics.wilson_plot(observations_as_f,
asu_contents,
e_statistics=True)
expected_f_sq = wp.expected_f_sq
mean_stol_sq = wp.mean_stol_sq
except Exception:
expected_f_sq = flex.double([0] * n_bins_plot)
mean_stol_sq = flex.double(range(n_bins_plot))
print "Warning: Wilson plot calculation in plot stats failed."
#setup list
params_array = np.array([[pres.R_init, pres.R_final, pres.R_xy_init, pres.R_xy_final, \
pres.G, pres.B, pres.rotx*180/math.pi, pres.roty*180/math.pi, \
pres.ry, pres.rz, pres.r0, pres.re, pres.voigt_nu, \
pres.uc_params[0], pres.uc_params[1], pres.uc_params[2], \
pres.uc_params[3], pres.uc_params[4], pres.uc_params[5], \
pres.CC_final, pres.pickle_filename] for pres in results])
params = ['Rinit','Rfinal','Rxyinit', 'Rxyfinal', \
'G','B','rot_x','rot_y','gamma_y','gamma_z','gamma_0','gamma_e','voigtnu' , \
'a','b','c','alpha','beta','gamma','CC','Filename']
#keep parameter history if verbose is selected
if iparams.flag_output_verbose:
fileseq_list = flex.int()
for file_in in os.listdir(iparams.run_no):
if file_in.endswith('.paramhist'):
file_split = file_in.split('.')
fileseq_list.append(int(file_split[0]))
if len(fileseq_list) == 0:
new_fileseq = 0
else:
new_fileseq = flex.max(fileseq_list) + 1
newfile_name = str(new_fileseq) + '.paramhist'
txt_out_verbose = '\n'.join([' '.join(p) for p in params_array])
f = open(iparams.run_no + '/' + newfile_name, 'w')
f.write(txt_out_verbose)
f.close()
#plotting
if iparams.flag_plot:
n_rows = 3
n_cols = int(math.ceil(len(params) / n_rows))
num_bins = 10
for i in xrange(len(params) - 1):
tmp_params = params_array[:, i].astype(np.float)
plt.subplot(n_rows, n_cols, i + 1)
plt.hist(tmp_params,
num_bins,
normed=0,
facecolor='green',
alpha=0.5)
plt.ylabel('Frequencies')
plt.title(params[i] + '\nmu %5.1f med %5.1f sigma %5.1f' %
(np.mean(tmp_params), np.median(tmp_params),
np.std(tmp_params)))
plt.show()
def __init__(self, **kwargs):
group_args.__init__(self, **kwargs)
print('finished Dij, now calculating rho_i and density')
from xfel.clustering import Rodriguez_Laio_clustering_2014 as RL
R = RL(distance_matrix=self.Dij, d_c=self.d_c)
#from IPython import embed; embed(); exit()
#from clustering.plot_with_dimensional_embedding import plot_with_dimensional_embedding
#plot_with_dimensional_embedding(1-self.Dij/flex.max(self.Dij), show_plot=True)
self.rho = rho = R.get_rho()
ave_rho = flex.mean(rho.as_double())
NN = self.Dij.focus()[0]
i_max = flex.max_index(rho)
delta_i_max = flex.max(
flex.double([self.Dij[i_max, j] for j in range(NN)]))
rho_order = flex.sort_permutation(rho, reverse=True)
rho_order_list = list(rho_order)
self.delta = delta = R.get_delta(rho_order=rho_order,
delta_i_max=delta_i_max)
cluster_id = flex.int(NN, -1) # -1 means no cluster
delta_order = flex.sort_permutation(delta, reverse=True)
MAX_PERCENTILE_RHO = self.max_percentile_rho # cluster centers have to be in the top percentile
n_cluster = 0
#
pick_top_solution = False
rho_stdev = flex.mean_and_variance(
rho.as_double()).unweighted_sample_standard_deviation()
delta_stdev = flex.mean_and_variance(
delta).unweighted_sample_standard_deviation()
if rho_stdev != 0.0 and delta_stdev != 0:
rho_z = (rho.as_double() -
flex.mean(rho.as_double())) / (rho_stdev)
delta_z = (delta - flex.mean(delta)) / (delta_stdev)
else:
pick_top_solution = True
if rho_stdev == 0.0:
centroids = [flex.first_index(delta, flex.max(delta))]
elif delta_stdev == 0.0:
centroids = [flex.first_index(rho, flex.max(rho))]
significant_delta = []
significant_rho = []
debug_fix_clustering = True
if debug_fix_clustering:
if not pick_top_solution:
delta_z_cutoff = min(1.0, max(delta_z))
rho_z_cutoff = min(1.0, max(rho_z))
for ic in range(NN):
# test the density & rho
if delta_z[ic] >= delta_z_cutoff:
significant_delta.append(ic)
if rho_z[ic] >= rho_z_cutoff:
significant_rho.append(ic)
centroid_candidates = list(
set(significant_delta).intersection(set(significant_rho)))
# Now compare the relative orders of the max delta_z and max rho_z to make sure they are within 1 stdev
centroids = []
max_delta_z_candidates = -999.9
max_rho_z_candidates = -999.9
for ic in centroid_candidates:
if delta_z[ic] > max_delta_z_candidates:
max_delta_z_candidates = delta_z[ic]
if rho_z[ic] > max_rho_z_candidates:
max_rho_z_candidates = rho_z[ic]
for ic in centroid_candidates:
if max_delta_z_candidates - delta_z[
ic] < 1.0 and max_rho_z_candidates - rho_z[
ic] < 1.0:
centroids.append(ic)
item_idxs = [
delta_order[ic] for ic, centroid in enumerate(centroids)
]
for item_idx in item_idxs:
cluster_id[item_idx] = n_cluster
print('CLUSTERING_STATS', item_idx, cluster_id[item_idx])
n_cluster += 1
####
else:
for ic in range(NN):
item_idx = delta_order[ic]
if ic != 0:
if delta[item_idx] <= 0.25 * delta[
delta_order[0]]: # too low to be a medoid
continue
item_rho_order = rho_order_list.index(item_idx)
if (item_rho_order) / NN < MAX_PERCENTILE_RHO:
cluster_id[item_idx] = n_cluster
print('CLUSTERING_STATS', ic, item_idx, item_rho_order,
cluster_id[item_idx])
n_cluster += 1
###
#
#
print('Found %d clusters' % n_cluster)
for x in range(NN):
if cluster_id[x] >= 0:
print("XC", x, cluster_id[x], rho[x], delta[x])
self.cluster_id_maxima = cluster_id.deep_copy()
R.cluster_assignment(rho_order, cluster_id)
self.cluster_id_full = cluster_id.deep_copy()
#halo = flex.bool(NN,False)
#border = R.get_border( cluster_id = cluster_id )
#for ic in range(n_cluster): #loop thru all border regions; find highest density
# this_border = (cluster_id == ic) & (border==True)
# if this_border.count(True)>0:
# highest_density = flex.max(rho.select(this_border))
# halo_selection = (rho < highest_density) & (this_border==True)
# if halo_selection.count(True)>0:
# cluster_id.set_selected(halo_selection,-1)
# core_selection = (cluster_id == ic) & ~halo_selection
# highest_density = flex.max(rho.select(core_selection))
# too_sparse = core_selection & (rho.as_double() < highest_density/10.) # another heuristic
# if too_sparse.count(True)>0:
# cluster_id.set_selected(too_sparse,-1)
self.cluster_id_final = cluster_id.deep_copy()
def run(args,
log=None,
ccp4_map=None,
return_as_miller_arrays=False,
nohl=False,
space_group_number=None,
out=sys.stdout):
if log is None: log = out
inputs = mmtbx.utils.process_command_line_args(
args=args, master_params=master_params())
got_map = False
if ccp4_map: got_map = True
broadcast(m="Parameters:", log=log)
inputs.params.show(prefix=" ", out=out)
params = inputs.params.extract()
if (ccp4_map is None and inputs.ccp4_map is not None):
broadcast(m="Processing input CCP4 map file: %s" %
inputs.ccp4_map_file_name,
log=log)
ccp4_map = inputs.ccp4_map
ccp4_map.show_summary(prefix=" ", out=out)
got_map = True
if (not got_map):
raise Sorry("Map file is needed.")
#
m = ccp4_map
if (m.space_group_number > 1):
raise Sorry("Input map space group: %d. Must be P1." %
m.space_group_number)
broadcast(m="Input map information:", log=log)
print >> out, "m.all() :", m.data.all()
print >> out, "m.focus() :", m.data.focus()
print >> out, "m.origin():", m.data.origin()
print >> out, "m.nd() :", m.data.nd()
print >> out, "m.size() :", m.data.size()
print >> out, "m.focus_size_1d():", m.data.focus_size_1d()
print >> out, "m.is_0_based() :", m.data.is_0_based()
print >> out, "map: min/max/mean:", flex.min(m.data), flex.max(
m.data), flex.mean(m.data)
print >> out, "unit cell:", m.unit_cell_parameters
#
map_data = m.data
shift_needed = not \
(map_data.focus_size_1d() > 0 and map_data.nd() == 3 and
map_data.is_0_based())
if (shift_needed):
map_data = map_data.shift_origin()
# generate complete set of Miller indices up to given high resolution d_min
n_real = map_data.focus()
if not space_group_number:
space_group_number = 1
if space_group_number <= 1:
symmetry_flags = None
else:
symmetry_flags = maptbx.use_space_group_symmetry,
cs = crystal.symmetry(m.unit_cell_parameters, space_group_number)
crystal_gridding = maptbx.crystal_gridding(
unit_cell=cs.unit_cell(),
space_group_info=cs.space_group_info(),
symmetry_flags=symmetry_flags,
pre_determined_n_real=n_real)
#
d_min = params.d_min
if (d_min is None and not params.box):
d_min = maptbx.d_min_from_map(map_data=map_data,
unit_cell=cs.unit_cell())
if (d_min is None):
# box of reflections in |h|<N1/2, |k|<N2/2, 0<=|l|<N3/2
max_index = [(i - 1) // 2 for i in n_real]
print >> out, "max_index:", max_index
complete_set = miller.build_set(crystal_symmetry=cs,
anomalous_flag=False,
max_index=max_index)
indices = complete_set.indices()
indices.append((0, 0, 0))
complete_set = complete_set.customized_copy(indices=indices)
#if(not params.box):
# # XXX What is sphere resolution corresponding to given box?
# uc = complete_set.unit_cell()
# d1 = uc.d([0,0,max_index[2]])
# d2 = uc.d([0,max_index[1],0])
# d3 = uc.d([max_index[0],1,0])
# print >>out, d1,d2,d3
# complete_set_sp = miller.build_set(
# crystal_symmetry = cs,
# anomalous_flag = False,
# d_min = min(d1,d2,d3))
# complete_set = complete_set.common_set(complete_set_sp)
else:
complete_set = miller.build_set(crystal_symmetry=cs,
anomalous_flag=False,
d_min=d_min)
broadcast(m="Complete set information:", log=log)
complete_set.show_comprehensive_summary(prefix=" ", f=out)
try:
f_obs_cmpl = complete_set.structure_factors_from_map(
map=map_data.as_double(),
use_scale=True,
anomalous_flag=False,
use_sg=False)
except Exception, e:
if (str(e) == "cctbx Error: Miller index not in structure factor map."
):
msg = "Too high resolution requested. Try running with larger d_min."
raise Sorry(msg)
else:
raise Sorry(str(e))
def exercise(space_group_info,
const_gaussian,
negative_gaussian,
anomalous_flag,
allow_mix,
use_u_iso,
use_u_aniso,
d_min=1.,
resolution_factor=1. / 3,
max_prime=5,
quality_factor=100,
wing_cutoff=1.e-6,
exp_table_one_over_step_size=-100,
force_complex=False,
verbose=0):
if (const_gaussian):
elements = ["const"] * 8
elif (negative_gaussian):
elements = ["H"] * 8
else:
elements = ["N", "C", "C", "O", "N", "C", "C", "O"]
if (random.random() < 0.5):
random_f_prime_scale = 0.6
else:
random_f_prime_scale = 0
structure = random_structure.xray_structure(
space_group_info,
elements=elements,
random_f_prime_d_min=1,
random_f_prime_scale=random_f_prime_scale,
random_f_double_prime=anomalous_flag,
use_u_aniso=True,
use_u_iso=False,
random_u_cart_scale=0.3,
random_u_iso=True,
random_occupancy=True)
random_structure.random_modify_adp_and_adp_flags_2(
scatterers=structure.scatterers(),
use_u_iso=use_u_iso,
use_u_aniso=use_u_aniso,
allow_mix=allow_mix,
random_u_iso_scale=0.3,
random_u_iso_min=0.0)
sampled_density_must_be_positive = True
if (negative_gaussian):
reg = structure.scattering_type_registry(
custom_dict={"H": eltbx.xray_scattering.gaussian(-1)})
assert reg.gaussian("H").n_terms() == 0
assert reg.gaussian("H").c() == -1
sampled_density_must_be_positive = False
elif (not const_gaussian and random.random() < 0.5):
if (random.random() < 0.5):
sampled_density_must_be_positive = False
assign_custom_gaussians(
structure, negative_a=not sampled_density_must_be_positive)
f_direct = structure.structure_factors(anomalous_flag=anomalous_flag,
d_min=d_min,
algorithm="direct").f_calc()
crystal_gridding = f_direct.crystal_gridding(
resolution_factor=resolution_factor, d_min=d_min, max_prime=max_prime)
assert crystal_gridding.symmetry_flags() is None
rfft = fftpack.real_to_complex_3d(crystal_gridding.n_real())
u_base = xray.calc_u_base(d_min, resolution_factor, quality_factor)
omptbx.env.num_threads = libtbx.introspection.number_of_processors()
sampled_density = xray.sampled_model_density(
unit_cell=structure.unit_cell(),
scatterers=structure.scatterers(),
scattering_type_registry=structure.scattering_type_registry(),
fft_n_real=rfft.n_real(),
fft_m_real=rfft.m_real(),
u_base=u_base,
wing_cutoff=wing_cutoff,
exp_table_one_over_step_size=exp_table_one_over_step_size,
force_complex=force_complex,
sampled_density_must_be_positive=sampled_density_must_be_positive,
tolerance_positive_definite=1.e-5,
use_u_base_as_u_extra=False)
focus = sampled_density.real_map_unpadded().focus()
all = sampled_density.real_map_unpadded().all()
last = sampled_density.real_map_unpadded().last()
assert approx_equal(focus, last)
assert approx_equal(all, last)
assert sampled_density.anomalous_flag() == (anomalous_flag
or force_complex)
if (0 or verbose):
print "const_gaussian:", const_gaussian
print "negative_gaussian:", negative_gaussian
print "number of scatterers passed:", \
sampled_density.n_scatterers_passed()
print "number of contributing scatterers:", \
sampled_density.n_contributing_scatterers()
print "number of anomalous scatterers:", \
sampled_density.n_anomalous_scatterers()
print "wing_cutoff:", sampled_density.wing_cutoff()
print "exp_table_one_over_step_size:", \
sampled_density.exp_table_one_over_step_size()
print "exp_table_size:", sampled_density.exp_table_size()
print "max_sampling_box_edges:", sampled_density.max_sampling_box_edges(
),
print "(%.4f, %.4f, %.4f)" % sampled_density.max_sampling_box_edges_frac(
)
if (not sampled_density.anomalous_flag()):
print "map min:", flex.min(sampled_density.real_map())
print "map max:", flex.max(sampled_density.real_map())
else:
print "map min:", flex.min(flex.real(
sampled_density.complex_map())),
print flex.min(flex.imag(sampled_density.complex_map()))
print "map max:", flex.max(flex.real(
sampled_density.complex_map())),
print flex.max(flex.imag(sampled_density.complex_map()))
if (not sampled_density.anomalous_flag() and negative_gaussian):
assert flex.min(sampled_density.real_map()) < 0
assert flex.max(sampled_density.real_map()) == 0
if (not sampled_density.anomalous_flag()):
map = sampled_density.real_map()
assert map.all() == rfft.m_real()
assert map.focus() == rfft.n_real()
sf_map = rfft.forward(map)
assert sf_map.all() == rfft.n_complex()
assert sf_map.focus() == rfft.n_complex()
collect_conj = True
else:
cfft = fftpack.complex_to_complex_3d(rfft.n_real())
map = sampled_density.complex_map()
assert map.all() == cfft.n()
assert map.focus() == cfft.n()
sf_map = cfft.backward(map)
assert sf_map.all() == cfft.n()
assert sf_map.focus() == cfft.n()
collect_conj = False
f_fft_data = maptbx.structure_factors.from_map(
space_group=f_direct.space_group(),
anomalous_flag=sampled_density.anomalous_flag(),
miller_indices=f_direct.indices(),
complex_map=sf_map,
conjugate_flag=collect_conj).data()
sampled_density.eliminate_u_extra_and_normalize(f_direct.indices(),
f_fft_data)
structure_factor_utils.check_correlation("direct/fft_regression",
f_direct.indices(),
0,
f_direct.data(),
f_fft_data,
min_corr_ampl=1 * 0.99,
max_mean_w_phase_error=1 * 3.,
verbose=verbose)
f_fft = xray.structure_factors.from_scatterers(
miller_set=f_direct,
grid_resolution_factor=resolution_factor,
quality_factor=quality_factor,
wing_cutoff=wing_cutoff,
exp_table_one_over_step_size=exp_table_one_over_step_size,
sampled_density_must_be_positive=sampled_density_must_be_positive,
max_prime=max_prime)(xray_structure=structure,
miller_set=f_direct,
algorithm="fft").f_calc()
structure_factor_utils.check_correlation("direct/fft_xray",
f_direct.indices(),
0,
f_direct.data(),
f_fft.data(),
min_corr_ampl=1 * 0.99,
max_mean_w_phase_error=1 * 3.,
verbose=verbose)
def __init__(self, **kwargs):
group_args.__init__(self, **kwargs)
# require Dij, d_c
P = Profiler("2. calculate rho density")
print("finished Dij, now calculating rho_i, the density")
from xfel.clustering import Rodriguez_Laio_clustering_2014
# alternative clustering algorithms: see http://scikit-learn.org/stable/modules/clustering.html
# also see https://cran.r-project.org/web/packages/dbscan/vignettes/hdbscan.html
# see also https://en.wikipedia.org/wiki/Hausdorff_dimension
R = Rodriguez_Laio_clustering_2014(distance_matrix=self.Dij,
d_c=self.d_c)
self.rho = rho = R.get_rho()
ave_rho = flex.mean(rho.as_double())
NN = self.Dij.focus()[0]
print("The average rho_i is %5.2f, or %4.1f%%" %
(ave_rho, 100 * ave_rho / NN))
i_max = flex.max_index(rho)
P = Profiler("3.transition")
print("the index with the highest density is %d" % (i_max))
delta_i_max = flex.max(
flex.double([self.Dij[i_max, j] for j in range(NN)]))
print("delta_i_max", delta_i_max)
rho_order = flex.sort_permutation(rho, reverse=True)
rho_order_list = list(rho_order)
P = Profiler("4. delta")
self.delta = delta = R.get_delta(rho_order=rho_order,
delta_i_max=delta_i_max)
P = Profiler("5. find cluster maxima")
#---- Now hunting for clusters ---Lot's of room for improvement (or simplification) here!!!
cluster_id = flex.int(NN, -1) # default -1 means no cluster
delta_order = flex.sort_permutation(delta, reverse=True)
N_CLUST = 10 # maximum of 10 points to be considered as possible clusters
#MAX_PERCENTILE_DELTA = 0.99 # cluster centers have to be in the top 10% percentile delta
MAX_PERCENTILE_RHO = 0.99 # cluster centers have to be in the top 75% percentile rho
n_cluster = 0
#max_n_delta = min(N_CLUST, int(MAX_PERCENTILE_DELTA*NN))
for ic in range(NN):
# test the density, rho
item_idx = delta_order[ic]
if delta[item_idx] > 100:
print("A: iteration", ic, "delta", delta[item_idx],
delta[item_idx] < 0.25 * delta[delta_order[0]])
if delta[item_idx] < 0.25 * delta[
delta_order[0]]: # too low (another heuristic!)
continue
item_rho_order = rho_order_list.index(item_idx)
if delta[item_idx] > 100:
print("B: iteration", ic, item_rho_order, item_rho_order / NN,
MAX_PERCENTILE_RHO)
if item_rho_order / NN < MAX_PERCENTILE_RHO:
cluster_id[item_idx] = n_cluster
print(ic, item_idx, item_rho_order, cluster_id[item_idx])
n_cluster += 1
print("Found %d clusters" % n_cluster)
for x in range(NN):
if cluster_id[x] >= 0:
print("XC", x, cluster_id[x], rho[x], delta[x])
self.cluster_id_maxima = cluster_id.deep_copy()
P = Profiler("6. assign all points")
R.cluster_assignment(rho_order, cluster_id)
self.cluster_id_full = cluster_id.deep_copy()
# assign the halos
P = Profiler("7. assign halos")
halo = flex.bool(NN, False)
border = R.get_border(cluster_id=cluster_id)
for ic in range(n_cluster
): #loop thru all border regions; find highest density
print("cluster", ic, "in border", border.count(True))
this_border = (cluster_id == ic) & (border == True)
print(len(this_border), this_border.count(True))
if this_border.count(True) > 0:
highest_density = flex.max(rho.select(this_border))
halo_selection = (rho < highest_density) & (this_border
== True)
if halo_selection.count(True) > 0:
cluster_id.set_selected(halo_selection, -1)
core_selection = (cluster_id == ic) & ~halo_selection
highest_density = flex.max(rho.select(core_selection))
too_sparse = core_selection & (
rho.as_double() < highest_density / 10.
) # another heuristic
if too_sparse.count(True) > 0:
cluster_id.set_selected(too_sparse, -1)
self.cluster_id_final = cluster_id.deep_copy()
print("%d in the excluded halo" % ((cluster_id == -1).count(True)))
def generate_view_data (self) :
from scitbx.array_family import flex
from scitbx import graphics_utils
settings = self.settings
data_for_colors = data_for_radii = None
data = self.data #self.work_array.data()
if (isinstance(data, flex.double) and data.all_eq(0)):
data = flex.double(data.size(), 1)
if ((self.multiplicities is not None) and
(settings.scale_colors_multiplicity)) :
data_for_colors = self.multiplicities.data().as_double()
assert data_for_colors.size() == data.size()
elif (settings.sqrt_scale_colors) and (isinstance(data, flex.double)) :
data_for_colors = flex.sqrt(data)
else :
data_for_colors = data.deep_copy()
if ((self.multiplicities is not None) and
(settings.scale_radii_multiplicity)) :
#data_for_radii = data.deep_copy()
data_for_radii = self.multiplicities.data().as_double()
assert data_for_radii.size() == data.size()
elif (settings.sqrt_scale_radii) and (isinstance(data, flex.double)) :
data_for_radii = flex.sqrt(data)
else :
data_for_radii = data.deep_copy()
if (settings.slice_mode) :
data = data.select(self.slice_selection)
if (not settings.keep_constant_scale) :
data_for_radii = data_for_radii.select(self.slice_selection)
data_for_colors = data_for_colors.select(self.slice_selection)
if (settings.color_scheme in ["rainbow", "heatmap", "redblue"]) :
colors = graphics_utils.color_by_property(
properties=data_for_colors,
selection=flex.bool(data_for_colors.size(), True),
color_all=False,
gradient_type=settings.color_scheme)
elif (settings.color_scheme == "grayscale") :
colors = graphics_utils.grayscale_by_property(
properties=data_for_colors,
selection=flex.bool(data_for_colors.size(), True),
shade_all=False,
invert=settings.black_background)
else :
if (settings.black_background) :
base_color = (1.0,1.0,1.0)
else :
base_color = (0.0,0.0,0.0)
colors = flex.vec3_double(data_for_colors.size(), base_color)
if (settings.slice_mode) and (settings.keep_constant_scale) :
colors = colors.select(self.slice_selection)
data_for_radii = data_for_radii.select(self.slice_selection)
uc = self.work_array.unit_cell()
abc = uc.parameters()[0:3]
min_dist = min(uc.reciprocal_space_vector((1,1,1)))
min_radius = 0.20 * min_dist
max_radius = 40 * min_dist
if (settings.sqrt_scale_radii) and (not settings.scale_radii_multiplicity):
data_for_radii = flex.sqrt(data_for_radii)
if len(data_for_radii):
max_value = flex.max(data_for_radii)
scale = max_radius / max_value
radii = data_for_radii * scale
too_small = radii < min_radius
if (too_small.count(True) > 0) :
radii.set_selected(too_small, flex.double(radii.size(), min_radius))
assert radii.size() == colors.size()
else:
radii = flex.double()
max_radius = 0
self.radii = radii
self.max_radius = max_radius
self.colors = colors
def run_detail(show_plot, save_plot):
P = Profiler("0. Read data")
import sys
file_name = sys.argv[1]
from xfel.clustering.singleframe import CellOnlyFrame
from cctbx import crystal
cells = []
for line in open(file_name, "r"):
tokens = line.strip().split()
unit_cell = tuple(float(x) for x in tokens[0:6])
space_group_symbol = tokens[6]
crystal_symmetry = crystal.symmetry(
unit_cell=unit_cell, space_group_symbol=space_group_symbol)
cells.append(CellOnlyFrame(crystal_symmetry, path=None))
MM = [c.mm for c in cells] # get all metrical matrices
MM_double = flex.double()
for i in range(len(MM)):
Tup = MM[i]
for j in range(6):
MM_double.append(Tup[j])
print(("There are %d cells" % (len(MM))))
coord_x = flex.double([c.uc[0] for c in cells])
coord_y = flex.double([c.uc[1] for c in cells])
if show_plot or save_plot:
import matplotlib
if not show_plot:
# http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear
matplotlib.use('Agg') # use a non-interactive backend
from matplotlib import pyplot as plt
plt.plot([c.uc[0] for c in cells], [c.uc[1] for c in cells],
"k.",
markersize=3.)
plt.axes().set_aspect("equal")
if save_plot:
plt.savefig(plot_name,
size_inches=(10, 10),
dpi=300,
bbox_inches='tight')
if show_plot:
plt.show()
print("Now constructing a Dij matrix.")
P = Profiler("1. compute Dij matrix")
NN = len(MM)
import omptbx
omptbx.omp_set_num_threads(64)
from cctbx.uctbx.determine_unit_cell import NCDist_matrix, NCDist_flatten
#Dij = NCDist_matrix(MM_double) # double loop, less efficient evaluation of upper triangle
Dij = NCDist_flatten(MM_double) # loop is flattened
#from cctbx.uctbx.determine_unit_cell import NCDist # can this be refactored with MPI?
#Dij = flex.double(flex.grid(NN,NN))
#for i in xrange(NN):
# for j in xrange(i+1,NN):
# Dij[i,j] = NCDist(MM[i], MM[j])
del P
d_c = 10 # the distance cutoff, such that average item neighbors 1-2% of all items
CM = clustering_manager(Dij=Dij, d_c=d_c)
# Summarize the results here
n_cluster = 1 + flex.max(CM.cluster_id_final)
print(len(cells), "have been analyzed")
print(("# ------------ %d CLUSTERS ----------------" % (n_cluster)))
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
print("Cluster %d. Central unit cell: item %d" % (i, item))
cells[item].crystal_symmetry.show_summary()
print("Cluster has %d items, or %d after trimming borders" %
((CM.cluster_id_full == i).count(True),
(CM.cluster_id_final == i).count(True)))
print()
appcolors = [
'b', 'r', '#ff7f0e', '#2ca02c', '#9467bd', '#8c564b', '#e377c2',
'#7f7f7f', '#bcbd22', '#17becf'
]
if show_plot:
#Decision graph
from matplotlib import pyplot as plt
plt.plot(CM.rho, CM.delta, "r.", markersize=3.)
for x in range(NN):
if CM.cluster_id_maxima[x] >= 0:
plt.plot([CM.rho[x]], [CM.delta[x]], "ro")
plt.show()
#No-halo plot
from matplotlib import pyplot as plt
colors = [appcolors[i % 10] for i in CM.cluster_id_full]
plt.scatter(coord_x,
coord_y,
marker='o',
color=colors,
linewidths=0.4,
edgecolor='k')
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
plt.plot([cells[item].uc[0]], [cells[item].uc[1]], 'y.')
plt.axes().set_aspect("equal")
plt.show()
#Final plot
halo = (CM.cluster_id_final == -1)
core = ~halo
plt.plot(coord_x.select(halo), coord_y.select(halo), "k.")
colors = [appcolors[i % 10] for i in CM.cluster_id_final.select(core)]
plt.scatter(coord_x.select(core),
coord_y.select(core),
marker="o",
color=colors,
linewidths=0.4,
edgecolor='k')
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
plt.plot([cells[item].uc[0]], [cells[item].uc[1]], 'y.')
plt.axes().set_aspect("equal")
plt.show()
def plot_overall_completeness(completeness):
completeness_range = range(-1, flex.max(completeness) + 1)
completeness_counts = [completeness.count(n) for n in completeness_range]
from matplotlib import pyplot as plt
plt.plot(completeness_range, completeness_counts, "r+")
plt.show()
def run(args,
command_name="phenix.reflection_file_converter",
simply_return_all_miller_arrays=False):
command_line = (option_parser(
usage="%s [options] reflection_file ..." % command_name,
description="Example: %s w1.sca --mtz ." % command_name
).enable_symmetry_comprehensive().option(
None,
"--weak_symmetry",
action="store_true",
default=False,
help="symmetry on command line is weaker than symmetry found in files"
).enable_resolutions().option(
None,
"--label",
action="store",
type="string",
help="Substring of reflection data label or number",
metavar="STRING"
).option(
None,
"--non_anomalous",
action="store_true",
default=False,
help="Averages Bijvoet mates to obtain a non-anomalous array"
).option(
None,
"--r_free_label",
action="store",
type="string",
help="Substring of reflection data label or number",
metavar="STRING"
).option(
None,
"--r_free_test_flag_value",
action="store",
type="int",
help="Value in R-free array indicating assignment to free set.",
metavar="FLOAT"
).option(
None,
"--generate_r_free_flags",
action="store_true",
default=False,
help="Generates a new array of random R-free flags"
" (MTZ and CNS output only)."
).option(
None,
"--use_lattice_symmetry_in_r_free_flag_generation",
dest="use_lattice_symmetry_in_r_free_flag_generation",
action="store_true",
default=True,
help="group twin/pseudo symmetry related reflections together"
" in r-free set (this is the default)."
).option(
None,
"--no_lattice_symmetry_in_r_free_flag_generation",
dest="use_lattice_symmetry_in_r_free_flag_generation",
action="store_false",
help="opposite of --use-lattice-symmetry-in-r-free-flag-generation"
).option(
None,
"--r_free_flags_fraction",
action="store",
default=0.10,
type="float",
help="Target fraction free/work reflections (default: 0.10).",
metavar="FLOAT"
).option(
None,
"--r_free_flags_max_free",
action="store",
default=2000,
type="int",
help="Maximum number of free reflections (default: 2000).",
metavar="INT"
).option(
None,
"--r_free_flags_format",
choices=(
"cns", "ccp4",
"shelx"),
default="cns",
help="Convention for generating R-free flags",
metavar="cns|ccp4"
).option(
None,
"--output_r_free_label",
action="store",
type="string",
help=
"Label for newly generated R-free flags (defaults to R-free-flags)",
default="R-free-flags",
metavar="STRING"
).option(
None,
"--random_seed",
action="store",
type="int",
help="Seed for random number generator (affects generation of"
" R-free flags).",
metavar="INT"
).option(
None,
"--change_of_basis",
action="store",
type="string",
help="Change-of-basis operator: h,k,l or x,y,z"
" or to_reference_setting, to_primitive_setting, to_niggli_cell,"
" to_inverse_hand",
metavar="STRING"
).option(
None,
"--eliminate_invalid_indices",
action="store_true",
default=False,
help="Remove indices which are invalid given the change of basis desired"
).option(
None,
"--expand_to_p1",
action="store_true",
default=False,
help="Generates all symmetrically equivalent reflections."
" The space group symmetry is reset to P1."
" May be used in combination with --change_to_space_group to"
" lower the symmetry."
).option(
None,
"--change_to_space_group",
action="store",
type="string",
help="Changes the space group and merges equivalent reflections"
" if necessary",
metavar="SYMBOL|NUMBER"
).option(
None,
"--write_mtz_amplitudes",
action="store_true",
default=False,
help="Converts intensities to amplitudes before writing MTZ format;"
" requires --mtz_root_label"
).option(
None,
"--write_mtz_intensities",
action="store_true",
default=False,
help="Converts amplitudes to intensities before writing MTZ format;"
" requires --mtz_root_label"
).option(
None,
"--remove_negatives",
action="store_true",
default=False,
help="Remove negative intensities or amplitudes from the data set"
).option(
None,
"--massage_intensities",
action="store_true",
default=False,
help="'Treat' negative intensities to get a positive amplitude."
" |Fnew| = sqrt((Io+sqrt(Io**2 +2sigma**2))/2.0). Requires"
" intensities as input and the flags --mtz,"
" --write_mtz_amplitudes and --mtz_root_label."
).option(
None,
"--scale_max",
action="store",
type="float",
help="Scales data such that the maximum is equal to the given value",
metavar="FLOAT"
).option(
None,
"--scale_factor",
action="store",
type="float",
help="Multiplies data with the given factor",
metavar="FLOAT"
).option(
None,
"--write_unmerged",
action="store_true",
default=False,
help="Do not perform any merging of input data"
).option(
None,
"--sca",
action="store",
type="string",
help=
"write data to Scalepack FILE ('--sca .' copies name of input file)",
metavar="FILE"
).option(
None,
"--mtz",
action="store",
type="string",
help="write data to MTZ FILE ('--mtz .' copies name of input file)",
metavar="FILE"
).option(
None,
"--mtz_root_label",
action="store",
type="string",
help="Root label for MTZ file (e.g. Fobs)",
metavar="STRING"
).option(
None,
"--cns",
action="store",
type="string",
help="write data to CNS FILE ('--cns .' copies name of input file)",
metavar="FILE"
).option(
None,
"--shelx",
action="store",
type="string",
help=
"write intensity or amplitude data to SHELX FILE ('--shelx .' copies name of input file)",
metavar="FILE")).process(args=args)
co = command_line.options
if (co.random_seed is not None):
random.seed(co.random_seed)
flex.set_random_seed(value=co.random_seed)
if (co.write_mtz_amplitudes and co.write_mtz_intensities):
print()
print("--write_mtz_amplitudes and --write_mtz_intensities" \
" are mutually exclusive.")
print()
return None
if (co.write_mtz_amplitudes or co.write_mtz_intensities):
if (co.mtz_root_label is None):
print()
print("--write_mtz_amplitudes and --write_mtz_intensities" \
" require --mtz_root_label.")
print()
return None
if (co.scale_max is not None and co.scale_factor is not None):
print()
print("--scale_max and --scale_factor are mutually exclusive.")
print()
return None
if (len(command_line.args) == 0):
command_line.parser.show_help()
return None
all_miller_arrays = reflection_file_reader.collect_arrays(
file_names=command_line.args,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=False,
discard_arrays=False,
verbose=1)
if (simply_return_all_miller_arrays):
return all_miller_arrays
if (len(all_miller_arrays) == 0):
print()
print("No reflection data found in input file%s." %
(plural_s(len(command_line.args))[1]))
print()
return None
label_table = reflection_file_utils.label_table(
miller_arrays=all_miller_arrays)
selected_array = label_table.select_array(label=co.label,
command_line_switch="--label")
if (selected_array is None): return None
r_free_flags = None
r_free_info = None
if (co.r_free_label is not None):
r_free_flags = label_table.match_data_label(
label=co.r_free_label, command_line_switch="--r_free_label")
if (r_free_flags is None):
return None
r_free_info = str(r_free_flags.info())
if (not r_free_flags.is_bool_array()):
test_flag_value = reflection_file_utils.get_r_free_flags_scores(
miller_arrays=[r_free_flags],
test_flag_value=co.r_free_test_flag_value).test_flag_values[0]
if (test_flag_value is None):
if (co.r_free_test_flag_value is None):
raise Sorry(
"Cannot automatically determine r_free_test_flag_value."
" Please use --r_free_test_flag_value to specify a value."
)
else:
raise Sorry("Invalid --r_free_test_flag_value.")
r_free_flags = r_free_flags.customized_copy(
data=(r_free_flags.data() == test_flag_value))
print("Selected data:")
print(" ", selected_array.info())
print(" Observation type:", selected_array.observation_type())
print()
if (r_free_info is not None):
print("R-free flags:")
print(" ", r_free_info)
print()
processed_array = selected_array.customized_copy(
crystal_symmetry=selected_array.join_symmetry(
other_symmetry=command_line.symmetry,
force=not co.weak_symmetry)).set_observation_type(
selected_array.observation_type())
if (r_free_flags is not None):
r_free_flags = r_free_flags.customized_copy(
crystal_symmetry=processed_array)
print("Input crystal symmetry:")
crystal.symmetry.show_summary(processed_array, prefix=" ")
print()
if (processed_array.unit_cell() is None):
command_line.parser.show_help()
print(
"Unit cell parameters unknown. Please use --symmetry or --unit_cell."
)
print()
return None
if (processed_array.space_group_info() is None):
command_line.parser.show_help()
print("Space group unknown. Please use --symmetry or --space_group.")
print()
return None
if (r_free_flags is not None):
r_free_flags = r_free_flags.customized_copy(
crystal_symmetry=processed_array)
if (co.change_of_basis is not None):
processed_array, cb_op = processed_array.apply_change_of_basis(
change_of_basis=co.change_of_basis,
eliminate_invalid_indices=co.eliminate_invalid_indices)
if (r_free_flags is not None):
r_free_flags = r_free_flags.change_basis(cb_op=cb_op)
if (not processed_array.is_unique_set_under_symmetry()
and not co.write_unmerged):
print("Merging symmetry-equivalent values:")
merged = processed_array.merge_equivalents()
merged.show_summary(prefix=" ")
print()
processed_array = merged.array()
del merged
processed_array.show_comprehensive_summary(prefix=" ")
print()
if (r_free_flags is not None
and not r_free_flags.is_unique_set_under_symmetry()
and not co.write_unmerged):
print("Merging symmetry-equivalent R-free flags:")
merged = r_free_flags.merge_equivalents()
merged.show_summary(prefix=" ")
print()
r_free_flags = merged.array()
del merged
r_free_flags.show_comprehensive_summary(prefix=" ")
print()
if (co.expand_to_p1):
print("Expanding symmetry and resetting space group to P1:")
if (r_free_flags is not None):
raise Sorry(
"--expand_to_p1 not supported for arrays of R-free flags.")
processed_array = processed_array.expand_to_p1()
processed_array.show_comprehensive_summary(prefix=" ")
print()
if (co.change_to_space_group is not None):
if (r_free_flags is not None):
raise Sorry(
"--change_to_space_group not supported for arrays of R-free flags."
)
new_space_group_info = sgtbx.space_group_info(
symbol=co.change_to_space_group)
print("Change to space group:", new_space_group_info)
new_crystal_symmetry = crystal.symmetry(
unit_cell=processed_array.unit_cell(),
space_group_info=new_space_group_info,
assert_is_compatible_unit_cell=False)
if (not new_crystal_symmetry.unit_cell().is_similar_to(
processed_array.unit_cell())):
print(" *************")
print(" W A R N I N G")
print(" *************")
print(
" Unit cell parameters adapted to new space group symmetry are"
)
print(" significantly different from input unit cell parameters:")
print(" Input unit cell parameters:", \
processed_array.unit_cell())
print(" Adapted unit cell parameters:", \
new_crystal_symmetry.unit_cell())
processed_array = processed_array.customized_copy(
crystal_symmetry=new_crystal_symmetry)
print()
if (not processed_array.is_unique_set_under_symmetry()
and not co.write_unmerged):
print(" Merging values symmetry-equivalent under new symmetry:")
merged = processed_array.merge_equivalents()
merged.show_summary(prefix=" ")
print()
processed_array = merged.array()
del merged
processed_array.show_comprehensive_summary(prefix=" ")
print()
if (processed_array.anomalous_flag() and co.non_anomalous):
print("Converting data array from anomalous to non-anomalous.")
if (not processed_array.is_xray_intensity_array()):
processed_array = processed_array.average_bijvoet_mates()
else:
processed_array = processed_array.average_bijvoet_mates()
processed_array.set_observation_type_xray_intensity()
if (r_free_flags is not None and r_free_flags.anomalous_flag()
and co.non_anomalous):
print("Converting R-free flags from anomalous to non-anomalous.")
r_free_flags = r_free_flags.average_bijvoet_mates()
d_max = co.low_resolution
d_min = co.resolution
if (d_max is not None or d_min is not None):
if (d_max is not None):
print("Applying low resolution cutoff: d_max=%.6g" % d_max)
if (d_min is not None):
print("Applying high resolution cutoff: d_min=%.6g" % d_min)
processed_array = processed_array.resolution_filter(d_max=d_max,
d_min=d_min)
print("Number of reflections:", processed_array.indices().size())
print()
if (co.scale_max is not None):
print("Scaling data such that the maximum value is: %.6g" %
co.scale_max)
processed_array = processed_array.apply_scaling(
target_max=co.scale_max)
print()
if (co.scale_factor is not None):
print("Multiplying data with the factor: %.6g" % co.scale_factor)
processed_array = processed_array.apply_scaling(factor=co.scale_factor)
print()
if (([co.remove_negatives, co.massage_intensities]).count(True) == 2):
raise Sorry("It is not possible to use --remove_negatives and"
" --massage_intensities at the same time.")
if (co.remove_negatives):
if processed_array.is_real_array():
print("Removing negatives items")
processed_array = processed_array.select(
processed_array.data() > 0)
if processed_array.sigmas() is not None:
processed_array = processed_array.select(
processed_array.sigmas() > 0)
else:
raise Sorry(
"--remove_negatives not applicable to complex data arrays.")
if (co.massage_intensities):
if processed_array.is_real_array():
if processed_array.is_xray_intensity_array():
if (co.mtz is not None):
if (co.write_mtz_amplitudes):
print(
"The supplied intensities will be used to estimate"
)
print(" amplitudes in the following way: ")
print(
" Fobs = Sqrt[ (Iobs + Sqrt(Iobs**2 + 2sigmaIobs**2))/2 ]"
)
print(" Sigmas are estimated in a similar manner.")
print()
processed_array = processed_array.enforce_positive_amplitudes(
)
else:
raise Sorry(
"--write_mtz_amplitudes has to be specified when using"
" --massage_intensities")
else:
raise Sorry(
"--mtz has to be used when using --massage_intensities"
)
else:
raise Sorry(
"Intensities must be supplied when using the option"
" --massage_intensities")
else:
raise Sorry(
"--massage_intensities not applicable to complex data arrays.")
if (not co.generate_r_free_flags):
if (r_free_flags is None):
r_free_info = []
else:
if (r_free_flags.anomalous_flag() !=
processed_array.anomalous_flag()):
if (processed_array.anomalous_flag()): is_not = ("", " not")
else: is_not = (" not", "")
raise Sorry(
"The data array is%s anomalous but the R-free array is%s.\n"
% is_not + " Please try --non_anomalous.")
r_free_info = ["R-free flags source: " + r_free_info]
if (not r_free_flags.indices().all_eq(processed_array.indices())):
processed_array = processed_array.map_to_asu()
r_free_flags = r_free_flags.map_to_asu().common_set(
processed_array)
n_missing_r_free_flags = processed_array.indices().size() \
- r_free_flags.indices().size()
if (n_missing_r_free_flags != 0):
raise Sorry(
"R-free flags not compatible with data array:"
" missing flag for %d reflections selected for output."
% n_missing_r_free_flags)
else:
if (r_free_flags is not None):
raise Sorry(
"--r_free_label and --generate_r_free_flags are mutually exclusive."
)
print("Generating a new array of R-free flags:")
r_free_flags = processed_array.generate_r_free_flags(
fraction=co.r_free_flags_fraction,
max_free=co.r_free_flags_max_free,
use_lattice_symmetry=co.
use_lattice_symmetry_in_r_free_flag_generation,
format=co.r_free_flags_format)
test_flag_value = True
if (co.r_free_flags_format == "ccp4"):
test_flag_value = 0
elif (co.r_free_flags_format == "shelx"):
test_flag_value = -1
r_free_as_bool = r_free_flags.customized_copy(
data=r_free_flags.data() == test_flag_value)
r_free_info = ["R-free flags generated by %s:" % command_name]
r_free_info.append(" " + date_and_time())
r_free_info.append(" fraction: %.6g" % co.r_free_flags_fraction)
r_free_info.append(" max_free: %s" % str(co.r_free_flags_max_free))
r_free_info.append(" size of work set: %d" %
r_free_as_bool.data().count(False))
r_free_info.append(" size of free set: %d" %
r_free_as_bool.data().count(True))
r_free_info_str = StringIO()
r_free_as_bool.show_r_free_flags_info(prefix=" ", out=r_free_info_str)
if (co.r_free_flags_format == "ccp4"):
r_free_info.append(" convention: CCP4 (test=0, work=1-%d)" %
flex.max(r_free_flags.data()))
elif (co.r_free_flags_format == "shelx"):
r_free_info.append(" convention: SHELXL (test=-1, work=1)")
else:
r_free_info.append(" convention: CNS/X-PLOR (test=1, work=0)")
print("\n".join(r_free_info[2:4]))
print(r_free_info[-1])
print(r_free_info_str.getvalue())
print()
n_output_files = 0
if (co.sca is not None):
if (co.generate_r_free_flags):
raise Sorry("Cannot write R-free flags to Scalepack file.")
file_name = reflection_file_utils.construct_output_file_name(
input_file_names=[selected_array.info().source],
user_file_name=co.sca,
file_type_label="Scalepack",
file_extension="sca")
print("Writing Scalepack file:", file_name)
iotbx.scalepack.merge.write(file_name=file_name,
miller_array=processed_array)
n_output_files += 1
print()
if (co.mtz is not None):
file_name = reflection_file_utils.construct_output_file_name(
input_file_names=[selected_array.info().source],
user_file_name=co.mtz,
file_type_label="MTZ",
file_extension="mtz")
print("Writing MTZ file:", file_name)
mtz_history_buffer = flex.std_string()
mtz_history_buffer.append(date_and_time())
mtz_history_buffer.append("> program: %s" % command_name)
mtz_history_buffer.append(
"> input file name: %s" %
os.path.basename(selected_array.info().source))
mtz_history_buffer.append(
"> input directory: %s" %
os.path.dirname(os.path.abspath(selected_array.info().source)))
mtz_history_buffer.append("> input labels: %s" %
selected_array.info().label_string())
mtz_output_array = processed_array
if (co.write_mtz_amplitudes):
if (not mtz_output_array.is_xray_amplitude_array()):
print(" Converting intensities to amplitudes.")
mtz_output_array = mtz_output_array.f_sq_as_f()
mtz_history_buffer.append(
"> Intensities converted to amplitudes.")
elif (co.write_mtz_intensities):
if (not mtz_output_array.is_xray_intensity_array()):
print(" Converting amplitudes to intensities.")
mtz_output_array = mtz_output_array.f_as_f_sq()
mtz_history_buffer.append(
"> Amplitudes converted to intensities.")
column_root_label = co.mtz_root_label
if (column_root_label is None):
# XXX 2013-03-29: preserve original root label by default
# XXX 2014-12-16: skip trailing "(+)" in root_label if anomalous
column_root_label = selected_array.info().labels[0]
column_root_label = remove_anomalous_suffix_if_necessary(
miller_array=selected_array, column_root_label=column_root_label)
mtz_dataset = mtz_output_array.as_mtz_dataset(
column_root_label=column_root_label)
del mtz_output_array
if (r_free_flags is not None):
mtz_dataset.add_miller_array(
miller_array=r_free_flags,
column_root_label=co.output_r_free_label)
for line in r_free_info:
mtz_history_buffer.append("> " + line)
mtz_history_buffer.append("> output file name: %s" %
os.path.basename(file_name))
mtz_history_buffer.append("> output directory: %s" %
os.path.dirname(os.path.abspath(file_name)))
mtz_object = mtz_dataset.mtz_object()
mtz_object.add_history(mtz_history_buffer)
mtz_object.write(file_name=file_name)
n_output_files += 1
print()
if (co.cns is not None):
file_name = reflection_file_utils.construct_output_file_name(
input_file_names=[selected_array.info().source],
user_file_name=co.cns,
file_type_label="CNS",
file_extension="cns")
print("Writing CNS file:", file_name)
processed_array.export_as_cns_hkl(
file_object=open(file_name, "w"),
file_name=file_name,
info=["source of data: " + str(selected_array.info())] +
r_free_info,
r_free_flags=r_free_flags)
n_output_files += 1
print()
if (co.shelx is not None):
if (co.generate_r_free_flags):
raise Sorry("Cannot write R-free flags to SHELX file.")
file_extension = "hkl"
if (co.shelx.endswith("shelx")):
file_extension = "shelx"
file_name = reflection_file_utils.construct_output_file_name(
input_file_names=[selected_array.info().source],
user_file_name=co.shelx,
file_type_label="SHELX",
file_extension=file_extension)
if processed_array.is_xray_intensity_array():
data_status = "HKLF 4 (intensity)"
else:
data_status = "HKLF 3 (amplitude)"
print("Writing SHELX", data_status, "file:", file_name)
processed_array.export_as_shelx_hklf(open(file_name, "w"),
full_dynamic_range=True)
n_output_files += 1
print()
if (n_output_files == 0):
command_line.parser.show_help()
print("Please specify at least one output file format,", end=' ')
print("e.g. --mtz, --sca, etc.")
print()
return None
return processed_array
def plot_projections(
projections,
filename=None,
colours=None,
marker_size=3,
font_size=6,
gridsize=None,
label_indices=False,
epochs=None,
colour_map=None,
):
projections_all = projections
# http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear
matplotlib.use("Agg") # use a non-interactive backend
from matplotlib import pylab, pyplot
if epochs is not None and colour_map is not None:
epochs = flex.double(epochs)
epochs -= flex.min(epochs)
epochs /= flex.max(epochs)
cmap = matplotlib.cm.get_cmap(colour_map)
colours = [cmap(e) for e in epochs]
elif colours is None or len(colours) == 0:
colours = ["b"] * len(projections_all)
elif len(colours) < len(projections_all):
colours = colours * len(projections_all)
fig = pyplot.figure()
pyplot.scatter([0], [0], marker="+", c="0.75", s=100)
cir = pylab.Circle((0, 0), radius=1.0, fill=False, color="0.75")
pylab.gca().add_patch(cir)
if gridsize is not None:
x = flex.double()
y = flex.double()
for i, projections in enumerate(projections_all):
x_, y_ = projections.parts()
x.extend(x_)
y.extend(y_)
hb = pyplot.hexbin(x, y, gridsize=gridsize, linewidths=0.2)
pyplot.colorbar(hb)
else:
for i, projections in enumerate(projections_all):
x, y = projections.parts()
pyplot.scatter(
x.as_numpy_array(),
y.as_numpy_array(),
c=colours[i],
s=marker_size,
edgecolors="none",
)
if label_indices:
for j, (hkl, proj) in enumerate(zip(label_indices,
projections)):
# hack to not write two labels on top of each other
p1, p2 = (projections - proj).parts()
if (flex.sqrt(flex.pow2(p1) + flex.pow2(p2)) <
1e-3).iselection()[0] != j:
continue
pyplot.text(proj[0], proj[1], str(hkl), fontsize=font_size)
fig.axes[0].set_aspect("equal")
pyplot.xlim(-1.1, 1.1)
pyplot.ylim(-1.1, 1.1)
if filename is not None:
pyplot.savefig(filename, dpi=300)
def refine_rotx_roty2(OO,enable_rotational_target=True):
helper = OO.per_frame_helper_factory()
helper.restart()
if enable_rotational_target:
print "Trying least squares minimization of excursions",
from scitbx.lstbx import normal_eqns_solving
iterations = normal_eqns_solving.naive_iterations(
non_linear_ls = helper,
gradient_threshold = 1.E-10)
results = helper.x
print "with %d reflections"%len(OO.parent.indexed_pairs),
print "result %6.2f degrees"%(results[1]*180./math.pi),
print "result %6.2f degrees"%(results[0]*180./math.pi)
if False: # Excursion histogram
print "The input mosaicity is %7.3f deg full width"%OO.parent.inputai.getMosaicity()
# final histogram
if OO.pvr_fix:
final = 360.* helper.fvec_callable_pvr(results)
else:
final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)
rmsdexc = math.sqrt(flex.mean(final*final))
from matplotlib import pyplot as plt
nbins = len(final)//20
n,bins,patches = plt.hist(final,
nbins, normed=0, facecolor="orange", alpha=0.75)
plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg"%rmsdexc)
plt.title("Histogram of cctbx.xfel misorientation")
plt.axis([-0.5,0.5,0,100])
plt.plot([rmsdexc],[18],"b|")
plt.show()
# Determine optimal mosaicity and domain size model (monochromatic)
if OO.pvr_fix:
final = 360.* helper.fvec_callable_pvr(results)
else:
final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)
#Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned
guard_stats = flex.max(final), flex.min(final)
if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS (guard_stats[0] > 2.0 or guard_stats[1] < -2.0):
raise Exception("Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)");
print "The mean excursion is %7.3f degrees"%(flex.mean(final))
two_thetas = helper.last_set_orientation.unit_cell().two_theta(OO.reserve_indices,OO.central_wavelength_ang,deg=True)
dspacings = helper.last_set_orientation.unit_cell().d(OO.reserve_indices)
dspace_sq = dspacings * dspacings
excursion_rad = final * math.pi/ 180.
# First -- try to get a reasonable envelope for the observed excursions.
## minimum of three regions; maximum of 50 measurements in each bin
print "fitting parameters on %d spots"%len(excursion_rad)
n_bins = min(max(3, len(excursion_rad)//25),50)
bin_sz = len(excursion_rad)//n_bins
print "nbins",n_bins,"bin_sz",bin_sz
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in xrange(0,n_bins):
subset = order[x*bin_sz:(x+1)*bin_sz]
two_thetas_env.append( flex.mean(two_thetas.select(subset)) )
dspacings_env.append( flex.mean(dspacings.select(subset)))
excursion_rads_env.append( flex.max( flex.abs( excursion_rad.select(subset))))
# Second -- parameter fit
## solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1./(2*solution[0])
print "Best LSQ fit Scheerer domain size is %9.2f ang"%(
s_ang)
tan_phi_rad = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. * s_ang)
tan_phi_deg = tan_phi_rad * 180./math.pi
k_degrees = solution[1]* 180./math.pi
print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees)
tan_outer_deg = tan_phi_deg + k_degrees
if OO.mosaic_refinement_target=="ML":
from xfel.mono_simulation.max_like import minimizer
print "input", s_ang,2. * solution[1]*180/math.pi
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = math.pow(
helper.last_set_orientation.unit_cell().volume(),
1./3.)*20 # 10-unit cell block size minimum reasonable domain
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]),
Deff = d_estimate)
print "output",1./M.x[0], M.x[1]*180./math.pi
tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi
# bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero)
tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi
if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic:
# add code here to perform polychromatic modeling.
"""
get miller indices DONE
get model-predicted mono-wavelength centroid S1 vectors
back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction
compare with spot centroid measured position
compare with locus of bodypixels
"""
print list(OO.reserve_indices)
print len(OO.reserve_indices), len(two_thetas)
positions = [
OO.ucbp3.simple_forward_calculation_spot_position(
wavelength = OO.central_wavelength_ang,
observation_no = obsno).position
for obsno in xrange(len(OO.parent.indexed_pairs))]
print len(positions)
print positions # model-predicted positions
print len(OO.parent.spots)
print OO.parent.indexed_pairs
print OO.parent.spots
print len(OO.parent.spots)
meas_spots = [OO.parent.spots[pair["spot"]] for pair in OO.parent.indexed_pairs]
# for xspot in meas_spots:
# xspot.ctr_mass_x(),xspot.ctr_mass_y()
# xspot.max_pxl_x()
# xspot.bodypixels
# xspot.ctr_mass_x()
# Do some work to calculate an rmsd
diff_vecs = flex.vec3_double()
for p,xspot in zip(positions, meas_spots):
diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0))
# could use diff_vecs.rms_length()
diff_vecs_sq = diff_vecs.dot(diff_vecs)
mean_diff_vec_sq = flex.mean(diff_vecs_sq)
rmsd = math.sqrt(mean_diff_vec_sq)
print "mean obs-pred diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd)
positions_to_fictitious = [
OO.ucbp3.simple_forward_calculation_spot_position(
wavelength = OO.central_wavelength_ang,
observation_no = obsno).position_to_fictitious
for obsno in xrange(len(OO.parent.indexed_pairs))]
# Do some work to calculate an rmsd
diff_vecs = flex.vec3_double()
for p,xspot in zip(positions_to_fictitious, meas_spots):
diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0))
rmsd = diff_vecs.rms_length()
print "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd)
"""
actually, it might be better if the entire set of experimental observations
is transformed into the ideal detector plane, for the purposes of poly_treatment.
start here. Now it would be good to actually implement probability of observing a body pixel given the model.
We have everything needed right here.
"""
if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B:
# Image plot: obs and predicted positions + bodypixels
from matplotlib import pyplot as plt
plt.plot( [p[0] for p in positions_to_fictitious], [p[1] for p in positions_to_fictitious], "r.")
plt.plot( [xspot.ctr_mass_y() for xspot in meas_spots],
[xspot.ctr_mass_x() for xspot in meas_spots], "g.")
bodypx = []
for xspot in meas_spots:
for body in xspot.bodypixels:
bodypx.append(body)
plt.plot( [b.y for b in bodypx], [b.x for b in bodypx], "b.")
plt.axes().set_aspect("equal")
plt.show()
print "MEAN excursion",flex.mean(final),
if OO.mosaic_refinement_target=="ML":
print "mosaicity deg FW=",M.x[1]*180./math.pi
else:
print
if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.
AD1TF7B_MAXDP = 1.
from matplotlib import pyplot as plt
fig = plt.figure()
plt.plot(two_thetas, final, "bo")
mean = flex.mean(final)
minplot = flex.min(two_thetas)
plt.plot([0,minplot],[mean,mean],"k-")
LR = flex.linear_regression(two_thetas, final)
#LR.show_summary()
model_y = LR.slope()*two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
print helper.last_set_orientation.unit_cell()
#for sdp,tw in zip (dspacings,two_thetas):
#print sdp,tw
if OO.mosaic_refinement_target=="ML":
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
else:
plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r|")
plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r|")
plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r-")
plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r-")
plt.plot(two_thetas, tan_phi_deg, "r.")
plt.plot(two_thetas, -tan_phi_deg, "r.")
plt.plot(two_thetas, tan_outer_deg, "g.")
plt.plot(two_thetas, -tan_outer_deg, "g.")
plt.xlim([0,AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP])
OO.parent.show_figure(plt,fig,"psi")
plt.close()
from xfel.mono_simulation.util import green_curve_area,ewald_proximal_volume
if OO.mosaic_refinement_target=="ML":
OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
OO.parent.inputai.setMosaicity(M.x[1]*180./math.pi) # full width, degrees
OO.parent.ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi)
OO.parent.ML_domain_size_ang = 1./M.x[0]
OO.parent.ewald_proximal_volume = ewald_proximal_volume(
wavelength_ang = OO.central_wavelength_ang,
resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff,
domain_size_ang = 1./M.x[0],
full_mosaicity_rad = M.x[1])
return results, helper.last_set_orientation,1./M.x[0] # full width domain size, angstroms
else:
assert OO.mosaic_refinement_target=="LSQ"
OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg)
OO.parent.inputai.setMosaicity(2*k_degrees) # full width
OO.parent.ML_half_mosaicity_deg = k_degrees
OO.parent.ML_domain_size_ang = s_ang
OO.parent.ewald_proximal_volume = ewald_proximal_volume(
wavelength_ang = OO.central_wavelength_ang,
resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff,
domain_size_ang = s_ang,
full_mosaicity_rad = 2*k_degrees*math.pi/180.)
return results, helper.last_set_orientation,s_ang # full width domain size, angstroms
def exercise_2():
for use_reference in [True, False, None]:
pdb_inp = iotbx.pdb.input(lines=flex.std_string(
pdb_str_2.splitlines()),
source_info=None)
model = manager(model_input=pdb_inp, log=null_out())
grm = model.get_restraints_manager().geometry
xrs2 = model.get_xray_structure()
awl2 = model.get_hierarchy().atoms_with_labels()
pdb_inp3 = iotbx.pdb.input(source_info=None, lines=pdb_str_3)
xrs3 = pdb_inp3.xray_structure_simple()
ph3 = pdb_inp3.construct_hierarchy()
ph3.atoms().reset_i_seq()
awl3 = ph3.atoms_with_labels()
sites_cart_reference = flex.vec3_double()
selection = flex.size_t()
reference_names = ["CG", "CD", "NE", "CZ", "NH1", "NH2"]
for a2, a3 in zip(tuple(awl2), tuple(awl3)):
assert a2.resname == a3.resname
assert a2.name == a3.name
assert a2.i_seq == a3.i_seq
if (a2.resname == "ARG" and a2.name.strip() in reference_names):
selection.append(a2.i_seq)
sites_cart_reference.append(a3.xyz)
assert selection.size() == len(reference_names)
selection_bool = flex.bool(xrs2.scatterers().size(), selection)
if (use_reference):
grm.adopt_reference_coordinate_restraints_in_place(
reference.add_coordinate_restraints(
sites_cart=sites_cart_reference,
selection=selection,
sigma=0.01))
elif (use_reference is None):
grm.adopt_reference_coordinate_restraints_in_place(
reference.add_coordinate_restraints(
sites_cart=sites_cart_reference,
selection=selection,
sigma=0.01))
grm.remove_reference_coordinate_restraints_in_place(
selection=selection)
d1 = flex.mean(
flex.sqrt((xrs2.sites_cart().select(selection) -
xrs3.sites_cart().select(selection)).dot()))
print "distance start (use_reference: %s): %6.4f" % (
str(use_reference), d1)
assert d1 > 4.0
assert approx_equal(
flex.max(
flex.sqrt((xrs2.sites_cart().select(~selection_bool) -
xrs3.sites_cart().select(~selection_bool)).dot())),
0)
from cctbx import geometry_restraints
import mmtbx.refinement.geometry_minimization
import scitbx.lbfgs
grf = geometry_restraints.flags.flags(default=True)
sites_cart = xrs2.sites_cart()
minimized = mmtbx.refinement.geometry_minimization.lbfgs(
sites_cart=sites_cart,
correct_special_position_tolerance=1.0,
geometry_restraints_manager=grm,
sites_cart_selection=flex.bool(sites_cart.size(), selection),
geometry_restraints_flags=grf,
lbfgs_termination_params=scitbx.lbfgs.termination_parameters(
max_iterations=5000))
xrs2.set_sites_cart(sites_cart=sites_cart)
d2 = flex.mean(
flex.sqrt((xrs2.sites_cart().select(selection) -
xrs3.sites_cart().select(selection)).dot()))
print "distance final (use_reference: %s): %6.4f" % (
str(use_reference), d2)
if (use_reference): assert d2 < 0.005, "failed: %f<0.05" % d2
else: assert d2 > 4.0, d2
assert approx_equal(
flex.max(
flex.sqrt((xrs2.sites_cart().select(~selection_bool) -
xrs3.sites_cart().select(~selection_bool)).dot())),
0)
def plot_result(metric, result):
if metric == metrics.CC_HALF:
return plots.cc_half_plot(
result.d_star_sq,
result.y_obs,
cc_half_critical_values=result.critical_values,
cc_half_fit=result.y_fit,
d_min=result.d_min,
)
else:
d = {
metrics.MISIGMA: "Merged <I/σ(I)>",
metrics.ISIGMA: "Unmerged <I/σ(I)>",
metrics.I_MEAN_OVER_SIGMA_MEAN: "<I>/<σ(I)>",
metrics.RMERGE: "R<sub>merge</sub> ",
metrics.COMPLETENESS: "Completeness",
}
d_star_sq_tickvals, d_star_sq_ticktext = plots.d_star_sq_to_d_ticks(
result.d_star_sq, 5)
return {
"data": [
{
"x": list(result.d_star_sq), # d_star_sq
"y": list(result.y_obs),
"type": "scatter",
"name": "y_obs",
},
({
"x": list(result.d_star_sq),
"y": list(result.y_fit),
"type": "scatter",
"name": "y_fit",
"line": {
"color": "rgb(47, 79, 79)"
},
} if result.y_fit else {}),
({
"x": [uctbx.d_as_d_star_sq(result.d_min)] * 2,
"y": [
0,
max(
1,
flex.max(result.y_obs),
flex.max(result.y_fit) if result.y_fit else 0,
),
],
"type":
"scatter",
"name":
f"d_min = {result.d_min:.2f} Å",
"mode":
"lines",
"line": {
"color": "rgb(169, 169, 169)",
"dash": "dot"
},
} if result.d_min else {}),
],
"layout": {
"title": f"{d.get(metric)} vs. resolution",
"xaxis": {
"title": "Resolution (Å)",
"tickvals": d_star_sq_tickvals,
"ticktext": d_star_sq_ticktext,
},
"yaxis": {
"title": d.get(metric),
"rangemode": "tozero"
},
},
}
def __init__(self,
fmodels,
model,
max_number_of_iterations=25,
number_of_macro_cycles=3,
occupancy_max=None,
occupancy_min=None,
log=None,
exclude_hd=False):
self.show(fmodels=fmodels,
log=log,
message="occupancy refinement: start")
fmodels.update_xray_structure(
xray_structure=model.get_xray_structure(), update_f_calc=True)
selections = model.refinement_flags.s_occupancies
# exclude H or D from refinement if requested
if (exclude_hd):
hd_sel = model.get_hd_selection()
tmp_sel = []
for sel in selections:
tmp_sel_ = []
for sel_ in sel:
tmp_sel__ = flex.size_t()
for sel__ in sel_:
if (not hd_sel[sel__]):
tmp_sel__.append(sel__)
if (tmp_sel__.size() > 0):
tmp_sel_.append(tmp_sel__)
if (len(tmp_sel_) > 0):
tmp_sel.append(tmp_sel_)
selections = tmp_sel
#
if (len(selections) > 0):
i_selection = flex.size_t()
for s in selections:
for ss in s:
i_selection.extend(ss)
fmodels.fmodel_xray().xray_structure.scatterers().flags_set_grads(
state=False)
fmodels.fmodel_xray().xray_structure.scatterers(
).flags_set_grad_occupancy(iselection=i_selection)
fmodels.fmodel_xray().xray_structure.adjust_occupancy(
occ_max=occupancy_max,
occ_min=occupancy_min,
selection=i_selection)
xray_structure_dc = fmodels.fmodel_xray().xray_structure.\
deep_copy_scatterers()
par_initial = flex.double()
occupancies = xray_structure_dc.scatterers().extract_occupancies()
constrained_groups_selections = []
group_counter = 0
for sel in selections:
ss = []
for sel_ in sel:
ss.append(group_counter)
group_counter += 1
val = flex.mean(occupancies.select(sel_))
par_initial.append(val)
constrained_groups_selections.append(ss)
minimized = None
for macro_cycle in xrange(number_of_macro_cycles):
if (minimized is not None): par_initial = minimized.par_min
minimized = minimizer(
fmodels=fmodels,
selections=selections,
constrained_groups_selections=constrained_groups_selections,
par_initial=par_initial,
max_number_of_iterations=max_number_of_iterations)
if (minimized is not None): par_initial = minimized.par_min
set_refinable_parameters(
xray_structure=fmodels.fmodel_xray().xray_structure,
parameters=par_initial,
selections=selections,
enforce_positivity=(occupancy_min >= 0))
fmodels.fmodel_xray().xray_structure.adjust_occupancy(
occ_max=occupancy_max,
occ_min=occupancy_min,
selection=i_selection)
xray_structure_final = fmodels.fmodel_xray().xray_structure
model.set_xray_structure(xray_structure_final)
fmodels.update_xray_structure(xray_structure=xray_structure_final,
update_f_calc=True)
refined_occ = xray_structure_final.scatterers().extract_occupancies().\
select(i_selection)
assert flex.min(refined_occ) >= occupancy_min
assert flex.max(refined_occ) <= occupancy_max
self.show(fmodels=fmodels,
log=log,
message="occupancy refinement: end")
def test_1():
tstart = time.time()
fraction_missing = 0.1
d_min = 1.5
# create dummy model
symmetry = crystal.symmetry(unit_cell=(15.67, 25.37, 35.68, 90, 90, 90),
space_group_symbol="P 21 21 21")
structure = xray.structure(crystal_symmetry=symmetry)
for k in xrange(1000):
scatterer = xray.scatterer(
site=((1. + k * abs(math.sin(k))) / 1000.0,
(1. + k * abs(math.cos(k))) / 1000.0, (1. + k) / 1000.0),
u=abs(math.cos(k)) * 100. / (8. * math.pi**2),
occupancy=1.0,
scattering_type="C")
structure.add_scatterer(scatterer)
# partial model
n_keep = int(round(structure.scatterers().size() * (1 - fraction_missing)))
partial_structure = xray.structure(special_position_settings=structure)
partial_structure.add_scatterers(structure.scatterers()[:n_keep])
# fcalc (partial model), fobs (fcalc full model)
f_calc = structure.structure_factors(d_min=d_min,
anomalous_flag=False).f_calc()
f_calc_partial = partial_structure.structure_factors(
d_min=d_min, anomalous_flag=False).f_calc()
f_obs = abs(f_calc)
f_calc = abs(f_calc_partial)
d_star_sq = 1. / flex.pow2(f_obs.d_spacings().data())
assert approx_equal(flex.max(f_calc.data()), 6810.19834824)
assert approx_equal(flex.min(f_calc.data()), 0.019589341727)
assert approx_equal(flex.mean(f_calc.data()), 76.651506629)
assert approx_equal(flex.max(f_obs.data()), 6962.58343229)
assert approx_equal(flex.min(f_obs.data()), 0.00111552904935)
assert approx_equal(flex.mean(f_obs.data()), 74.5148786464)
assert f_obs.size() == f_calc.size()
# define test set reflections
flags = flex.bool(f_calc_partial.indices().size(), False)
k = 0
for i in xrange(f_calc_partial.indices().size()):
k = k + 1
if (k != 10):
flags[i] = False
else:
k = 0
flags[i] = True
assert flags.count(True) == 250
assert flags.count(False) == 2258
assert flags.size() == 2508
# *********************************************************TEST = 1
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=f_obs,
f_calc=f_calc,
free_reflections_per_bin=1000,
flags=flags,
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.data().size() == beta.data().size()
assert alpha.data().size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 0.914152454693)
assert approx_equal(flex.max(alpha.data()), 0.914152454693)
assert approx_equal(flex.min(beta.data()), 818.503411782)
assert approx_equal(flex.max(beta.data()), 818.503411782)
# *********************************************************TEST = 2
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=f_obs,
f_calc=f_calc,
free_reflections_per_bin=50,
flags=flags,
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.data().size() == beta.data().size()
assert alpha.data().size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 0.910350007113)
assert approx_equal(flex.max(alpha.data()), 1.07104387776)
assert approx_equal(flex.min(beta.data()), 21.7374310013)
assert approx_equal(flex.max(beta.data()), 4222.81104745)
# *********************************************************TEST = 3
alpha, beta = maxlik.alpha_beta_calc(f=f_obs,
n_atoms_absent=100,
n_atoms_included=900,
bf_atoms_absent=25.0,
final_error=0.0,
absent_atom_type="C").alpha_beta()
fom = max_lik.fom_and_phase_error(
f_obs=f_obs.data(),
f_model=flex.abs(f_calc.data()),
alpha=alpha.data(),
beta=beta.data(),
epsilons=f_obs.epsilons().data().as_double(),
centric_flags=f_obs.centric_flags().data()).fom()
assert flex.max(fom) <= 1.0
assert flex.min(fom) >= 0.0
assert flex.min(alpha.data()) == flex.max(alpha.data()) == 1.0
assert approx_equal(flex.min(beta.data()), 7.964134920)
assert approx_equal(flex.max(beta.data()), 13695.1589364)
# *********************************************************TEST = 4
xs = crystal.symmetry((3, 4, 5), "P 2 2 2")
mi = flex.miller_index(((1, -2, 3), (0, 0, -4)))
ms = miller.set(xs, mi)
fc = flex.double((1., 2.))
fo = flex.double((1., 2.))
mso = miller.set(xs, mi)
mac = miller.array(ms, fc)
mao = miller.array(ms, fo)
alp = flex.double(2, 0.0)
bet = flex.double(2, 1e+9)
fom = max_lik.fom_and_phase_error(
f_obs=mao.data(),
f_model=mac.data(),
alpha=alp,
beta=bet,
epsilons=mao.epsilons().data().as_double(),
centric_flags=mao.centric_flags().data()).fom()
assert approx_equal(fom, [0.0, 0.0])
alp = flex.double(2, 1.0)
bet = flex.double(2, 1e+9)
fom = max_lik.fom_and_phase_error(
f_obs=mao.data(),
f_model=mac.data(),
alpha=alp,
beta=bet,
epsilons=mao.epsilons().data().as_double(),
centric_flags=mao.centric_flags().data()).fom()
assert approx_equal(fom, [0.0, 0.0])
alp = flex.double(2, 0.0)
bet = flex.double(2, 1e-9)
fom = max_lik.fom_and_phase_error(
f_obs=mao.data(),
f_model=mac.data(),
alpha=alp,
beta=bet,
epsilons=mao.epsilons().data().as_double(),
centric_flags=mao.centric_flags().data()).fom()
assert approx_equal(fom, [0.0, 0.0])
alp = flex.double(2, 1.0)
bet = flex.double(2, 1e-9)
fom = max_lik.fom_and_phase_error(
f_obs=mao.data(),
f_model=mac.data(),
alpha=alp,
beta=bet,
epsilons=mao.epsilons().data().as_double(),
centric_flags=mao.centric_flags().data()).fom()
assert approx_equal(fom, [1.0, 1.0])
def miller_array_export_as_shelx_hklf(miller_array,
file_object=None,
normalise_if_format_overflow=False,
full_dynamic_range=False,
scale_range=None):
"""\
For the full_dynamic_range option, normalise data outside the range that
fits into 8.6g format, regardless of settings for normalise_if_format_overflow
or scale_range
Otherwise, if the maximum data value does not fit into the f8.2/f8.0 format:
normalise_if_format_overflow=False: RuntimeError is thrown
normalise_if_format_overflow=True: data is normalised to the largest
number to fit f8.2/f8.0 format or within specified scale_range
"""
assert miller_array.is_real_array()
if (file_object is None): file_object = sys.stdout
def raise_f8_overflow(v):
raise RuntimeError("SHELX HKL file F8.2/F8.0 format overflow: %.8g" %
v)
data = miller_array.data()
sigmas = miller_array.sigmas()
assert data is not None
min_val = flex.min(data)
max_val = flex.max(data)
if (sigmas is not None):
min_val = min(min_val, flex.min(sigmas))
max_val = max(max_val, flex.max(sigmas))
min_sc = 1
max_sc = 1
scale = 1
if full_dynamic_range:
max_abs = 999999.
max_val = max(abs(max_val), abs(min_val))
if (max_val > max_abs):
scale = max_abs / max_val
else:
if scale_range is None:
scale_range = (-999999., 9999999.)
if (min_val < scale_range[0]):
if not normalise_if_format_overflow:
raise_f8_overflow(min_val)
min_sc = scale_range[0] / min_val
if (max_val > scale_range[1]):
if (not normalise_if_format_overflow):
raise_f8_overflow(max_val)
max_sc = scale_range[1] / max_val
scale = min(min_sc, max_sc)
sigmas = miller_array.sigmas()
s = 0.01
for i, h in enumerate(miller_array.indices()):
if (sigmas is not None): s = sigmas[i]
def fmt_3i4(h):
result = "%4d%4d%4d" % h
if (len(result) != 12):
raise RuntimeError("SHELXL HKL file 3I4 format overflow: %s" %
result)
return result
def fmt_f8(v):
result = "%8.2f" % v
if (len(result) != 8):
result = "%8.1f" % v
if (len(result) != 8):
result = "%7d." % round(v)
assert len(result) == 8
return result
def fmt_fullrange_data(v):
if (abs(v) >= 1.):
result = "%8.6g" % v
else:
result = "%8.5f" % v
return result
def fmt_fullrange_sigma(v):
if (abs(v) >= 1.):
result = "%8.6g" % v
elif (abs(v) < 0.00001):
result = "%8.5f" % 0.00001
else:
result = "%8.5f" % v
return result
if (full_dynamic_range):
line = fmt_3i4(h) + fmt_fullrange_data(
data[i] * scale) + fmt_fullrange_sigma(s * scale)
else:
line = fmt_3i4(h) + fmt_f8(data[i] * scale) + fmt_f8(s * scale)
print(line, file=file_object)
print(" 0 0 0 0.00 0.00", file=file_object)
def keywise_printout(data):
for key in data:
if key == 'ACTIVE_AREAS':
print int(len(data[key]) / 4), "active areas, first one: ", list(
data[key][0:4])
elif key == 'observations':
print key, data[key], "Showing unit cell/spacegroup:"
obs = data[key][0]
uc = obs.unit_cell()
uc.show_parameters()
obs.space_group().info().show_summary()
d = uc.d(obs.indices())
print "Number of observations:", len(obs.indices())
print "Max resolution: %f" % flex.min(d)
print "Mean I/sigma:", flex.mean(obs.data()) / flex.mean(
obs.sigmas())
print "I/sigma > 1 count:", (obs.data() / obs.sigmas() >
1).count(True)
print "I <= 0:", len(obs.data().select(obs.data() <= 0))
from cctbx.crystal import symmetry
sym = symmetry(unit_cell=uc, space_group=obs.space_group())
mset = sym.miller_set(indices=obs.indices(), anomalous_flag=False)
binner = mset.setup_binner(n_bins=20)
acceptable_resolution_bins = []
binned_avg_i_sigi = []
for i in binner.range_used():
d_max, d_min = binner.bin_d_range(i)
sel = (d <= d_max) & (d > d_min)
sel &= obs.data() > 0
intensities = obs.data().select(sel)
sigmas = obs.sigmas().select(sel)
n_refls = len(intensities)
avg_i = flex.mean(intensities) if n_refls > 0 else 0
avg_i_sigi = flex.mean(intensities /
sigmas) if n_refls > 0 else 0
acceptable_resolution_bins.append(avg_i_sigi >= 1.0)
acceptable_resolution_bins = [
acceptable_resolution_bins[i]
if False not in acceptable_resolution_bins[:i + 1] else False
for i in range(len(acceptable_resolution_bins))
]
best_res = None
for i, ok in zip(binner.range_used(), acceptable_resolution_bins):
d_max, d_min = binner.bin_d_range(i)
if ok:
best_res = d_min
else:
break
if best_res is None:
print "Highest resolution with I/sigI >= 1.0: None"
else:
print "Highest resolution with I/sigI >= 1.0: %f" % d_min
elif key == 'mapped_predictions':
print key, data[key][0][0], "(only first shown of %d)" % len(
data[key][0])
elif key == 'correction_vectors' and data[key] is not None and data[
key][0] is not None:
if data[key][0] is None:
print key, "None"
else:
print key, data[key][0][0], "(only first shown)"
elif key == "DATA":
print key, "len=%d max=%f min=%f dimensions=%s" % (
data[key].size(), flex.max(data[key]), flex.min(
data[key]), str(data[key].focus()))
elif key == "WAVELENGTH":
print "WAVELENGTH", data[
key], ", converted to eV:", 12398.4187 / data[key]
elif key == "fuller_kapton_absorption_correction":
print key, data[key]
if doplots:
c = data[key][0]
hist = flex.histogram(c, n_slots=30)
from matplotlib import pyplot as plt
plt.scatter(hist.slot_centers(), hist.slots())
plt.show()
obs = data['observations'][0]
preds = data['mapped_predictions'][0]
p1 = preds.select(c == 1.0)
p2 = preds.select((c != 1.0) & (c <= 1.5))
plt.scatter(preds.parts()[1], preds.parts()[0], c='g')
plt.scatter(p1.parts()[1], p1.parts()[0], c='b')
plt.scatter(p2.parts()[1], p2.parts()[0], c='r')
plt.show()
else:
print key, data[key]
def get_uc_consensus(experiments_list,
show_plot=False,
return_only_first_indexed_model=False,
finalize_method=None,
clustering_params=None):
'''
Uses the Rodriguez Laio 2014 method to do a clustering of the unit cells and then vote for the highest
consensus unit cell. Input needs to be a list of experiments object.
Clustering code taken from github.com/cctbx-xfel/cluster_regression
Returns an experiment object with crystal unit cell from the cluster with the most points
'''
if return_only_first_indexed_model:
return [experiments_list[0].crystals()[0]], None
cells = []
from xfel.clustering.singleframe import CellOnlyFrame
save_plot = False
# Flag for testing Lysozyme data from NKS.Make sure cluster_regression repository is present and configured
# Program will exit after plots are displayed if this flag is true
test_nks = False
if test_nks:
from cctbx import crystal
import libtbx.load_env
cluster_regression = libtbx.env.find_in_repositories(
relative_path="cluster_regression", test=os.path.isdir)
file_name = os.path.join(cluster_regression, 'examples',
'lysozyme1341.txt')
for line in open(file_name, "r").xreadlines():
tokens = line.strip().split()
unit_cell = tuple(float(x) for x in tokens[0:6])
space_group_symbol = tokens[6]
crystal_symmetry = crystal.symmetry(
unit_cell=unit_cell, space_group_symbol=space_group_symbol)
cells.append(CellOnlyFrame(crystal_symmetry))
else:
for experiment in experiments_list:
if len(experiment.crystals()) > 1:
print('IOTA:Should have only one crystal model')
crystal_symmetry = experiment.crystals()[0].get_crystal_symmetry()
cells.append(CellOnlyFrame(crystal_symmetry))
MM = [c.mm for c in cells] # metrical matrices
MM_double = flex.double()
for i in range(len(MM)):
Tup = MM[i]
for j in range(6):
MM_double.append(Tup[j])
print('There are %d cells' % len(MM))
coord_x = flex.double([c.uc[0] for c in cells])
coord_y = flex.double([c.uc[1] for c in cells])
if show_plot or save_plot:
import matplotlib
if not show_plot:
matplotlib.use('Agg')
import matplotlib.pyplot as plt
#from IPython import embed; embed(); exit()
plt.plot([c.uc[0] for c in cells], [c.uc[1] for c in cells],
"k.",
markersize=3.)
plt.axes().set_aspect("equal")
if save_plot:
plot_name = 'uc_cluster.png'
plt.savefig(plot_name,
size_inches=(10, 10),
dpi=300,
bbox_inches='tight')
if show_plot:
plt.show()
print('Now constructing a Dij matrix: Starting Unit Cell clustering')
NN = len(MM)
from cctbx.uctbx.determine_unit_cell import NCDist_flatten
Dij = NCDist_flatten(MM_double)
d_c = flex.mean_and_variance(
Dij.as_1d()).unweighted_sample_standard_deviation() #6.13
#FIXME should be a PHIL param
if len(cells) < 5:
return [experiments_list[0].crystals()[0]], None
CM = clustering_manager(Dij=Dij, d_c=d_c, max_percentile_rho=0.95)
n_cluster = 1 + flex.max(CM.cluster_id_final)
print(len(cells), ' datapoints have been analyzed')
print('%d CLUSTERS' % n_cluster)
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
print('Cluster %d central Unit cell = %d' % (i, item))
cells[item].crystal_symmetry.show_summary()
# More plots for debugging
appcolors = [
'b', 'r', '#ff7f0e', '#2ca02c', '#9467bd', '#8c564b', '#e377c2',
'#7f7f7f', '#bcbd22', '#17becf'
]
if show_plot:
# Decision graph
import matplotlib.pyplot as plt
plt.plot(CM.rho, CM.delta, "r.", markersize=3.)
for x in range(NN):
if CM.cluster_id_maxima[x] >= 0:
plt.plot([CM.rho[x]], [CM.delta[x]], "ro")
plt.show()
if show_plot:
import matplotlib.pyplot as plt
colors = [appcolors[i % 10] for i in CM.cluster_id_full]
plt.scatter(coord_x,
coord_y,
marker='o',
color=colors,
linewidth=0.4,
edgecolor='k')
for i in range(n_cluster):
item = flex.first_index(CM.cluster_id_maxima, i)
plt.plot([cells[item].uc[0]], cells[item].uc[1], 'y.')
plt.axes().set_aspect("equal")
plt.show()
if test_nks:
exit()
# Now look at each unit cell cluster for orientational clustering
# idea is to cluster the orientational component in each of the unit cell clusters
#
do_orientational_clustering = not return_only_first_indexed_model # temporary.
dxtbx_crystal_models = []
if do_orientational_clustering:
print('IOTA: Starting orientational clustering')
Dij_ori = {} # dictionary to store Dij for each cluster
uc_experiments_list = {
} # dictionary to store experiments_lists for each cluster
from collections import Counter
uc_cluster_count = Counter(list(CM.cluster_id_final))
# instantiate the Dij_ori flat 1-d array
# Put all experiments list from same uc cluster together
if True:
from scitbx.matrix import sqr
from cctbx_orientation_ext import crystal_orientation
#crystal_orientation_list = []
#for i in range(len(experiments_list)):
# crystal_orientation_list.append(crystal_orientation(experiments_list[i].crystals()[0].get_A(), True))
#from IPython import embed; embed(); exit()
#A_direct = sqr(crystal_orientation_list[i].reciprocal_matrix()).transpose().inverse()
#print ("Direct A matrix 1st element = %12.6f"%A_direct[0])
for i in range(len(experiments_list)):
if CM.cluster_id_full[i] not in uc_experiments_list:
uc_experiments_list[CM.cluster_id_full[i]] = []
uc_experiments_list[CM.cluster_id_full[i]].append(
experiments_list[i])
for cluster in uc_cluster_count:
# Make sure there are atleast a minimum number of samples in the cluster
if uc_cluster_count[cluster] < 5:
continue
Dij_ori[cluster] = flex.double(
[[0.0] * uc_cluster_count[cluster]] *
uc_cluster_count[cluster])
# Now populate the Dij_ori array
N_samples_in_cluster = len(uc_experiments_list[cluster])
for i in range(N_samples_in_cluster - 1):
for j in range(i + 1, N_samples_in_cluster):
dij_ori = get_dij_ori(
uc_experiments_list[cluster][i].crystals()[0],
uc_experiments_list[cluster][j].crystals()[0])
Dij_ori[cluster][N_samples_in_cluster * i + j] = dij_ori
Dij_ori[cluster][N_samples_in_cluster * j + i] = dij_ori
# Now do the orientational cluster analysis
#from IPython import embed; embed(); exit()
d_c_ori = 0.13
from exafel_project.ADSE13_25.clustering.plot_with_dimensional_embedding import plot_with_dimensional_embedding
#plot_with_dimensional_embedding(1-Dij_ori[1]/flex.max(Dij_ori[1]), show_plot=True)
for cluster in Dij_ori:
d_c_ori = flex.mean_and_variance(Dij_ori[cluster].as_1d(
)).unweighted_sample_standard_deviation()
CM_ori = clustering_manager(Dij=Dij_ori[cluster],
d_c=d_c_ori,
max_percentile_rho=0.85)
n_cluster_ori = 1 + flex.max(CM_ori.cluster_id_final)
#from IPython import embed; embed()
#FIXME should be a PHIL param
for i in range(n_cluster_ori):
if len([zz for zz in CM_ori.cluster_id_final if zz == i]) < 5:
continue
item = flex.first_index(CM_ori.cluster_id_maxima, i)
dxtbx_crystal_model = uc_experiments_list[cluster][
item].crystals()[0]
dxtbx_crystal_models.append(dxtbx_crystal_model)
from scitbx.matrix import sqr
from cctbx_orientation_ext import crystal_orientation
crystal_orientation = crystal_orientation(
dxtbx_crystal_model.get_A(), True)
A_direct = sqr(crystal_orientation.reciprocal_matrix()
).transpose().inverse()
print(
"IOTA: Direct A matrix 1st element of orientational cluster %d = %12.6f"
% (i, A_direct[0]))
if show_plot:
# Decision graph
stretch_plot_factor = 1.05 # (1+fraction of limits by which xlim,ylim should be set)
import matplotlib.pyplot as plt
plt.plot(CM_ori.rho, CM_ori.delta, "r.", markersize=3.)
for x in range(len(list(CM_ori.cluster_id_final))):
if CM_ori.cluster_id_maxima[x] >= 0:
plt.plot([CM_ori.rho[x]], [CM_ori.delta[x]], "ro")
#from IPython import embed; embed(); exit()
plt.xlim([-10, stretch_plot_factor * flex.max(CM_ori.rho)])
plt.ylim([-10, stretch_plot_factor * flex.max(CM_ori.delta)])
plt.show()
# Make sure the crystal models are not too close to each other
# FIXME should be a PHIL
min_angle = 5.0 # taken from indexer.py
close_models_list = []
if len(dxtbx_crystal_models) > 1:
from dials.algorithms.indexing.compare_orientation_matrices import difference_rotation_matrix_axis_angle
for i_a in range(0, len(dxtbx_crystal_models) - 1):
for i_b in range(i_a, len(dxtbx_crystal_models)):
cryst_a = dxtbx_crystal_models[i_a]
cryst_b = dxtbx_crystal_models[i_b]
R_ab, axis, angle, cb_op_ab = difference_rotation_matrix_axis_angle(
cryst_a, cryst_b)
# FIXME
if abs(angle) < min_angle: # degrees
close_models_list.append((i_a, i_b))
# Now prune the dxtbx_crystal_models list
for close_models in close_models_list:
i_a, i_b = close_models
if dxtbx_crystal_models[i_a] is not None and dxtbx_crystal_models[
i_b] is not None:
dxtbx_crystal_models[i_a] = None
dxtbx_crystal_models = [x for x in dxtbx_crystal_models if x is not None]
if len(dxtbx_crystal_models) > 0:
return dxtbx_crystal_models, None
else:
# If nothing works, atleast return the 1st crystal model that was found
return [experiments_list[0].crystals()[0]], None
def exercise(space_group_info,
anomalous_flag,
use_u_aniso,
n_elements=3,
d_min=3.,
verbose=0):
structure_z = random_structure.xray_structure(
space_group_info,
elements=("Se", ) * n_elements,
volume_per_atom=200,
random_f_prime_d_min=1.0,
random_f_double_prime=anomalous_flag,
random_u_iso=True,
use_u_aniso=use_u_aniso,
random_occupancy=True)
check_weight_without_occupancy(structure_z)
f_z = structure_z.structure_factors(anomalous_flag=anomalous_flag,
d_min=d_min,
algorithm="direct").f_calc()
f_abs_z = abs(f_z)
f_rad_z = f_z.phases()
f_deg_z = f_z.phases(deg=True)
hl_z = generate_random_hl(miller_set=f_z)
hl_z_rad = hl_z.phase_integrals()
if (0 or verbose):
structure_z.show_summary().show_scatterers()
print "n_special_positions:", \
structure_z.special_position_indices().size()
z2p_op = structure_z.space_group().z2p_op()
z2p_op = sgtbx.change_of_basis_op(z2p_op.c() + sgtbx.tr_vec(
(2, -1, 3), 12).new_denominator(z2p_op.c().t().den()))
for change_hand in [False, True]:
if (change_hand):
z2p_op = z2p_op * sgtbx.change_of_basis_op("-x,-y,-z")
structure_p = structure_z.change_basis(z2p_op)
check_weight_without_occupancy(structure_p)
check_site_symmetry_table(structure_z, z2p_op, structure_p)
if (0 or verbose):
structure_p.show_summary().show_scatterers()
print "n_special_positions:", \
structure_p.special_position_indices().size()
assert tuple(structure_p.special_position_indices()) \
== tuple(structure_z.special_position_indices())
structure_pz = structure_p.change_basis(z2p_op.inverse())
check_weight_without_occupancy(structure_pz)
check_site_symmetry_table(structure_p, z2p_op.inverse(), structure_pz)
assert structure_pz.unit_cell().is_similar_to(structure_z.unit_cell())
assert structure_pz.space_group() == structure_z.space_group()
f_pz = f_z.structure_factors_from_scatterers(
xray_structure=structure_pz, algorithm="direct").f_calc()
f_abs_pz = abs(f_pz)
f_rad_pz = f_pz.phases()
f_deg_pz = f_pz.phases(deg=True)
c = flex.linear_correlation(f_abs_z.data(), f_abs_pz.data())
assert c.is_well_defined()
if (0 or verbose):
print "correlation:", c.coefficient()
assert c.coefficient() > 0.999
f_p_cb = f_z.change_basis(z2p_op)
f_abs_p_cb = f_abs_z.change_basis(z2p_op)
f_rad_p_cb = f_rad_z.change_basis(z2p_op, deg=False)
f_deg_p_cb = f_deg_z.change_basis(z2p_op, deg=True)
hl_p_cb = hl_z.change_basis(z2p_op)
hl_p_cb_rad = hl_p_cb.phase_integrals()
assert approx_equal(
hl_z_rad.change_basis(z2p_op).data(), hl_p_cb_rad.data())
assert f_abs_p_cb.indices().all_eq(f_p_cb.indices())
if (not change_hand):
o = flex.order(f_abs_p_cb.indices(), f_abs_z.indices())
else:
o = flex.order(-f_abs_p_cb.indices(), f_abs_z.indices())
if (f_abs_z.space_group().n_ltr() == 1):
assert o == 0
else:
assert o != 0
f_pz = f_p_cb.change_basis(z2p_op.inverse())
f_abs_pz = f_abs_p_cb.change_basis(z2p_op.inverse())
f_rad_pz = f_rad_p_cb.change_basis(z2p_op.inverse(), deg=False)
f_deg_pz = f_deg_p_cb.change_basis(z2p_op.inverse(), deg=True)
hl_pz = hl_p_cb.change_basis(z2p_op.inverse())
hl_pz_rad = hl_pz.phase_integrals()
assert approx_equal(hl_z_rad.data(), hl_pz_rad.data())
for i, o in zip(hl_z.data(), hl_pz.data()):
assert approx_equal(i, o)
assert approx_equal(flex.max(flex.abs(f_pz.data() - f_z.data())), 0)
assert flex.order(f_abs_pz.indices(), f_abs_z.indices()) == 0
assert f_abs_pz.indices().all_eq(f_pz.indices())
assert approx_equal(
flex.max(
scitbx.math.phase_error(phi1=f_rad_pz.data(),
phi2=f_rad_z.data())), 0)
assert approx_equal(
flex.max(
scitbx.math.phase_error(phi1=f_deg_pz.data(),
phi2=f_deg_z.data(),
deg=True)), 0)
assert approx_equal(f_deg_pz.data(), f_rad_pz.data() * (180 / math.pi))
f_p_sf = f_p_cb.structure_factors_from_scatterers(
xray_structure=structure_p, algorithm="direct").f_calc()
det = abs(z2p_op.c().r().determinant())
assert approx_equal(
flex.max(flex.abs(f_p_sf.data() * complex(det) - f_p_cb.data())),
0)
f_abs_p_sf = abs(f_p_sf)
f_rad_p_sf = f_p_sf.phases()
f_deg_p_sf = f_p_sf.phases(deg=True)
assert approx_equal(
flex.max(
scitbx.math.phase_error(phi1=f_rad_p_sf.data(),
phi2=f_rad_p_cb.data())), 0)
assert approx_equal(
flex.max(
scitbx.math.phase_error(phi1=f_deg_p_sf.data(),
phi2=f_deg_p_cb.data(),
deg=True)), 0)
c = flex.linear_correlation(f_abs_p_sf.data(), f_abs_p_cb.data())
assert c.is_well_defined()
if (0 or verbose):
print "correlation:", c.coefficient()
assert c.coefficient() > 0.999