def weighted_correlation_coefficient(x, y, w):
# may be computationally unstable?
if isinstance(x, numpy.ndarray):
assert isinstance(y, numpy.ndarray)
assert isinstance(w, numpy.ndarray)
m_x = numpy.average(x, weights=w)
m_y = numpy.average(y, weights=w)
# 1/sum_w is omitted
cov = numpy.sum(w * (x - m_x) * (y - m_y))
var_x = numpy.sum(w * (x - m_x)**2)
var_y = numpy.sum(w * (y - m_y)**2)
return cov / numpy.sqrt(var_x) / numpy.sqrt(var_y)
elif isinstance(x, flex.double):
assert isinstance(y, flex.double)
assert isinstance(w, flex.double)
sum_w = flex.sum(w)
m_x = flex.sum(w * x) / sum_w
m_y = flex.sum(w * y) / sum_w
cov = flex.sum(w * (x - m_x) * (y - m_y))
var_x = flex.sum(w * flex.pow2(x - m_x))
var_y = flex.sum(w * flex.pow2(y - m_y))
return cov / math.sqrt(var_x) / math.sqrt(var_y)
else:
return weighted_correlation_coefficient(numpy.array(x), numpy.array(y),
numpy.array(w))
def exercise_py_LS(obs, f_calc, weighting, verbose):
weighting.computing_derivatives_wrt_f_c = True
r = xray.unified_least_squares_residual(obs, weighting=weighting)
rt = r(f_calc, compute_derivatives=True)
if obs.is_xray_amplitude_array():
assert(isinstance(rt, xray.targets_least_squares_residual))
elif obs.is_xray_intensity_array():
assert(isinstance(rt, xray.targets_least_squares_residual_for_intensity))
scale_factor = rt.scale_factor()
gr_ana = rt.derivatives()
K = scale_factor
w = weighting.weights
if w is not None: w = w.deep_copy()
dw_dfc = weighting.derivatives_wrt_f_c
if dw_dfc is not None: dw_dfc = dw_dfc.deep_copy()
y_o = obs.data()
if w is None: w = flex.double(obs.size(), 1)
sum_w_y_o_sqr = flex.sum(w * y_o * y_o)
f_c = f_calc.data().deep_copy()
if obs.is_xray_amplitude_array():
y_c = flex.abs(f_c)
der = f_c * (1/y_c)
elif obs.is_xray_intensity_array():
y_c = flex.norm(f_c)
der = 2 * f_c
gr_explicit = w*2*K*(K*y_c - y_o) * der / sum_w_y_o_sqr
sum_w_squares = flex.sum(w*flex.pow2(K*y_c - y_o))
assert approx_equal(gr_ana, gr_explicit)
gr_fin = flex.complex_double()
eps = 1.e-6
for i_refl in xrange(obs.size()):
gc = []
for i_part in [0,1]:
fc0 = f_calc.data()[i_refl]
ts = []
for signed_eps in [eps,-eps]:
if (i_part == 0):
f_calc.data()[i_refl] = complex(fc0.real + signed_eps, fc0.imag)
else:
f_calc.data()[i_refl] = complex(fc0.real, fc0.imag + signed_eps)
rt = r(f_calc, compute_derivatives=False, scale_factor=scale_factor)
ts.append(rt.target())
f_calc.data()[i_refl] = fc0
gc.append((ts[0]-ts[1])/(2*eps))
gr_fin.append(complex(*gc))
if (verbose):
print "ana:", list(gr_ana)
print "fin:", list(gr_fin)
if dw_dfc is None:
assert approx_equal(gr_fin, gr_ana)
else:
gr_total_ana = ( gr_ana
+ dw_dfc*(flex.pow2(K*y_c - y_o)/sum_w_y_o_sqr
- sum_w_squares*flex.pow2(y_o)/sum_w_y_o_sqr**2) )
assert approx_equal(gr_fin, gr_total_ana)
def get_rmsds_obs_pred(self, observations, experiment):
reflections = observations.select(observations.get_flags(
observations.flags.used_in_refinement))
assert len(reflections) > 0
obs_x, obs_y, obs_z = reflections['xyzobs.mm.value'].parts()
calc_x, calc_y, calc_z = reflections['xyzcal.mm'].parts()
rmsd_x = flex.mean(flex.pow2(obs_x-calc_x))**0.5
rmsd_y = flex.mean(flex.pow2(obs_y-calc_y))**0.5
rmsd_z = flex.mean(flex.pow2(obs_z-calc_z))**0.5
return (rmsd_x, rmsd_y, rmsd_z)
def get_rmsds_obs_pred(self, observations, experiment):
reflections = observations.select(
observations.get_flags(observations.flags.used_in_refinement))
assert len(reflections) > 0
obs_x, obs_y, obs_z = reflections['xyzobs.mm.value'].parts()
calc_x, calc_y, calc_z = reflections['xyzcal.mm'].parts()
rmsd_x = flex.mean(flex.pow2(obs_x - calc_x))**0.5
rmsd_y = flex.mean(flex.pow2(obs_y - calc_y))**0.5
rmsd_z = flex.mean(flex.pow2(obs_z - calc_z))**0.5
return (rmsd_x, rmsd_y, rmsd_z)
def calc_w(wa, wb, i_obs, i_sig, i_calc, k):
assert i_sig.size() == i_obs.size()
assert i_calc.size() == i_obs.size()
ik = i_obs / k**2
sk = i_sig / k**2
ik.set_selected(ik < 0, 0)
p = (ik + 2 * i_calc) / 3
den = flex.pow2(sk) + flex.pow2(wa * p) + wb * p
assert den.all_gt(1e-8)
weights = 1 / den
return weights
def plot_projections(projections,
filename=None,
show=None,
colours=None,
marker_size=3,
font_size=6,
label_indices=False):
assert [filename, show].count(None) < 2
projections_all = projections
try:
import matplotlib
if not show:
# http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear
matplotlib.use('Agg') # use a non-interactive backend
from matplotlib import pyplot
from matplotlib import pylab
except ImportError:
raise Sorry("matplotlib must be installed to generate a plot.")
if colours is None or len(colours) == 0:
colours = ['b'] * len(projections_all)
elif len(colours) < len(projections_all):
colours = colours * len(projections_all)
fig = pyplot.figure()
pyplot.scatter([0], [0], marker='+', c='0.75', s=100)
cir = pylab.Circle((0, 0), radius=1.0, fill=False, color='0.75')
pylab.gca().add_patch(cir)
for i, projections in enumerate(projections_all):
x, y = projections.parts()
pyplot.scatter(x.as_numpy_array(),
y.as_numpy_array(),
c=colours[i],
s=marker_size,
edgecolors='none')
if label_indices:
for j, (hkl, proj) in enumerate(zip(miller_indices, projections)):
# hack to not write two labels on top of each other
p1, p2 = (projections - proj).parts()
if (flex.sqrt(flex.pow2(p1) + flex.pow2(p2)) <
1e-3).iselection()[0] != j:
continue
pyplot.text(proj[0], proj[1], str(hkl), fontsize=font_size)
pyplot.axes().set_aspect('equal')
pyplot.xlim(-1.1, 1.1)
pyplot.ylim(-1.1, 1.1)
if filename is not None:
pyplot.savefig(filename, size_inches=(24, 18), dpi=300)
if show:
pyplot.show()
def calc_w(wa, wb, i_obs, i_sig, i_calc, k): assert i_sig.size() == i_obs.size() assert i_calc.size() == i_obs.size() ik = i_obs / k**2 sk = i_sig / k**2 ik.set_selected(ik < 0, 0) p = (ik + 2 * i_calc) / 3 den = flex.pow2(sk) + flex.pow2(wa*p) + wb*p assert den.all_gt(1e-8) weights = 1 / den return weights
def plot_projections(projections, filename=None, show=None,
colours=None, marker_size=3, font_size=6,
label_indices=False):
assert [filename, show].count(None) < 2
projections_all = projections
try:
import matplotlib
if not show:
# http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear
matplotlib.use('Agg') # use a non-interactive backend
from matplotlib import pyplot
from matplotlib import pylab
except ImportError:
raise Sorry("matplotlib must be installed to generate a plot.")
if colours is None or len(colours) == 0:
colours = ['b'] * len(projections_all)
elif len(colours) < len(projections_all):
colours = colours * len(projections_all)
fig = pyplot.figure()
pyplot.scatter([0], [0], marker='+', c='0.75', s=100)
cir = pylab.Circle((0,0), radius=1.0, fill=False, color='0.75')
pylab.gca().add_patch(cir)
for i, projections in enumerate(projections_all):
x, y = projections.parts()
pyplot.scatter(x.as_numpy_array(), y.as_numpy_array(),
c=colours[i], s=marker_size, edgecolors='none')
if label_indices:
for j, (hkl, proj) in enumerate(zip(miller_indices, projections)):
# hack to not write two labels on top of each other
p1, p2 = (projections - proj).parts()
if (flex.sqrt(flex.pow2(p1)+flex.pow2(p2)) < 1e-3).iselection()[0] != j:
continue
pyplot.text(proj[0], proj[1], str(hkl), fontsize=font_size)
pyplot.axes().set_aspect('equal')
pyplot.xlim(-1.1,1.1)
pyplot.ylim(-1.1,1.1)
if filename is not None:
pyplot.savefig(filename,
size_inches=(24,18),
dpi=300)
if show:
pyplot.show()
def log_p_obs_given_gamma(self, gamma):
dof = self.degrees_of_freedom
x_gamma = (gamma * self.delta_fc2.data() - self.delta_fo2.data()) \
/ self.delta_fo2.sigmas()
if self.probability_plot_slope is not None:
x_gamma /= self.probability_plot_slope
return -(1+dof)/2 * flex.sum(flex.log(flex.pow2(x_gamma) + dof))
def f_obs_and_f_calc_agree_well(O, co):
if (O.c_obs.indices().size() == 0): return False
from cctbx.array_family import flex
f_obs = O.c_obs.as_amplitude_array(algorithm="xtal_3_7")
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=O.xray_structure).f_calc().amplitudes()
fan_out_sel = f_obs.f_obs_f_calc_fan_outlier_selection(
f_calc=f_calc,
offset_low=co.fan_offset_low,
offset_high=co.fan_offset_high,
also_return_x_and_y=True)
if (fan_out_sel is None):
return False
if (co.i_obs_i_calc_plot and f_obs.indices().size() != 0):
from libtbx import pyplot
xs = O.c_obs.as_intensity_array(algorithm="simple").data()
ys = flex.pow2(f_calc.data())
pyplot.plot(xs.as_numpy_array(), ys.as_numpy_array(), "ro")
pyplot.show()
fan_out_sel, x, y = fan_out_sel
fan_in_sel = ~fan_out_sel
if (co.f_obs_f_calc_plot):
from libtbx import pyplot
xs = x.select(fan_out_sel)
ys = y.select(fan_out_sel)
if (xs.size() == 0):
pyplot.plot(x.as_numpy_array(), y.as_numpy_array(), "bo")
else:
pyplot.plot(xs.as_numpy_array(), ys.as_numpy_array(), "ro")
xs = x.select(fan_in_sel)
ys = y.select(fan_in_sel)
if (xs.size() != 0):
pyplot.plot(xs.as_numpy_array(), ys.as_numpy_array(), "bo")
pyplot.plot_pairs([(co.fan_offset_low, 0),
(1, 1 - co.fan_offset_high)], "r-")
pyplot.plot_pairs([(0, co.fan_offset_low),
(1 - co.fan_offset_high, 1)], "r-")
pyplot.plot_pairs([(0, 0), (1, 1)], "k--")
pyplot.show()
fan_outlier_fraction = fan_out_sel.count(True) / fan_out_sel.size()
def cc_r1(fo, fc):
lc = flex.linear_correlation(fo.data(), fc.data())
assert lc.is_well_defined()
cc = lc.coefficient()
from libtbx import Auto
r1 = f_obs.r1_factor(other=f_calc, scale_factor=Auto)
return cc, r1
cc_all, r1_all = cc_r1(f_obs, f_calc)
cc_in, r1_in = cc_r1(f_obs.select(fan_in_sel),
f_calc.select(fan_in_sel))
print "f_obs_f_calc %s" % O.cod_id, \
"| cc_all %.4f | r1_all %.4f | out %.4f | cc_in %.4f | r1_in %.4f |" % (
cc_all, r1_all, fan_outlier_fraction, cc_in, r1_in)
if (fan_outlier_fraction > co.max_fan_outlier_fraction):
return False
if (cc_all < co.min_f_obs_f_calc_correlation):
return False
return True
def direct_space_squaring(start, selection_fixed):
map_gridding = miller.index_span(
miller.set.expand_to_p1(start).indices()).map_grid()
if (selection_fixed is None):
fixed = start
var = start
else:
fixed = start.select(selection_fixed)
var = start.select(~selection_fixed)
rfft = fftpack.real_to_complex_3d([n * 3 // 2 for n in map_gridding])
conjugate_flag = True
structure_factor_map = maptbx.structure_factors.to_map(
space_group=fixed.space_group(),
anomalous_flag=fixed.anomalous_flag(),
miller_indices=fixed.indices(),
structure_factors=fixed.data(),
n_real=rfft.n_real(),
map_grid=flex.grid(rfft.n_complex()),
conjugate_flag=conjugate_flag)
real_map = rfft.backward(structure_factor_map.complex_map())
squared_map = flex.pow2(real_map)
squared_sf_map = rfft.forward(squared_map)
allow_miller_indices_outside_map = False
from_map = maptbx.structure_factors.from_map(
anomalous_flag=var.anomalous_flag(),
miller_indices=var.indices(),
complex_map=squared_sf_map,
conjugate_flag=conjugate_flag,
allow_miller_indices_outside_map=allow_miller_indices_outside_map)
if (selection_fixed is None):
return from_map.data()
result = start.data().deep_copy()
result.set_selected(~selection_fixed, from_map.data())
assert result.select(selection_fixed).all_eq(fixed.data())
return result
def run(xray_structure, f_map=None, map_data=None, d_fsc_model=None):
assert [f_map, map_data].count(None) == 1
xrs = xray_structure.deep_copy_scatterers().set_b_iso(value=0)
if (f_map is None):
f_map = miller.structure_factor_box_from_map(
map=map_data, crystal_symmetry=xray_structure.crystal_symmetry())
fc = f_map.structure_factors_from_scatterers(xray_structure=xrs).f_calc()
d_model_b0 = run_at_b(b=0, f_map=f_map, f_calc=fc).d_min
del xrs
if (d_fsc_model is None):
d_fsc_model = fc.d_min_from_fsc(other=f_map, fsc_cutoff=0).d_min
fo = f_map.resolution_filter(d_min=d_fsc_model)
fo, fc, = fo.common_sets(fc)
cc = -999
b = None
ss = 1. / flex.pow2(fc.d_spacings().data()) / 4.
data = fc.data()
for b_ in range(-500, 500, 5):
sc = flex.exp(-b_ * ss)
fc_ = fc.customized_copy(data=data * sc)
cc_ = fo.map_correlation(other=fc_)
if (cc_ > cc):
cc = cc_
b = b_
o = run_at_b(b=b, f_map=fo, f_calc=fc)
return group_args(d_min=o.d_min,
b_iso=b,
d_model_b0=d_model_b0,
d_fsc_model=d_fsc_model)
def log_p_obs_given_gamma(self, gamma):
dof = self.degrees_of_freedom
x_gamma = (gamma * self.delta_fc2.data() - self.delta_fo2.data()) \
/ self.delta_fo2.sigmas()
if self.probability_plot_slope is not None:
x_gamma /= self.probability_plot_slope
return -(1+dof)/2 * flex.sum(flex.log(flex.pow2(x_gamma) + dof))
def __init__(self, f, n_atoms_absent, n_atoms_included, bf_atoms_absent,
final_error, absent_atom_type):
#
# ss=s**2 = (2*sin(teta)/lambda)**2 for the given reflection;
# final_error - desired mean error in atomic positions (in A);
# it must be specified as 0., if the user has
# no idea about its other value;
# n_atoms_included - an approximate number of non-hydrogen atoms in
# the ASYMMETRIC PART OF THE UNIT CELL, which
# are INCLUDED into the current model for refinement;
# n_atoms_absent - an approximate number of non-hydrogen atoms in
# the ASYMMETRIC PART OF THE UNIT CELL, which are
# NOT INCLUDED into the current model for refinement;
#.....................................................................
# P.Afonine, V.Lunin & A.Urzhumtsev.(2003).J.Appl.Cryst.36,158-159
#
self.f = f
assert n_atoms_absent >= 0
assert n_atoms_included >= 0
assert f.size() > 0
self.ss = 1. / flex.pow2(f.d_spacings().data())
assert self.ss.size() == f.data().size()
self.nsym = f.space_group().order_z()
assert self.nsym >= 1
self.n_atoms_absent = n_atoms_absent
self.n_atoms_included = n_atoms_included
self.bf_atoms_absent = bf_atoms_absent
if final_error is None: final_error = 0.0
self.final_error = final_error
assert final_error >= 0.0
if absent_atom_type is None: absent_atom_type = "C"
self.absent_atom_type = absent_atom_type
def compute_chi_sq(fo_sq, fc_sq, a,b):
weighting.a = a
weighting.b = b
weights = weighting(
fo_sq.data(), fo_sq.sigmas(), fc_sq.data(), scale_factor)
return (flex.sum(
weights * flex.pow2(fo_sq.data() - scale_factor * fc_sq.data())))
def dump_R_in_bins(obs, calc, scale_B=True, log_out=sys.stdout, n_bins=20):
#obs, calc = obs.common_sets(calc, assert_is_similar_symmetry=False)
if scale_B:
scale, B = kBdecider(obs, calc).run()
d_star_sq = calc.d_star_sq().data()
calc = calc.customized_copy(data = scale * flex.exp(-B*d_star_sq) * calc.data())
binner = obs.setup_binner(n_bins=n_bins)
count=0
log_out.write("dmax - dmin: R (nref) <I1> <I2> scale\n")
for i_bin in binner.range_used():
tmp_obs = obs.select(binner.bin_indices() == i_bin)
tmp_calc = calc.select(binner.bin_indices() == i_bin)
low = binner.bin_d_range(i_bin)[0]
high = binner.bin_d_range(i_bin)[1]
if scale_B:
scale = 1.
else:
scale = flex.sum(tmp_obs.data()*tmp_calc.data()) / flex.sum(flex.pow2(tmp_calc.data()))
R = flex.sum(flex.abs(tmp_obs.data() - scale*tmp_calc.data())) / flex.sum(0.5 * tmp_obs.data() + 0.5 * scale*tmp_calc.data())
log_out.write("%5.2f - %5.2f: %.5f (%d) %.1f %.1f %.3e\n" % (low, high, R, len(tmp_obs.data()),
flex.mean(tmp_obs.data()), flex.mean(tmp_calc.data()),
scale))
log_out.write("Overall R = %.5f (scale=%.3e, %%comp=%.3f)\n\n" % (calc_R(obs, calc, do_scale=not scale_B) + (obs.completeness()*100.,)) )
def local_standard_deviations_target_per_site(
unit_cell, density_map, weight_map, weight_map_scale_factor,
sites_cart, site_radii):
if (weight_map is None):
return maptbx.standard_deviations_around_sites(
unit_cell=unit_cell,
density_map=density_map,
sites_cart=sites_cart,
site_radii=site_radii)
d = maptbx.real_space_target_simple_per_site(
unit_cell=unit_cell,
density_map=density_map,
sites_cart=sites_cart)
w = flex.pow2(maptbx.standard_deviations_around_sites(
unit_cell=unit_cell,
density_map=weight_map,
sites_cart=sites_cart,
site_radii=site_radii))
w_min = 0.01
w.set_selected((w < w_min), w_min)
w = 1. / w
if (weight_map_scale_factor is not None):
assert weight_map_scale_factor > 0
w *= weight_map_scale_factor
return d / w
def amplitude_quasi_normalisations(ma, d_star_power=1, set_to_minimum=None,
pseudo_likelihood=False): # Used for pseudo-likelihood calculation
epsilons = ma.epsilons().data().as_double()
mean_f_sq_over_epsilon = flex.double()
for i_bin in ma.binner().range_used():
sel = ma.binner().selection(i_bin)
if pseudo_likelihood:
sel_f_sq = flex.pow2(ma.data().select(sel)) # original method used
else: # usual
sel_f_sq = ma.data().select(sel)
if (sel_f_sq.size() > 0):
sel_epsilons = epsilons.select(sel)
sel_f_sq_over_epsilon = sel_f_sq / sel_epsilons
mean_f_sq_over_epsilon.append(flex.mean(sel_f_sq_over_epsilon))
else:
mean_f_sq_over_epsilon.append(0)
mean_f_sq_over_epsilon_interp = ma.binner().interpolate(
mean_f_sq_over_epsilon, d_star_power)
if set_to_minimum and not mean_f_sq_over_epsilon_interp.all_gt(0):
# HACK NO REASON THIS SHOULD WORK BUT IT GETS BY THE FAILURE
sel = (mean_f_sq_over_epsilon_interp <= set_to_minimum)
mean_f_sq_over_epsilon_interp.set_selected(sel,-mean_f_sq_over_epsilon_interp)
sel = (mean_f_sq_over_epsilon_interp <= set_to_minimum)
mean_f_sq_over_epsilon_interp.set_selected(sel,set_to_minimum)
assert mean_f_sq_over_epsilon_interp.all_gt(0)
from cctbx.miller import array
return array(ma, flex.sqrt(mean_f_sq_over_epsilon_interp))
def scale_data(indices, iobs, scale_ref, parameter, calc_cc):
k, b, cc = 1, float("nan"), float("nan")
sortp = yamtbx_utils_ext.sort_permutation_fast_less(indices)
indices = indices.select(sortp)
iobs = iobs.select(sortp)
sel0, sel1 = yamtbx_utils_ext.my_common_indices(scale_ref.indices(), indices)
#indices = indices.select(sel1)
iobs_c = iobs.select(sel1)
ref_c = scale_ref.data().select(sel0)
if iobs_c.size() < 10 and ref_c.size() < 10:
return k, b, cc
if parameter == "k":
k = flex.sum(ref_c*iobs_c) / flex.sum(flex.pow2(iobs_c))
elif parameter == "kb":
from yamtbx.dataproc.scale_data import kBdecider
kbd = kBdecider(scale_ref,
miller.array(scale_ref.customized_copy(indices=indices),data=iobs))
k, b = kbd.run()
else:
raise "Never reaches here"
if calc_cc:
corr = flex.linear_correlation(ref_c, iobs_c)
if corr.is_well_defined(): cc = corr.coefficient()
return k, b, cc
def get_sf(k_sol,
b_sol,
b_cart,
xrs,
miller_set=None,
d_min=None,
twin_law=None,
sfg_params=None):
random.seed(0)
flex.set_random_seed(0)
if (miller_set is None):
assert d_min is not None
f_dummy = abs(
xrs.structure_factors(d_min=d_min, anomalous_flag=False).f_calc())
else:
f_dummy = miller_set
assert d_min is None
r_free_flags = f_dummy.generate_r_free_flags(fraction=0.1)
fmodel = mmtbx.f_model.manager(r_free_flags=r_free_flags,
f_obs=f_dummy,
sf_and_grads_accuracy_params=sfg_params,
xray_structure=xrs,
twin_law=twin_law)
ss = 1. / flex.pow2(r_free_flags.d_spacings().data()) / 4.
k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol)
u_star = adptbx.u_cart_as_u_star(xrs.unit_cell(), adptbx.b_as_u(b_cart))
k_anisotropic = mmtbx.f_model.ext.k_anisotropic(r_free_flags.indices(),
u_star)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True,
update_f_mask=True)
fmodel.update_core(k_mask=k_mask, k_anisotropic=k_anisotropic)
f_obs = abs(fmodel.f_model())
return f_obs, r_free_flags
def sharp_map(sites_frac,
map_coeffs,
ss=None,
b_sharp=None,
b_min=-150,
b_max=150,
step=10):
if (ss is None):
ss = 1. / flex.pow2(map_coeffs.d_spacings().data()) / 4.
from cctbx import miller
if (b_sharp is None):
t = -1
map_coeffs_best = None
b_sharp_best = None
for b_sharp in range(b_min, b_max, step):
map_coeffs_ = map_coeffs.deep_copy()
sc2 = flex.exp(b_sharp * ss)
map_coeffs_ = map_coeffs_.customized_copy(data=map_coeffs_.data() *
sc2)
t_ = sharp_evaluation_target(sites_frac=sites_frac,
map_coeffs=map_coeffs_)
if (t_ > t):
t = t_
b_sharp_best = b_sharp
map_coeffs_best = map_coeffs_.deep_copy()
print "b_sharp:", b_sharp_best, t
else:
scale = flex.exp(b_sharp * ss)
map_coeffs_best = map_coeffs.customized_copy(data=map_coeffs.data() *
scale)
b_sharp_best = b_sharp
return map_coeffs_best, b_sharp_best
def compute_chi_sq(fo_sq, fc_sq, a,b):
weighting.a = a
weighting.b = b
weights = weighting(
fo_sq.data(), fo_sq.sigmas(), fc_sq.data(), scale_factor)
return (flex.sum(
weights * flex.pow2(fo_sq.data() - scale_factor * fc_sq.data())))
def ls_ff_weights(f_obs, atom, B):
d_star_sq_data = f_obs.d_star_sq().data()
table = wk1995(atom).fetch()
ff = table.at_d_star_sq(d_star_sq_data) * flex.exp(
-B / 4.0 * d_star_sq_data)
weights = 1.0 / flex.pow2(ff)
return weights
def ls_sigma_weights(f_obs):
if(f_obs.sigmas() is not None):
sigmas_squared = flex.pow2(f_obs.sigmas())
else:
sigmas_squared = flex.double(f_obs.data().size(), 1.0)
assert sigmas_squared.all_gt(0)
weights = 1 / sigmas_squared
return weights
def f(self, x):
print x
B = float(x[0])
#d_star_sq = self.calc.d_star_sq().data()
#obs = self.obs.data()
#calc = flex.exp(-B*d_star_sq)*self.calc.data()
k = self.get_linear_scale(self.obs, self.calc, B)
return flex.sum(flex.pow2(self.obs.data() - k*self.calc.data()))
def ls_sigma_weights(f_obs):
if (f_obs.sigmas() is not None):
sigmas_squared = flex.pow2(f_obs.sigmas())
else:
sigmas_squared = flex.double(f_obs.data().size(), 1.0)
assert sigmas_squared.all_gt(0)
weights = 1 / sigmas_squared
return weights
def target(self, drho, a):
self.she_object.update_solvent_params(self.rho,drho)
this_scale = self.compute_scale_array( a )
i_calc = self.she_object.Iscale(this_scale)
s, off = linear_fit( i_calc, self.obs.i, self.obs.s )
i_calc = s*i_calc + off
result = flex.sum(flex.pow2( (i_calc-self.obs.i)/self.obs.s ) )
return result, i_calc, s, off
def __call__(self, f_calc, compute_derivatives):
assert f_calc.is_similar_symmetry(self.f_obs())
return ext.targets_correlation(
obs_type="I",
obs=flex.pow2(self.f_obs().data()),
weights=self.weights(),
r_free_flags=None,
f_calc=f_calc.data(),
derivatives_depth=int(compute_derivatives))
def __init__(
self,
fmodel,
coeffs):
# XXX see f_model.py: duplication! Consolidate.
self.fmodel = fmodel
self.coeffs = coeffs
crystal_gridding = fmodel.f_obs().crystal_gridding(
d_min = self.fmodel.f_obs().d_min(),
resolution_factor = 1./3)
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = self.coeffs)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
rho_atoms = flex.double()
for site_frac in self.fmodel.xray_structure.sites_frac():
rho_atoms.append(map_data.eight_point_interpolation(site_frac))
rho_mean = flex.mean_default(rho_atoms.select(rho_atoms>0.5), 0.5)
sel_exclude = rho_atoms > min(rho_mean/2., 1)
sites_cart = fmodel.xray_structure.sites_cart()
#
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = self.fmodel.f_model())
fft_map.apply_sigma_scaling()
map_data2 = fft_map.real_map_unpadded()
#
for i_seq, site_cart in enumerate(sites_cart):
selection = maptbx.grid_indices_around_sites(
unit_cell = self.coeffs.unit_cell(),
fft_n_real = map_data.focus(),
fft_m_real = map_data.all(),
sites_cart = flex.vec3_double([site_cart]),
site_radii = flex.double([1.5]))
cc = flex.linear_correlation(x=map_data.select(selection),
y=map_data2.select(selection)).coefficient()
if(cc<0.7): sel_exclude[i_seq] = False
#
del map_data, fft_map, rho_atoms
self.d_min = fmodel.f_obs().d_min()
cs = self.fmodel.f_obs().average_bijvoet_mates().complete_set(d_min=self.d_min)
self.complete_set = cs.array(data = flex.double(cs.indices().size(), 0))
self.xray_structure_cut = self.fmodel.xray_structure.select(sel_exclude)
self.missing_set = self.complete_set.common_set(self.coeffs)
#
self.f_calc_missing = self.complete_set.structure_factors_from_scatterers(
xray_structure = self.xray_structure_cut).f_calc()
self.ss_missing = 1./flex.pow2(self.f_calc_missing.d_spacings().data()) / 4.
mask_manager = mmtbx.masks.manager(
miller_array = self.f_calc_missing,
miller_array_twin = None,
mask_params = None)
self.f_mask_missing = mask_manager.shell_f_masks(
xray_structure = self.xray_structure_cut,
force_update = True)
self.zero_data = flex.complex_double(self.f_calc_missing.data().size(), 0)
def _resolution_from_map_and_model_helper_2(map,
f_obs,
fc,
xray_structure,
b_range,
nproc,
simple,
d_mins,
radius,
ofn=None):
f_obs, fc = mask_out(map=map,
xrs=xray_structure,
f_obs=f_obs,
fc=fc,
d_mins=d_mins,
radius=radius)
ccs = flex.double()
bs = flex.double()
d_min_opt = flex.double()
cntr = 0
of = None
if (ofn is not None): of = open(ofn, "w")
d_spacings = fc.d_spacings().data()
ss = 1. / flex.pow2(d_spacings) / 4.
for b in b_range:
o = run_loop(fc=fc,
f_obs=f_obs,
b_iso=b,
map=map,
d_mins=d_mins,
d_spacings=d_spacings,
ss=ss)
ccs.append(o.cc)
d_min_opt.append(o.d_min)
bs.append(b)
maxima = canal(x=d_min_opt,
y=ccs,
simple=simple,
smooth=True,
xround=True,
show=True,
of=of)
if (ofn is not None): of.close()
d_min_result = None
cc_result = None
b_result = None
if (len(maxima) > 0):
d_min_result = maxima[0][0]
cc_result = maxima[0][1]
dist = 1.e+9
for i, cc in enumerate(ccs):
dist_ = abs(cc - cc_result)
if (dist_ < dist):
dist = dist_
b_result = bs[i]
return d_min_result, b_result, cc_result
def __call__(self, f_calc, compute_derivatives):
assert f_calc.is_similar_symmetry(self.f_obs())
return ext.targets_correlation(
obs_type="I",
obs=flex.pow2(self.f_obs().data()),
weights=self.weights(),
r_free_flags=None,
f_calc=f_calc.data(),
derivatives_depth=int(compute_derivatives))
def poly_residual(xp, y, params):
"""Compute the residual between the observations y[i] and sum_j
params[j] x[i]^j. For efficiency, x[i]^j are pre-calculated in xp."""
c = len(y)
e = flex.double([flex.sum(xp[j] * params) for j in range(c)])
return flex.sum(flex.pow2(y - e))
def __init__(
self,
fmodel,
coeffs):
# XXX see f_model.py: duplication! Consolidate.
self.fmodel = fmodel
self.coeffs = coeffs
crystal_gridding = fmodel.f_obs().crystal_gridding(
d_min = self.fmodel.f_obs().d_min(),
resolution_factor = 1./3)
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = self.coeffs)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
rho_atoms = flex.double()
for site_frac in self.fmodel.xray_structure.sites_frac():
rho_atoms.append(map_data.eight_point_interpolation(site_frac))
rho_mean = flex.mean_default(rho_atoms.select(rho_atoms>0.5), 0.5)
sel_exclude = rho_atoms > min(rho_mean/2., 1)
sites_cart = fmodel.xray_structure.sites_cart()
#
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = self.fmodel.f_model())
fft_map.apply_sigma_scaling()
map_data2 = fft_map.real_map_unpadded()
#
for i_seq, site_cart in enumerate(sites_cart):
selection = maptbx.grid_indices_around_sites(
unit_cell = self.coeffs.unit_cell(),
fft_n_real = map_data.focus(),
fft_m_real = map_data.all(),
sites_cart = flex.vec3_double([site_cart]),
site_radii = flex.double([1.5]))
cc = flex.linear_correlation(x=map_data.select(selection),
y=map_data2.select(selection)).coefficient()
if(cc<0.7): sel_exclude[i_seq] = False
#
del map_data, fft_map, rho_atoms
self.d_min = fmodel.f_obs().d_min()
cs = self.fmodel.f_obs().average_bijvoet_mates().complete_set(d_min=self.d_min)
self.complete_set = cs.array(data = flex.double(cs.indices().size(), 0))
self.xray_structure_cut = self.fmodel.xray_structure.select(sel_exclude)
self.missing_set = self.complete_set.common_set(self.coeffs)
#
self.f_calc_missing = self.complete_set.structure_factors_from_scatterers(
xray_structure = self.xray_structure_cut).f_calc()
self.ss_missing = 1./flex.pow2(self.f_calc_missing.d_spacings().data()) / 4.
mask_manager = mmtbx.masks.manager(
miller_array = self.f_calc_missing,
miller_array_twin = None,
mask_params = None)
self.f_mask_missing = mask_manager.shell_f_masks(
xray_structure = self.xray_structure_cut,
force_update = True)
self.zero_data = flex.complex_double(self.f_calc_missing.data().size(), 0)
def fft(self):
if self.params.fft3d.reciprocal_space_grid.d_min is libtbx.Auto:
# rough calculation of suitable d_min based on max cell
# see also Campbell, J. (1998). J. Appl. Cryst., 31(3), 407-413.
# fft_cell should be greater than twice max_cell, so say:
# fft_cell = 2.5 * max_cell
# then:
# fft_cell = n_points * d_min/2
# 2.5 * max_cell = n_points * d_min/2
# a little bit of rearrangement:
# d_min = 5 * max_cell/n_points
max_cell = self.params.max_cell
d_min = (5 * max_cell /
self.params.fft3d.reciprocal_space_grid.n_points)
d_spacings = 1 / self.reflections['rlp'].norms()
self.params.fft3d.reciprocal_space_grid.d_min = max(
d_min, min(d_spacings))
logger.info("Setting d_min: %.2f" %
self.params.fft3d.reciprocal_space_grid.d_min)
n_points = self.params.fft3d.reciprocal_space_grid.n_points
self.gridding = fftpack.adjust_gridding_triple(
(n_points, n_points, n_points), max_prime=5)
n_points = self.gridding[0]
self.map_centroids_to_reciprocal_space_grid()
self.d_min = self.params.fft3d.reciprocal_space_grid.d_min
logger.info("Number of centroids used: %i" %
((self.reciprocal_space_grid > 0).count(True)))
#gb_to_bytes = 1073741824
#bytes_to_gb = 1/gb_to_bytes
#(128**3)*8*2*bytes_to_gb
#0.03125
#(256**3)*8*2*bytes_to_gb
#0.25
#(512**3)*8*2*bytes_to_gb
#2.0
fft = fftpack.complex_to_complex_3d(self.gridding)
grid_complex = flex.complex_double(
reals=self.reciprocal_space_grid,
imags=flex.double(self.reciprocal_space_grid.size(), 0))
grid_transformed = fft.forward(grid_complex)
#self.grid_real = flex.pow2(flex.abs(grid_transformed))
self.grid_real = flex.pow2(flex.real(grid_transformed))
#self.grid_real = flex.pow2(flex.imag(self.grid_transformed))
del grid_transformed
if self.params.debug:
self.debug_write_ccp4_map(map_data=self.grid_real,
file_name="fft3d.map")
if self.params.fft3d.peak_search == 'flood_fill':
self.find_peaks()
elif self.params.fft3d.peak_search == 'clean':
self.find_peaks_clean()
def compute(self, f_calc, scale_factor=None):
self.calculated = f_calc
assert(self.observed.is_xray_intensity_array())
assert(self.calculated.is_complex_array())
a,b = self._params
f_c = self.calculated.data()
if scale_factor is None:
scale_factor = self.observed.scale_factor(
self.calculated, cutoff_factor=0.99)
self.scale_factor = scale_factor
f_c = f_c * math.sqrt(scale_factor) # don't modify f_c in place
sigmas_square = flex.pow2(self.observed.sigmas())
f_obs_square_plus = self.observed.data().deep_copy()
negatives = self.observed.data() < 0
f_obs_square_plus.set_selected(negatives, 0)
p = (f_obs_square_plus + 2*flex.norm(f_c))/3
w = 1/(sigmas_square + flex.pow2(a*p) + b*p)
dw_dfc = -(2*a*a*p + b) * flex.pow2(w) * (4./3*f_c)
self.weights, self.derivatives_wrt_f_c = w, dw_dfc
def calc_R(obs, calc, do_scale=True):
#obs, calc = obs.common_sets(calc, assert_is_similar_symmetry=False)
if do_scale:
scale = flex.sum(obs.data()*calc.data()) / flex.sum(flex.pow2(calc.data()))
else:
scale = 1.
R = flex.sum(flex.abs(obs.data() - scale*calc.data())) / flex.sum(0.5 * obs.data() + 0.5 * scale*calc.data())
return R, scale
def compute(self, f_calc, scale_factor=None):
self.calculated = f_calc
assert (self.observed.is_xray_intensity_array())
assert (self.calculated.is_complex_array())
a, b = self._params
f_c = self.calculated.data()
if scale_factor is None:
scale_factor = self.observed.scale_factor(self.calculated,
cutoff_factor=0.99)
self.scale_factor = scale_factor
f_c = f_c * math.sqrt(scale_factor) # don't modify f_c in place
sigmas_square = flex.pow2(self.observed.sigmas())
f_obs_square_plus = self.observed.data().deep_copy()
negatives = self.observed.data() < 0
f_obs_square_plus.set_selected(negatives, 0)
p = (f_obs_square_plus + 2 * flex.norm(f_c)) / 3
w = 1 / (sigmas_square + flex.pow2(a * p) + b * p)
dw_dfc = -(2 * a * a * p + b) * flex.pow2(w) * (4. / 3 * f_c)
self.weights, self.derivatives_wrt_f_c = w, dw_dfc
def get_hl(f_obs_cmpl, k_blur, b_blur): f_model_phases = f_obs_cmpl.phases().data() sin_f_model_phases = flex.sin(f_model_phases) cos_f_model_phases = flex.cos(f_model_phases) ss = 1./flex.pow2(f_obs_cmpl.d_spacings().data()) / 4. t = 2*k_blur * flex.exp(-b_blur*ss) hl_a_model = t * cos_f_model_phases hl_b_model = t * sin_f_model_phases hl_data = flex.hendrickson_lattman(a = hl_a_model, b = hl_b_model) hl = f_obs_cmpl.customized_copy(data = hl_data) return hl
def get_hl(f_obs_cmpl, k_blur, b_blur):
f_model_phases = f_obs_cmpl.phases().data()
sin_f_model_phases = flex.sin(f_model_phases)
cos_f_model_phases = flex.cos(f_model_phases)
ss = 1. / flex.pow2(f_obs_cmpl.d_spacings().data()) / 4.
t = 2 * k_blur * flex.exp(-b_blur * ss)
hl_a_model = t * cos_f_model_phases
hl_b_model = t * sin_f_model_phases
hl_data = flex.hendrickson_lattman(a=hl_a_model, b=hl_b_model)
hl = f_obs_cmpl.customized_copy(data=hl_data)
return hl
def poly_residual(xp, y, params): '''Compute the residual between the observations y[i] and sum_j params[j] x[i]^j. For efficiency, x[i]^j are pre-calculated in xp.''' r = 0.0 n = len(params) c = len(y) e = flex.double([flex.sum(xp[j] * params) for j in range(c)]) return flex.sum(flex.pow2(y - e))
def find_peaks(self):
grid_real_binary = self.grid_real.deep_copy()
rmsd = math.sqrt(
flex.mean(
flex.pow2(grid_real_binary.as_1d() -
flex.mean(grid_real_binary.as_1d()))))
grid_real_binary.set_selected(
grid_real_binary < (self.params.rmsd_cutoff) * rmsd, 0)
grid_real_binary.as_1d().set_selected(grid_real_binary.as_1d() > 0, 1)
grid_real_binary = grid_real_binary.iround()
from cctbx import masks
flood_fill = masks.flood_fill(grid_real_binary, self.fft_cell)
if flood_fill.n_voids() < 4:
# Require at least peak at origin and one peak for each basis vector
raise Sorry(
"Indexing failed: fft3d peak search failed to find sufficient number of peaks."
)
# the peak at the origin might have a significantly larger volume than the
# rest so exclude this peak from determining maximum volume
isel = (flood_fill.grid_points_per_void() > int(
self.params.fft3d.peak_volume_cutoff *
flex.max(flood_fill.grid_points_per_void()[1:]))).iselection()
if self.params.optimise_initial_basis_vectors:
self.volumes = flood_fill.grid_points_per_void().select(isel)
sites_cart = flood_fill.centres_of_mass_cart().select(isel)
sites_cart_optimised = optimise_basis_vectors(
self.reflections['rlp'].select(
self.reflections_used_for_indexing), sites_cart)
self.sites = self.fft_cell.fractionalize(sites_cart_optimised)
diffs = (sites_cart_optimised - sites_cart)
norms = diffs.norms()
flex.min_max_mean_double(norms).show()
perm = flex.sort_permutation(norms, reverse=True)
for p in perm[:10]:
logger.debug(sites_cart[p], sites_cart_optimised[p], norms[p])
# only use those vectors which haven't shifted too far from starting point
sel = norms < (5 * self.fft_cell.parameters()[0] /
self.gridding[0])
self.sites = self.sites.select(sel)
self.volumes = self.volumes.select(sel)
#diff = (self.sites - flood_fill.centres_of_mass_frac().select(isel))
#flex.min_max_mean_double(diff.norms()).show()
else:
self.sites = flood_fill.centres_of_mass_frac().select(isel)
self.volumes = flood_fill.grid_points_per_void().select(isel)
def modified_intensities(observations, f_model, f_mask):
"""Subtracts the solvent contribution from the observed structure
factors to obtain modified structure factors, suitable for refinement
with other refinement programs such as ShelXL"""
f_obs = observations.as_amplitude_array()
if f_obs.sigmas() is not None:
weights = weights = 1 / flex.pow2(f_obs.sigmas())
else:
weights = None
scale_factor = f_obs.scale_factor(f_model, weights=weights)
f_obs = f_obs.phase_transfer(phase_source=f_model)
modified_f_obs = miller.array(miller_set=f_obs, data=(f_obs.data() - f_mask.data() * scale_factor))
if observations.is_xray_intensity_array():
# it is better to use the original sigmas for intensity if possible
return modified_f_obs.as_intensity_array().customized_copy(sigmas=observations.sigmas())
else:
return modified_f_obs.customized_copy(sigmas=f_obs.sigmas()).as_intensity_array()
def solvent_flipping(self):
if not self.params.solvent_modification.method == "flipping": return
if (self.i_cycle + 1) == self.max_iterations:
self.k_flip = 0
else:
self.k_flip = -(1-self.params.solvent_fraction)/self.params.solvent_fraction
if self.params.solvent_modification.scale_flip:
rms_protein_density_new = math.sqrt(
flex.mean(flex.pow2(self.map.select(self.protein_iselection))))
self.k_flip *= math.pow(
rms_protein_density_new/self.rms_protein_density, 2)
self.map.as_1d().copy_selected(
self.solvent_iselection,
(self.mean_solvent_density
+ self.k_flip * (self.map - self.mean_solvent_density)).as_1d())
self.mean_solvent_density = flex.mean(
self.map.select(self.solvent_iselection))
def solvent_flipping(self):
if not self.params.solvent_modification.method == "flipping": return
if (self.i_cycle + 1) == self.max_iterations:
self.k_flip = 0
else:
self.k_flip = -(1-self.params.solvent_fraction)/self.params.solvent_fraction
if self.params.solvent_modification.scale_flip:
rms_protein_density_new = math.sqrt(
flex.mean(flex.pow2(self.map.select(self.protein_iselection))))
self.k_flip *= math.pow(
rms_protein_density_new/self.rms_protein_density, 2)
self.map.as_1d().copy_selected(
self.solvent_iselection,
(self.mean_solvent_density
+ self.k_flip * (self.map - self.mean_solvent_density)).as_1d())
self.mean_solvent_density = flex.mean(
self.map.select(self.solvent_iselection))
def compute(self):
assert(self.observed.is_xray_intensity_array())
f_sqr = self.observed.data()
sig_f_sqr = self.observed.sigmas()
w = flex.double(f_sqr.size())
if sig_f_sqr is None:
strongs = flex.bool(f_sqr.size(), True)
else:
if self.n_sigma == 1:
strongs = f_sqr > sig_f_sqr
else:
strongs = f_sqr > self.n_sigma * sig_f_sqr
weaks = ~strongs
w.set_selected(strongs, 0.25/f_sqr.select(strongs))
if sig_f_sqr is not None:
w.set_selected(weaks, 0.25/flex.pow2( sig_f_sqr.select(weaks) ))
self.weights = w
self.derivatives_wrt_f_c = None
def bond_similarity_as_cif_loops(xray_structure, proxies):
space_group_info = sgtbx.space_group_info(group=xray_structure.space_group())
unit_cell = xray_structure.unit_cell()
sites_cart = xray_structure.sites_cart()
site_labels = xray_structure.scatterers().extract_labels()
fmt = "%.4f"
loop = model.loop(header=(
"_restr_equal_distance_atom_site_label_1",
"_restr_equal_distance_atom_site_label_2",
"_restr_equal_distance_site_symmetry_2",
"_restr_equal_distance_class_id",
))
class_loop = model.loop(header=(
"_restr_equal_distance_class_class_id",
"_restr_equal_distance_class_target_weight_param",
"_restr_equal_distance_class_average",
"_restr_equal_distance_class_esd",
"_restr_equal_distance_class_diff_max",
))
class_id = 0
for proxy in proxies:
restraint = geometry_restraints.bond_similarity(
unit_cell=unit_cell,
sites_cart=sites_cart,
proxy=proxy)
class_id += 1
esd = math.sqrt(flex.sum(flex.pow2(restraint.deltas())) *
(1./proxy.i_seqs.size()))
class_loop.add_row((class_id,
fmt % math.sqrt(1/proxy.weights[0]),# assume equal weights
fmt % restraint.mean_distance(),
fmt % esd,
fmt % flex.max_absolute(restraint.deltas())))
for i in range(proxy.i_seqs.size()):
i_seq, j_seq = proxy.i_seqs[i]
if proxy.sym_ops is None:
sym_op = sgtbx.rt_mx()
else:
sym_op = proxy.sym_ops[i]
loop.add_row((site_labels[i_seq],
site_labels[j_seq],
space_group_info.cif_symmetry_code(sym_op),
class_id))
return class_loop, loop
def find_peaks(self):
grid_real_binary = self.grid_real.deep_copy()
rmsd = math.sqrt(
flex.mean(flex.pow2(grid_real_binary.as_1d()-flex.mean(grid_real_binary.as_1d()))))
grid_real_binary.set_selected(grid_real_binary < (self.params.rmsd_cutoff)*rmsd, 0)
grid_real_binary.as_1d().set_selected(grid_real_binary.as_1d() > 0, 1)
grid_real_binary = grid_real_binary.iround()
from cctbx import masks
flood_fill = masks.flood_fill(grid_real_binary, self.fft_cell)
if flood_fill.n_voids() < 4:
# Require at least peak at origin and one peak for each basis vector
raise Sorry("Indexing failed: fft3d peak search failed to find sufficient number of peaks.")
# the peak at the origin might have a significantly larger volume than the
# rest so exclude this peak from determining maximum volume
isel = (flood_fill.grid_points_per_void() > int(
self.params.fft3d.peak_volume_cutoff * flex.max(
flood_fill.grid_points_per_void()[1:]))).iselection()
if self.params.optimise_initial_basis_vectors:
self.volumes = flood_fill.grid_points_per_void().select(isel)
sites_cart = flood_fill.centres_of_mass_cart().select(isel)
sites_cart_optimised = optimise_basis_vectors(
self.reflections['rlp'].select(self.reflections_used_for_indexing),
sites_cart)
self.sites = self.fft_cell.fractionalize(sites_cart_optimised)
diffs = (sites_cart_optimised - sites_cart)
norms = diffs.norms()
flex.min_max_mean_double(norms).show()
perm = flex.sort_permutation(norms, reverse=True)
for p in perm[:10]:
logger.debug(sites_cart[p], sites_cart_optimised[p], norms[p])
# only use those vectors which haven't shifted too far from starting point
sel = norms < (5 * self.fft_cell.parameters()[0]/self.gridding[0])
self.sites = self.sites.select(sel)
self.volumes = self.volumes.select(sel)
#diff = (self.sites - flood_fill.centres_of_mass_frac().select(isel))
#flex.min_max_mean_double(diff.norms()).show()
else:
self.sites = flood_fill.centres_of_mass_frac().select(isel)
self.volumes = flood_fill.grid_points_per_void().select(isel)
def main(filenames, map_file, npoints=192, max_resolution=6, reverse_phi=False):
rec_range = 1 / max_resolution
image = ImageFactory(filenames[0])
panel = image.get_detector()[0]
beam = image.get_beam()
s0 = beam.get_s0()
pixel_size = panel.get_pixel_size()
xlim, ylim = image.get_raw_data().all()
xy = recviewer.get_target_pixels(panel, s0, xlim, ylim, max_resolution)
s1 = panel.get_lab_coord(xy * pixel_size[0]) # FIXME: assumed square pixel
s1 = s1 / s1.norms() * (1 / beam.get_wavelength()) # / is not supported...
S = s1 - s0
grid = flex.double(flex.grid(npoints, npoints, npoints), 0)
cnts = flex.int(flex.grid(npoints, npoints, npoints), 0)
for filename in filenames:
print "Processing image", filename
try:
fill_voxels(ImageFactory(filename), grid, cnts, S, xy, reverse_phi, rec_range)
except:
print " Failed to process. Skipped this."
recviewer.normalize_voxels(grid, cnts)
uc = uctbx.unit_cell((npoints, npoints, npoints, 90, 90, 90))
ccp4_map.write_ccp4_map(map_file, uc, sgtbx.space_group("P1"),
(0, 0, 0), grid.all(), grid,
flex.std_string(["cctbx.miller.fft_map"]))
return
from scitbx import fftpack
fft = fftpack.complex_to_complex_3d(grid.all())
grid_complex = flex.complex_double(
reals=flex.pow2(grid),
imags=flex.double(grid.size(), 0))
grid_transformed = flex.abs(fft.backward(grid_complex))
print flex.max(grid_transformed), flex.min(grid_transformed), grid_transformed.all()
ccp4_map.write_ccp4_map(map_file, uc, sgtbx.space_group("P1"),
(0, 0, 0), grid.all(), grid_transformed,
flex.std_string(["cctbx.miller.fft_map"]))
def sigma_miss(miller_array, n_atoms_absent, bf_atoms_absent, absent_atom_type):
result = flex.double()
if(n_atoms_absent == 0): return flex.double(miller_array.indices().size(), 0)
def form_factor(ssi, absent_atom_type):
table=wk1995(absent_atom_type).fetch()
a_wk=table.array_of_a()
b_wk=table.array_of_b()
c_wk=table.c()
result_wk=c_wk
for i in xrange(5):
result_wk += a_wk[i]*math.exp(-b_wk[i]*ssi/4.0)
return result_wk
ss = 1./flex.pow2(miller_array.d_spacings().data())
nsym = miller_array.space_group().order_z()
#
for ssi in ss:
fact = form_factor(ssi, absent_atom_type)*math.exp(-bf_atoms_absent/4.0*ssi)
result.append(fact * nsym * n_atoms_absent)
return result
def fft(self):
#gb_to_bytes = 1073741824
#bytes_to_gb = 1/gb_to_bytes
#(128**3)*8*2*bytes_to_gb
#0.03125
#(256**3)*8*2*bytes_to_gb
#0.25
#(512**3)*8*2*bytes_to_gb
#2.0
fft = fftpack.complex_to_complex_3d(self.gridding)
grid_complex = flex.complex_double(
reals=self.reciprocal_space_grid,
imags=flex.double(self.reciprocal_space_grid.size(), 0))
grid_transformed = fft.forward(grid_complex)
#self.grid_real = flex.pow2(flex.abs(grid_transformed))
self.grid_real = flex.pow2(flex.real(grid_transformed))
#self.grid_real = flex.pow2(flex.imag(self.grid_transformed))
del grid_transformed
def exercise_rigid_bond_test():
"""
Results compared with THMA11 (Ver. 20-04-91) - TLS Thermal Motion
Analysis used as a part of WinGX (WinGX - Crystallographic Program
System for Windows)
"""
ins_file = libtbx.env.find_in_repositories(
relative_path="phenix_regression/pdb/enk_11i.res", test=os.path.isfile)
if (ins_file is None):
print "Skipping exercise_rigid_bond_test(): input file not available"
return
ins_xray_structure = cctbx.xray.structure.from_shelx(file=open(ins_file))
sites_frac = ins_xray_structure.sites_frac()
sites_cart = ins_xray_structure.sites_cart()
ustars = ins_xray_structure.scatterers().extract_u_star()
scatterers = ins_xray_structure.scatterers()
j = 0
for site_cart_1,site_frac_1,ustar_1,scat_1 in zip(sites_cart,sites_frac,ustars,scatterers):
for site_cart_2,site_frac_2,ustar_2, scat_2 in zip(sites_cart,sites_frac,ustars,scatterers):
d = math.sqrt(flex.sum(flex.pow2(flex.double(site_cart_1)-\
flex.double(site_cart_2))))
if(d > 1.1 and d < 1.55):
p = adp_restraints.rigid_bond_pair(site_frac_1,
site_frac_2,
ustar_1,
ustar_2,
ins_xray_structure.unit_cell())
if(0):
print "%4s %4s %7.4f %7.4f %7.4f" % \
(scat_1.label,scat_2.label,p.delta_z(),p.z_12(),p.z_21())
r = result[j]
assert r[0] == scat_1.label
assert r[1] == scat_2.label
assert approx_equal(r[2], p.delta_z(), 1.e-4)
assert approx_equal(r[3], p.z_12(), 1.e-4)
assert approx_equal(r[4], p.z_21(), 1.e-4)
j += 1
assert j == 56
def __init__(self,f,
n_atoms_absent,
n_atoms_included,
bf_atoms_absent,
final_error,
absent_atom_type):
#
# ss=s**2 = (2*sin(teta)/lambda)**2 for the given reflection;
# final_error - desired mean error in atomic positions (in A);
# it must be specified as 0., if the user has
# no idea about its other value;
# n_atoms_included - an approximate number of non-hydrogen atoms in
# the ASYMMETRIC PART OF THE UNIT CELL, which
# are INCLUDED into the current model for refinement;
# n_atoms_absent - an approximate number of non-hydrogen atoms in
# the ASYMMETRIC PART OF THE UNIT CELL, which are
# NOT INCLUDED into the current model for refinement;
#.....................................................................
# P.Afonine, V.Lunin & A.Urzhumtsev.(2003).J.Appl.Cryst.36,158-159
#
self.f = f
assert n_atoms_absent >= 0
assert n_atoms_included >= 0
assert f.size() > 0
self.ss = 1./flex.pow2(f.d_spacings().data())
assert self.ss.size() == f.data().size()
self.nsym = f.space_group().order_z()
assert self.nsym >= 1
self.n_atoms_absent = n_atoms_absent
self.n_atoms_included = n_atoms_included
self.bf_atoms_absent = bf_atoms_absent
if final_error is None : final_error = 0.0
self.final_error = final_error
assert final_error >= 0.0
if absent_atom_type is None : absent_atom_type="C"
self.absent_atom_type = absent_atom_type
def direct_space_squaring(start, selection_fixed):
map_gridding = miller.index_span(miller.set.expand_to_p1(start).indices()).map_grid()
if selection_fixed is None:
fixed = start
var = start
else:
fixed = start.select(selection_fixed)
var = start.select(~selection_fixed)
rfft = fftpack.real_to_complex_3d([n * 3 // 2 for n in map_gridding])
conjugate_flag = True
structure_factor_map = maptbx.structure_factors.to_map(
space_group=fixed.space_group(),
anomalous_flag=fixed.anomalous_flag(),
miller_indices=fixed.indices(),
structure_factors=fixed.data(),
n_real=rfft.n_real(),
map_grid=flex.grid(rfft.n_complex()),
conjugate_flag=conjugate_flag,
)
real_map = rfft.backward(structure_factor_map.complex_map())
squared_map = flex.pow2(real_map)
squared_sf_map = rfft.forward(squared_map)
allow_miller_indices_outside_map = False
from_map = maptbx.structure_factors.from_map(
anomalous_flag=var.anomalous_flag(),
miller_indices=var.indices(),
complex_map=squared_sf_map,
conjugate_flag=conjugate_flag,
allow_miller_indices_outside_map=allow_miller_indices_outside_map,
)
if selection_fixed is None:
return from_map.data()
result = start.data().deep_copy()
result.set_selected(~selection_fixed, from_map.data())
assert result.select(selection_fixed).all_eq(fixed.data())
return result
def sharp_map(sites_frac, map_coeffs, ss = None, b_sharp=None, b_min = -150,
b_max = 150, step = 10):
if(ss is None):
ss = 1./flex.pow2(map_coeffs.d_spacings().data()) / 4.
from cctbx import miller
if(b_sharp is None):
t=-1
map_coeffs_best = None
b_sharp_best = None
for b_sharp in range(b_min,b_max,step):
map_coeffs_ = map_coeffs.deep_copy()
sc2 = flex.exp(b_sharp*ss)
map_coeffs_ = map_coeffs_.customized_copy(data = map_coeffs_.data()*sc2)
t_=sharp_evaluation_target(sites_frac=sites_frac, map_coeffs=map_coeffs_)
if(t_>t):
t=t_
b_sharp_best = b_sharp
map_coeffs_best = map_coeffs_.deep_copy()
print "b_sharp:", b_sharp_best, t
else:
scale = flex.exp(b_sharp*ss)
map_coeffs_best = map_coeffs.customized_copy(data=map_coeffs.data()*scale)
b_sharp_best = b_sharp
return map_coeffs_best, b_sharp_best
def alpha_beta(f_dist,
n_atoms_included,
n_nonwater_atoms_absent,
n_water_atoms_absent,
bf_atoms_absent,
final_error,
absent_atom_type):
nsym = f_dist.space_group().order_z()
ss = 1./flex.pow2(f_dist.d_spacings().data())
n_part = nsym * n_atoms_included
n_lost_p = nsym * n_nonwater_atoms_absent
n_lost_w = nsym * n_water_atoms_absent
f_dist_data = flex.abs(f_dist.data())
a_d = flex.exp( -0.25 * ss * final_error**2 * math.pi**3 )
d_star_sq_data = f_dist.d_star_sq().data()
assert approx_equal(ss,d_star_sq_data)
table = wk1995(absent_atom_type).fetch()
ff = table.at_d_star_sq(d_star_sq_data)
factor = ff * flex.exp(-bf_atoms_absent/4.0*d_star_sq_data)
b_d = ((1.-a_d*a_d)*n_part+n_lost_p+n_lost_w*(1.-f_dist_data*f_dist_data))*\
factor*factor
alpha = f_dist.array(data = a_d)
beta = f_dist.array(data = b_d)
return alpha, beta
def __init__(self, f_obs = None,
f_calc = None,
free_reflections_per_bin = None,
flags = None,
verbose = None,
n_atoms_absent = None,
n_atoms_included = None,
bf_atoms_absent = None,
final_error = None,
absent_atom_type = None,
method = None,
interpolation = None):
adopt_init_args(self, locals())
assert self.method == "calc" or self.method == "est" or \
self.method == "calc_and_est"
assert self.verbose is not None
if (self.method == "est"):
assert self.interpolation is not None
assert self.f_obs.data().size() == self.f_calc.data().size()
assert self.flags.size() == self.f_obs.data().size()
assert self.f_calc.indices().all_eq(self.f_obs.indices()) == 1
self.alpha, self.beta = alpha_beta_est_manager(
f_obs = self.f_obs,
f_calc = abs(self.f_calc),
free_reflections_per_bin = self.free_reflections_per_bin,
flags = self.flags,
interpolation = self.interpolation).alpha_beta()
if (self.method == "calc"):
assert self.f_obs is not None or self.f_calc is not None
if (self.f_obs is not None): f = self.f_obs
else: f = self.f_calc
assert self.n_atoms_absent is not None
assert self.n_atoms_included is not None
assert self.bf_atoms_absent is not None
assert self.absent_atom_type is not None
self.alpha, self.beta = alpha_beta_calc(
f = f,
n_atoms_absent = self.n_atoms_absent,
n_atoms_included = self.n_atoms_included,
bf_atoms_absent = self.bf_atoms_absent,
final_error = self.final_error,
absent_atom_type = self.absent_atom_type).alpha_beta()
if (self.method == "calc_and_est"):
assert self.interpolation is not None
assert self.f_obs is not None
assert self.f_calc is not None
assert self.free_reflections_per_bin is not None
assert self.flags is not None
assert self.n_atoms_absent is not None
assert self.n_atoms_included is not None
assert self.bf_atoms_absent is not None
assert self.final_error is not None
assert self.absent_atom_type is not None
assert self.f_obs.data().size() == self.f_calc.data().size() == \
self.flags.size()
assert self.f_calc.indices().all_eq(self.f_obs.indices()) == 1
self.alpha_calc, self.beta_calc = alpha_beta_calc(
f = self.f_obs,
n_atoms_absent = self.n_atoms_absent,
n_atoms_included = self.n_atoms_included,
bf_atoms_absent = self.bf_atoms_absent,
final_error = self.final_error,
absent_atom_type = self.absent_atom_type).alpha_beta()
self.alpha_est, self.beta_est = alpha_beta_est(
f_obs = self.f_obs,
f_calc = abs(self.f_calc),
free_reflections_per_bin = self.free_reflections_per_bin,
flags = self.flags,
interpolation = self.interpolation).alpha_beta()
alpha_calc_ma = miller.array(miller_set= self.f_obs,data= self.alpha_calc)
beta_calc_ma = miller.array(miller_set= self.f_obs,data= self.beta_calc)
alpha_est_ma = miller.array(miller_set= self.f_obs,data= self.alpha_est)
beta_est_ma = miller.array(miller_set= self.f_obs,data= self.beta_est)
ss = 1./flex.pow2(alpha_calc_ma.d_spacings().data())
omega_calc = []
omega_est = []
for ac,ae,ssi in zip(self.alpha_calc, self.alpha_est,ss):
if(ac > 1.0): ac = 1.0
if(ae > 1.0): ae = 1.0
if(ac <= 0.0): ac = 1.e-6
if(ae <= 0.0): ae = 1.e-6
coeff = -4./(math.pi**3*ssi)
omega_calc.append( math.sqrt( math.log(ac) * coeff ) )
omega_est.append( math.sqrt( math.log(ae) * coeff ) )
omega_calc_ma = miller.array(miller_set= self.f_obs,data= flex.double(omega_calc))
omega_est_ma = miller.array(miller_set= self.f_obs,data= flex.double(omega_est))
if(self.flags.count(True) > 0):
omega_calc_ma_test = omega_calc_ma.select(self.flags)
omega_est_ma_test = omega_est_ma.select(self.flags)
alpha_calc_ma_test= alpha_calc_ma.select(self.flags)
beta_calc_ma_test= beta_calc_ma.select(self.flags)
alpha_est_ma_test= alpha_est_ma.select(self.flags)
beta_est_ma_test= beta_est_ma.select(self.flags)
if(self.flags.count(True) == 0):
omega_calc_ma_test = omega_calc_ma.select(~self.flags)
omega_est_ma_test = omega_est_ma.select(~self.flags)
alpha_calc_ma_test= alpha_calc_ma.select(~self.flags)
beta_calc_ma_test= beta_calc_ma.select(~self.flags)
alpha_est_ma_test= alpha_est_ma.select(~self.flags)
beta_est_ma_test= beta_est_ma.select(~self.flags)
alpha_calc_ma_test.setup_binner(
reflections_per_bin = self.free_reflections_per_bin)
beta_calc_ma_test.use_binning_of(alpha_calc_ma_test)
alpha_est_ma_test.use_binning_of(alpha_calc_ma_test)
beta_est_ma_test.use_binning_of(alpha_calc_ma_test)
omega_calc_ma_test.use_binning_of(alpha_calc_ma_test)
omega_est_ma_test.use_binning_of(alpha_calc_ma_test)
print " Resolution Estimated and calculated alpha, beta and model error"
print " d1 d2 nref alpha_e beta_e err_e alpha_c beta_c err_c"
for i_bin in alpha_calc_ma_test.binner().range_used():
sel = alpha_calc_ma_test.binner().selection(i_bin)
sel_alpha_calc_ma_test = alpha_calc_ma_test.select(sel)
sel_beta_calc_ma_test = beta_calc_ma_test.select(sel)
sel_alpha_est_ma_test = alpha_est_ma_test.select(sel)
sel_beta_est_ma_test = beta_est_ma_test.select(sel)
sel_omega_calc_ma_test = omega_calc_ma_test.select(sel)
sel_omega_est_ma_test = omega_est_ma_test.select(sel)
size = sel_alpha_calc_ma_test.data().size()
if(self.interpolation == False):
i=0
while i < size:
v_alpha_est = sel_alpha_est_ma_test.data()[i]
if(sel_alpha_est_ma_test.data().count(v_alpha_est) > size/2):
v_beta_est = sel_beta_est_ma_test.data()[i]
if(sel_beta_est_ma_test.data().count(v_beta_est) > size/2): break
i+=1
elif(self.interpolation == True):
v_alpha_est = flex.mean(sel_alpha_est_ma_test.data())
v_beta_est = flex.mean(sel_beta_est_ma_test.data())
alpha_calc = flex.mean(sel_alpha_calc_ma_test.data())
beta_calc = flex.mean(sel_beta_calc_ma_test.data())
omega_c = flex.mean(sel_omega_calc_ma_test.data())
omega_e = flex.mean(sel_omega_est_ma_test.data())
d1 = alpha_calc_ma_test.binner().bin_d_range(i_bin)[0]
d2 = alpha_calc_ma_test.binner().bin_d_range(i_bin)[1]
print "%8.4f %8.4f %5d %6.5f %12.3f %5.3f %6.5f %12.3f %5.3f" % \
(d1,d2,size,v_alpha_est,\
v_beta_est,omega_e,alpha_calc,beta_calc,omega_c)