def show(pdb_hierarchy, tm, xrs, grm, prefix):
map = compute_map(target_map=tm, xray_structure=xrs)
cc = flex.linear_correlation(
x=map.as_1d(),
y=tm.data.as_1d()).coefficient()
es = grm.energies_sites(sites_cart = xrs.sites_cart())
rmsd_a = es.angle_deviations()[2]
rmsd_b = es.bond_deviations()[2]
print "%s: overall CC: %6.4f rmsd_bonds=%6.3f rmsd_angles=%6.3f"%(
prefix, cc, rmsd_b, rmsd_a)
pdb_hierarchy.adopt_xray_structure(xrs)
rotamer_manager = RotamerEval()
for model in pdb_hierarchy.models():
for chain in model.chains():
for residue in chain.residues():
sites_cart = residue.atoms().extract_xyz()
sel = maptbx.grid_indices_around_sites(
unit_cell = xrs.unit_cell(),
fft_n_real = map.focus(),
fft_m_real = map.all(),
sites_cart = sites_cart,
site_radii = flex.double(sites_cart.size(), 2))
ccr = flex.linear_correlation(
x=map.select(sel).as_1d(),
y=tm.data.select(sel).as_1d()).coefficient()
fmt = "%s: %4s %10s CC: %6.4f"
print fmt%(prefix, residue.resname, rotamer_manager.evaluate_residue(residue),ccr)
def approx_curvs(O, shift_limit_factor=0.1):
x_on_entry = O.x
shifts = flex.double()
for ix,dsl,g in zip(count(), O.dynamic_shift_limits, O.grads):
shift = dsl.pair(x=O.x[ix]).get(grad=g) * shift_limit_factor
if (g > 0):
shift *= -1
assert shift != 0
shifts.append(shift)
if (0): print "shifts:", list(shifts)
grads_z = O.grads
O.x = x_on_entry + shifts
O.update_fgc()
grads_p = O.grads
O.x = x_on_entry - shifts
O.update_fgc()
grads_m = O.grads
O.x = x_on_entry
O.update_fgc()
O.xfgc_infos.pop()
O.xfgc_infos.pop()
O.xfgc_infos.pop()
apprx = (grads_p - grads_m) / (2*shifts)
if (0):
print "curvs:", list(O.curvs)
print "apprx:", list(apprx)
flex.linear_correlation(O.curvs, apprx).show_summary()
O.curvs = apprx
def eval_cc(f, merged_iobs):
print "reading",f
xac = xds_ascii.XDS_ASCII(f)
iobs = xac.i_obs(anomalous_flag=merged_iobs.anomalous_flag()).merge_equivalents(use_internal_variance=False).array()
n_all = iobs.size()
m, i = merged_iobs.common_sets(iobs, assert_is_similar_symmetry=False)
n_common = m.size()
corr = flex.linear_correlation(m.data(), i.data())
cc = corr.coefficient() if corr.is_well_defined() else float("nan")
ret1 = (n_all, n_common, cc)
ret2 = []
for frame in xrange(min(xac.iframe), max(xac.iframe)+1):
sel = xac.iframe == frame
iobs = xac.i_obs(anomalous_flag=merged_iobs.anomalous_flag()).select(sel)
iobs = iobs.merge_equivalents(use_internal_variance=False).array()
n_all = iobs.size()
m, i = merged_iobs.common_sets(iobs, assert_is_similar_symmetry=False)
n_common = m.size()
corr = flex.linear_correlation(m.data(), i.data())
cc = corr.coefficient() if corr.is_well_defined() else float("nan")
ret2.append([frame, n_all, n_common, cc])
return ret1, ret2
def approx_curvs(O, shift_limit_factor=0.1):
x_on_entry = O.x
shifts = flex.double()
for ix,dsl,g in zip(count(), O.dynamic_shift_limits, O.grads):
shift = dsl.pair(x=O.x[ix]).get(grad=g) * shift_limit_factor
if (g > 0):
shift *= -1
assert shift != 0
shifts.append(shift)
if (0): print "shifts:", list(shifts)
grads_z = O.grads
O.x = x_on_entry + shifts
O.update_fgc()
grads_p = O.grads
O.x = x_on_entry - shifts
O.update_fgc()
grads_m = O.grads
O.x = x_on_entry
O.update_fgc()
O.xfgc_infos.pop()
O.xfgc_infos.pop()
O.xfgc_infos.pop()
apprx = (grads_p - grads_m) / (2*shifts)
if (0):
print "curvs:", list(O.curvs)
print "apprx:", list(apprx)
flex.linear_correlation(O.curvs, apprx).show_summary()
O.curvs = apprx
def correlation(G_ref, I_ref, G_model, I_model, mi=None, plot=False):
if G_ref:
ref = open(G_ref)
I_ref_sub = []
for i in ref:
I_ref_sub.append(float(i))
R = flex.linear_correlation(flex.double(G_model), flex.double(I_ref_sub)).coefficient()
print "The Gm correlation: ", round(R, 5)
if I_ref:
ref = open(I_ref)
I_ref_sub = []
I_ref_match = []
for i in ref:
I_ref_sub.append(float(i))
if mi:
for j in mi:
I_ref_match.append(I_ref_sub[j])
else:
model_match = []
for i, j in zip(I_model, I_ref_sub):
if i != 0:
model_match.append(i)
I_ref_match.append(j)
I_model = model_match
R = flex.linear_correlation(flex.double(I_model), flex.double(I_ref_match)).coefficient()
print "The Ih correlation: ", round(R, 5)
def show(pdb_hierarchy, tm, xrs, grm, prefix):
map = compute_map(target_map=tm, xray_structure=xrs)
cc = flex.linear_correlation(x=map.as_1d(),
y=tm.data.as_1d()).coefficient()
es = grm.energies_sites(sites_cart=xrs.sites_cart())
rmsd_a = es.angle_deviations()[2]
rmsd_b = es.bond_deviations()[2]
print "%s: overall CC: %6.4f rmsd_bonds=%6.3f rmsd_angles=%6.3f" % (
prefix, cc, rmsd_b, rmsd_a)
pdb_hierarchy.adopt_xray_structure(xrs)
rotamer_manager = RotamerEval()
for model in pdb_hierarchy.models():
for chain in model.chains():
for residue in chain.residues():
sites_cart = residue.atoms().extract_xyz()
sel = maptbx.grid_indices_around_sites(
unit_cell=xrs.unit_cell(),
fft_n_real=map.focus(),
fft_m_real=map.all(),
sites_cart=sites_cart,
site_radii=flex.double(sites_cart.size(), 2))
ccr = flex.linear_correlation(
x=map.select(sel).as_1d(),
y=tm.data.select(sel).as_1d()).coefficient()
fmt = "%s: %4s %10s CC: %6.4f"
print fmt % (prefix, residue.resname,
rotamer_manager.evaluate_residue(residue), ccr)
def eval_cc_with_original_file(f, merged_iobs):
print "reading", f
xac = xds_ascii.XDS_ASCII(f)
iobs = xac.i_obs(
anomalous_flag=merged_iobs.anomalous_flag()).merge_equivalents(
use_internal_variance=False).array()
n_all = iobs.size()
m, i = merged_iobs.common_sets(iobs, assert_is_similar_symmetry=False)
n_common = m.size()
corr = flex.linear_correlation(m.data(), i.data())
cc = corr.coefficient() if corr.is_well_defined() else float("nan")
ret1 = (n_all, n_common, cc)
ret2 = []
for frame in xrange(min(xac.iframe), max(xac.iframe) + 1):
sel = xac.iframe == frame
iobs = xac.i_obs(
anomalous_flag=merged_iobs.anomalous_flag()).select(sel)
iobs = iobs.merge_equivalents(use_internal_variance=False).array()
n_all = iobs.size()
m, i = merged_iobs.common_sets(iobs, assert_is_similar_symmetry=False)
n_common = m.size()
corr = flex.linear_correlation(m.data(), i.data())
cc = corr.coefficient() if corr.is_well_defined() else float("nan")
ret2.append([frame, n_all, n_common, cc])
return ret1, ret2
def get_cc(mc1, mc2, xrs):
crystal_gridding = mc1.crystal_gridding(
d_min = mc1.d_min(),
symmetry_flags = maptbx.use_space_group_symmetry,
resolution_factor = 0.25)
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = mc1)
fft_map.apply_sigma_scaling()
m1 = fft_map.real_map_unpadded()
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = mc2)
fft_map.apply_sigma_scaling()
m2 = fft_map.real_map_unpadded()
assert m1.focus()==m2.focus()
assert m1.all()==m2.all()
sel = maptbx.grid_indices_around_sites(
unit_cell = mc1.unit_cell(),
fft_n_real = m1.focus(),
fft_m_real = m1.all(),
sites_cart = flex.vec3_double(xrs.sites_cart()),
site_radii = flex.double([1.5]*xrs.scatterers().size()))
cc = flex.linear_correlation(x=m1.select(sel), y=m2.select(sel)).coefficient()
def md(m, xrs):
r = flex.double()
for sf in xrs.sites_frac():
r.append(m.eight_point_interpolation(sf))
return flex.mean(r)
return cc, md(m=m1, xrs=xrs), md(m=m2, xrs=xrs)
def get_cc(a1, a2, title):
a1, a2 = a1.common_sets(a2)
corr = flex.linear_correlation(a1.data(), a2.data())
assert corr.is_well_defined()
#print "%30s CC= %.4f" %(title,
cc = corr.coefficient()
return cc**2 / (2 - cc**2)
def test1():
import libtbx.easy_pickle
import os.path
structure_ideal = libtbx.easy_pickle.load(os.path.expanduser("structure_ideal.pickle"))
structure_start = libtbx.easy_pickle.load(os.path.expanduser("structure_start.pickle"))
if 1:
structure_ideal.show_scatterers()
structure_start.show_scatterers()
rnd_f_calc = structure_ideal.structure_factors(
anomalous_flag=False, d_min=2.5, algorithm="direct", cos_sin_table=True
).f_calc()
y_obs = abs(rnd_f_calc)
structure_shake = structure_start.deep_copy_scatterers()
structure_shake_bis = structure_ideal.deep_copy_scatterers()
tst_xray_minimization.shift_u_aniso(structure_shake_bis, 0.001)
minimizer = xray.minimization.lbfgs(
target_functor=xray.target_functors.intensity_correlation(y_obs),
xray_structure=structure_shake,
use_special_position_constraints=False,
occupancy_penalty=None,
structure_factor_algorithm="direct",
)
assert minimizer.final_target_value < minimizer.first_target_value
if 1:
structure_shake.show_scatterers()
f_final = y_obs.structure_factors_from_scatterers(
xray_structure=structure_shake, algorithm="direct", cos_sin_table=True
).f_calc()
f_final = abs(f_final)
c = flex.linear_correlation(y_obs.data(), f_final.data())
assert c.is_well_defined()
c_coefficient = c.coefficient()
assert c_coefficient > 0.999
def linear_regression_test(d_analytical, d_numerical, test_hard=True,
slope_tolerance=1.e-3,
correlation_min=0.999,
verbose=0):
if (type(d_analytical) != type(flex.double())):
d_analytical = flex_tuple_as_flex_double(d_analytical)
if (type(d_numerical) != type(flex.double())):
d_numerical = flex_tuple_as_flex_double(d_numerical)
if (0 or verbose):
print "analytical:", tuple(d_analytical)
print "numerical: ", tuple(d_numerical)
if ( flex.max(flex.abs(d_analytical)) == 0
and flex.max(flex.abs(d_numerical)) == 0):
return
regr = flex.linear_regression(d_analytical, d_numerical)
corr = flex.linear_correlation(d_analytical, d_numerical).coefficient()
assert regr.is_well_defined()
if (abs(regr.slope() - 1) > slope_tolerance or corr < correlation_min):
print "Error: finite difference mismatch:"
print "slope:", regr.slope()
print "correlation:", corr
if (0 or verbose):
for a, n in zip(d_analytical, d_numerical):
print a, n
assert not test_hard
def anomalous_probability_plot(intensities, expected_delta=None):
from scitbx.math import distributions
from scitbx.array_family import flex
assert intensities.is_unique_set_under_symmetry()
assert intensities.anomalous_flag()
dI = intensities.anomalous_differences()
y = dI.data()/dI.sigmas()
perm = flex.sort_permutation(y)
y = y.select(perm)
distribution = distributions.normal_distribution()
x = distribution.quantiles(y.size())
if expected_delta is not None:
sel = flex.abs(x) < expected_delta
x = x.select(sel)
y = y.select(sel)
fit = flex.linear_regression(x, y)
correlation = flex.linear_correlation(x, y)
assert fit.is_well_defined()
if 0:
from matplotlib import pyplot
pyplot.scatter(x, y)
m = fit.slope()
c = fit.y_intercept()
pyplot.plot(pyplot.xlim(), [m * x_ + c for x_ in pyplot.xlim()])
pyplot.show()
return fit.slope(), fit.y_intercept(), x.size()
def run(lstin):
data = []
for l in open(lstin):
xdsasc = l.strip()
xa = XDS_ASCII(xdsasc, sys.stdout, i_only=True)
ma = miller.array(miller_set=xa.as_miller_set(anomalous_flag=False),
data=xa.iobs)
data.append((xdsasc, ma))
print "index filename"
for i, d in enumerate(data):
print i, d[0]
print "i j n.i n.j n.common cc"
for i in xrange(len(data)-1):
for j in xrange(i+1, len(data)):
di, dj = data[i][1].common_sets(data[j][1], assert_is_similar_symmetry=False)
print i, j, data[i][1].data().size(), data[j][1].data().size(),
if len(di.data()) == 0:
print 0, "nan"
else:
corr = flex.linear_correlation(di.data(), dj.data())
assert corr.is_well_defined()
cc = corr.coefficient()
print len(di.data()), cc
def cc_r1(fo, fc): lc = flex.linear_correlation(fo.data(), fc.data()) assert lc.is_well_defined() cc = lc.coefficient() from libtbx import Auto r1 = f_obs.r1_factor(other=f_calc, scale_factor=Auto) return cc, r1
def run(lstin):
data = []
for l in open(lstin):
xdsasc = l.strip()
xa = XDS_ASCII(xdsasc, sys.stdout, i_only=True)
ma = miller.array(miller_set=xa.as_miller_set(anomalous_flag=False),
data=xa.iobs)
data.append((xdsasc, ma))
print "index filename"
for i, d in enumerate(data):
print i, d[0]
print "i j n.i n.j n.common cc"
for i in xrange(len(data) - 1):
for j in xrange(i + 1, len(data)):
di, dj = data[i][1].common_sets(data[j][1],
assert_is_similar_symmetry=False)
print i, j, data[i][1].data().size(), data[j][1].data().size(),
if len(di.data()) == 0:
print 0, "nan"
else:
corr = flex.linear_correlation(di.data(), dj.data())
assert corr.is_well_defined()
cc = corr.coefficient()
print len(di.data()), cc
def exercise(space_group_info, anomalous_flag=False, d_min=2., verbose=0):
sg_fcalc = random_structure.xray_structure(
space_group_info,
elements=("N", "C", "C", "O"),
random_f_double_prime=anomalous_flag,
random_u_iso=True,
random_occupancy=True
).structure_factors(
anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct").f_calc()
sg_hl = generate_random_hl(sg_fcalc)
write_cns_input(sg_fcalc, sg_hl.data())
try: os.unlink("tmp_sg.hkl")
except OSError: pass
try: os.unlink("tmp_p1.hkl")
except OSError: pass
easy_run.fully_buffered(command="cns < tmp.cns > tmp.out") \
.raise_if_errors_or_output()
sg_cns = read_reflection_arrays("tmp_sg.hkl", anomalous_flag, verbose)
p1_cns = read_reflection_arrays("tmp_p1.hkl", anomalous_flag, verbose)
verify(sg_fcalc, sg_hl.data(), sg_cns, p1_cns)
if (anomalous_flag):
hl_merged = sg_hl.average_bijvoet_mates()
fc_merged = sg_fcalc.average_bijvoet_mates()
write_cns_input(sg_fcalc, sg_hl.data(), test_merge=True)
try: os.unlink("tmp_merged.hkl")
except OSError: pass
easy_run.fully_buffered(command="cns < tmp.cns > tmp.out") \
.raise_if_errors_or_output()
reflection_file = reflection_reader.cns_reflection_file(
open("tmp_merged.hkl"))
if (not sg_fcalc.space_group().is_centric()):
fc_merged_cns = reflection_file.reciprocal_space_objects["FCALC"]
fc_merged_cns = fc_merged.customized_copy(
indices=fc_merged_cns.indices,
data=fc_merged_cns.data).map_to_asu().common_set(fc_merged)
assert fc_merged_cns.indices().all_eq(fc_merged.indices())
fc_merged_a = fc_merged.select_acentric()
fc_merged_cns_a = fc_merged_cns.select_acentric()
for part in [flex.real, flex.imag]:
cc = flex.linear_correlation(
part(fc_merged_a.data()),
part(fc_merged_cns_a.data())).coefficient()
if (cc < 1-1.e-6):
print "FAILURE acentrics", sg_fcalc.space_group_info()
if (0): return
raise AssertionError
names, miller_indices, hl = reflection_file.join_hl_group()
assert names == ["PA", "PB", "PC", "PD"]
hl_merged_cns = hl_merged.customized_copy(indices=miller_indices, data=hl)\
.map_to_asu().common_set(hl_merged)
assert hl_merged_cns.indices().all_eq(hl_merged.indices())
for h,a,b in zip(hl_merged.indices(),
hl_merged.data(),
hl_merged_cns.data()):
if (not approx_equal(a, b, eps=5.e-3)):
print h
print "cctbx:", a
print " cns:", b
if (0): return
raise AssertionError
def calc_cc(ari, arj):
ari, arj = ari.common_sets(arj, assert_is_similar_symmetry=False)
corr = flex.linear_correlation(ari.data(), arj.data())
if corr.is_well_defined():
return corr.coefficient(), ari.size()
else:
return float("nan"), ari.size()
def scale_data(indices, iobs, scale_ref, parameter, calc_cc):
k, b, cc = 1, float("nan"), float("nan")
sortp = yamtbx_utils_ext.sort_permutation_fast_less(indices)
indices = indices.select(sortp)
iobs = iobs.select(sortp)
sel0, sel1 = yamtbx_utils_ext.my_common_indices(scale_ref.indices(), indices)
#indices = indices.select(sel1)
iobs_c = iobs.select(sel1)
ref_c = scale_ref.data().select(sel0)
if iobs_c.size() < 10 and ref_c.size() < 10:
return k, b, cc
if parameter == "k":
k = flex.sum(ref_c*iobs_c) / flex.sum(flex.pow2(iobs_c))
elif parameter == "kb":
from yamtbx.dataproc.scale_data import kBdecider
kbd = kBdecider(scale_ref,
miller.array(scale_ref.customized_copy(indices=indices),data=iobs))
k, b = kbd.run()
else:
raise "Never reaches here"
if calc_cc:
corr = flex.linear_correlation(ref_c, iobs_c)
if corr.is_well_defined(): cc = corr.coefficient()
return k, b, cc
def linear_regression_test(d_analytical,
d_numerical,
test_hard=True,
slope_tolerance=1.e-3,
correlation_min=0.999,
verbose=0):
if (type(d_analytical) != type(flex.double())):
d_analytical = flex_tuple_as_flex_double(d_analytical)
if (type(d_numerical) != type(flex.double())):
d_numerical = flex_tuple_as_flex_double(d_numerical)
if (0 or verbose):
print("analytical:", tuple(d_analytical))
print("numerical: ", tuple(d_numerical))
if (flex.max(flex.abs(d_analytical)) == 0
and flex.max(flex.abs(d_numerical)) == 0):
return
regr = flex.linear_regression(d_analytical, d_numerical)
corr = flex.linear_correlation(d_analytical, d_numerical).coefficient()
assert regr.is_well_defined()
if (abs(regr.slope() - 1) > slope_tolerance or corr < correlation_min):
print("Error: finite difference mismatch:")
print("slope:", regr.slope())
print("correlation:", corr)
if (0 or verbose):
for a, n in zip(d_analytical, d_numerical):
print(a, n)
assert not test_hard
def ccv(map_1, map_2, modified, centered, cutoff=None, n_bins=10000):
if (modified):
map_1 = volume_scale(map=map_1, n_bins=n_bins).map_data()
map_2 = volume_scale(map=map_2, n_bins=n_bins).map_data()
if (cutoff is not None):
map_1 = map_1 - cutoff
map_2 = map_2 - cutoff
s1 = map_1 < 0
s2 = map_2 < 0
map_1 = map_1.set_selected(s1, 0)
map_2 = map_2.set_selected(s2, 0)
def corr(x, y, centered):
s1 = x > 0
s2 = y > 0
s = s1 | s2
s = s.iselection()
x_ = x.select(s)
y_ = y.select(s)
return flex.linear_correlation(
x=x_, y=y_, subtract_mean=centered).coefficient()
return corr(x=map_1, y=map_2, centered=centered)
else:
return flex.linear_correlation(x=map_1.as_1d(),
y=map_2.as_1d(),
subtract_mean=centered).coefficient()
def __init__(self, params,
fo,
hl_coeffs,
ncs_object=None,
map_coeffs=None,
model_map_coeffs=None,
log=None,
as_gui_program=False):
self.model_map_coeffs = model_map_coeffs
self.correlation_coeffs = flex.double()
self.mean_phase_errors = flex.double()
density_modification.density_modification.__init__(
self, params, fo, hl_coeffs,
ncs_object=ncs_object,
map_coeffs=map_coeffs,
log=log,
as_gui_program=as_gui_program)
if len(self.correlation_coeffs) > 1:
model_coeffs, start_coeffs = self.model_map_coeffs.common_sets(self.map_coeffs_start)
model_fft_map = model_coeffs.fft_map(
resolution_factor=self.params.grid_resolution_factor).apply_sigma_scaling()
fft_map = start_coeffs.fft_map(
resolution_factor=self.params.grid_resolution_factor
).apply_sigma_scaling()
corr = flex.linear_correlation(
model_fft_map.real_map_unpadded().as_1d(), fft_map.real_map_unpadded().as_1d())
print "Starting dm/model correlation: %.6f" %corr.coefficient()
print "Final dm/model correlation: %.6f" %self.correlation_coeffs[-1]
fft_map.as_ccp4_map(file_name="starting.map", labels=[])
def calc_cc(obs, model, as_intensity=True):
if as_intensity:
obs, model = map(lambda x:x.as_intensity_array(), (obs, model))
corr = flex.linear_correlation(obs.data(), model.data())
if corr.is_well_defined(): return corr.coefficient()
else: return float("nan")
def get_cc(a1, a2, title):
a1, a2 = a1.common_sets(a2)
corr = flex.linear_correlation(a1.data(), a2.data())
assert corr.is_well_defined()
#print "%30s CC= %.4f" %(title,
cc = corr.coefficient()
return cc**2 / (2 - cc**2)
def ccano(hkl1, hkl2):
hkl1, hkl2 = hkl1.as_intensity_array(), hkl2.as_intensity_array()
corr = flex.linear_correlation(hkl1.anomalous_differences().data(),
hkl2.anomalous_differences().data())
assert corr.is_well_defined()
return corr.coefficient()
def ccano(hkl1, hkl2):
hkl1, hkl2 = hkl1.as_intensity_array(), hkl2.as_intensity_array()
corr = flex.linear_correlation(hkl1.anomalous_differences().data(),
hkl2.anomalous_differences().data())
assert corr.is_well_defined()
return corr.coefficient()
def __init__(self,
hooft_analysis,
use_students_t_distribution=False,
students_t_nu=None,
probability_plot_slope=None):
self.delta_fo2, minus_fo2 =\
hooft_analysis.delta_fo2.generate_bijvoet_mates().hemispheres_acentrics()
self.delta_fc2, minus_fc2 =\
hooft_analysis.delta_fc2.generate_bijvoet_mates().hemispheres_acentrics()
# we want to plot both hemispheres
self.delta_fo2.indices().extend(minus_fo2.indices())
self.delta_fo2.data().extend(minus_fo2.data() * -1)
self.delta_fo2.sigmas().extend(minus_fo2.sigmas())
self.delta_fc2.indices().extend(minus_fc2.indices())
self.delta_fc2.data().extend(minus_fc2.data() * -1)
self.indices = self.delta_fo2.indices()
observed_deviations = (hooft_analysis.G * self.delta_fc2.data()
- self.delta_fo2.data())/self.delta_fo2.sigmas()
if probability_plot_slope is not None:
observed_deviations /= probability_plot_slope
selection = flex.sort_permutation(observed_deviations)
observed_deviations = observed_deviations.select(selection)
if use_students_t_distribution:
if students_t_nu is None:
students_t_nu = maximise_students_t_correlation_coefficient(
observed_deviations, 1, 200)
self.distribution = distributions.students_t_distribution(students_t_nu)
else:
self.distribution = distributions.normal_distribution()
self.x = self.distribution.quantiles(observed_deviations.size())
self.y = observed_deviations
self.fit = flex.linear_regression(self.x[5:-5], self.y[5:-5])
self.correlation = flex.linear_correlation(self.x[5:-5], self.y[5:-5])
assert self.fit.is_well_defined()
def get_cc(mc1, mc2, xrs):
crystal_gridding = mc1.crystal_gridding(
d_min=mc1.d_min(),
symmetry_flags=maptbx.use_space_group_symmetry,
resolution_factor=0.25)
fft_map = miller.fft_map(crystal_gridding=crystal_gridding,
fourier_coefficients=mc1)
fft_map.apply_sigma_scaling()
m1 = fft_map.real_map_unpadded()
fft_map = miller.fft_map(crystal_gridding=crystal_gridding,
fourier_coefficients=mc2)
fft_map.apply_sigma_scaling()
m2 = fft_map.real_map_unpadded()
assert m1.focus() == m2.focus()
assert m1.all() == m2.all()
sel = maptbx.grid_indices_around_sites(
unit_cell=mc1.unit_cell(),
fft_n_real=m1.focus(),
fft_m_real=m1.all(),
sites_cart=flex.vec3_double(xrs.sites_cart()),
site_radii=flex.double([1.5] * xrs.scatterers().size()))
cc = flex.linear_correlation(x=m1.select(sel),
y=m2.select(sel)).coefficient()
def md(m, xrs):
r = flex.double()
for sf in xrs.sites_frac():
r.append(m.eight_point_interpolation(sf))
return flex.mean(r)
return cc, md(m=m1, xrs=xrs), md(m=m2, xrs=xrs)
def anomalous_probability_plot(intensities, expected_delta=None):
from scitbx.math import distributions
from scitbx.array_family import flex
assert intensities.is_unique_set_under_symmetry()
assert intensities.anomalous_flag()
dI = intensities.anomalous_differences()
y = dI.data() / dI.sigmas()
perm = flex.sort_permutation(y)
y = y.select(perm)
distribution = distributions.normal_distribution()
x = distribution.quantiles(y.size())
if expected_delta is not None:
sel = flex.abs(x) < expected_delta
x = x.select(sel)
y = y.select(sel)
fit = flex.linear_regression(x, y)
correlation = flex.linear_correlation(x, y)
assert fit.is_well_defined()
if 0:
from matplotlib import pyplot
pyplot.scatter(x, y)
m = fit.slope()
c = fit.y_intercept()
pyplot.plot(pyplot.xlim(), [m * x_ + c for x_ in pyplot.xlim()])
pyplot.show()
return fit.slope(), fit.y_intercept(), x.size()
def calc_cc(ari, arj):
ari, arj = ari.common_sets(arj, assert_is_similar_symmetry=False)
corr = flex.linear_correlation(ari.data(), arj.data())
if corr.is_well_defined():
return corr.coefficient(), ari.size()
else:
return float("nan"), ari.size()
def __init__(self,
hooft_analysis,
use_students_t_distribution=False,
students_t_nu=None,
probability_plot_slope=None):
self.delta_fo2, minus_fo2 =\
hooft_analysis.delta_fo2.generate_bijvoet_mates().hemispheres_acentrics()
self.delta_fc2, minus_fc2 =\
hooft_analysis.delta_fc2.generate_bijvoet_mates().hemispheres_acentrics()
# we want to plot both hemispheres
self.delta_fo2.indices().extend(minus_fo2.indices())
self.delta_fo2.data().extend(minus_fo2.data() * -1)
self.delta_fo2.sigmas().extend(minus_fo2.sigmas())
self.delta_fc2.indices().extend(minus_fc2.indices())
self.delta_fc2.data().extend(minus_fc2.data() * -1)
self.indices = self.delta_fo2.indices()
observed_deviations = (hooft_analysis.G * self.delta_fc2.data()
- self.delta_fo2.data())/self.delta_fo2.sigmas()
if probability_plot_slope is not None:
observed_deviations /= probability_plot_slope
selection = flex.sort_permutation(observed_deviations)
observed_deviations = observed_deviations.select(selection)
if use_students_t_distribution:
if students_t_nu is None:
students_t_nu = maximise_students_t_correlation_coefficient(
observed_deviations, 1, 200)
self.distribution = distributions.students_t_distribution(students_t_nu)
else:
self.distribution = distributions.normal_distribution()
self.x = self.distribution.quantiles(observed_deviations.size())
self.y = observed_deviations
self.fit = flex.linear_regression(self.x[5:-5], self.y[5:-5])
self.correlation = flex.linear_correlation(self.x[5:-5], self.y[5:-5])
assert self.fit.is_well_defined()
def analyze_map(self,
map,
model_map=None,
min=None,
max=None,
compare_at_sites_only=False):
"""
Extract statistics for the given map, plus statistics for the model map
if given. The CC can either be calculated across grid points within the
given radius of the sites, or at the sites directly.
"""
assert (map.focus() == self.fft_n_real) and (map.all()
== self.fft_m_real)
map_sel = map.select(self.map_sel)
map_values = flex.double()
model_map_sel = model_map_mean = model_map_values = None
if (model_map is not None):
assert ((model_map.focus() == self.fft_n_real)
and (model_map.all() == self.fft_m_real))
model_map_sel = model_map.select(self.map_sel)
model_map_values = flex.double()
for site_frac in self.sites_frac:
map_values.append(map.eight_point_interpolation(site_frac))
if (model_map is not None):
model_map_values.append(
model_map.eight_point_interpolation(site_frac))
cc = None
if (model_map is not None):
if (compare_at_sites_only):
cc = flex.linear_correlation(x=map_values,
y=model_map_values).coefficient()
else:
cc = flex.linear_correlation(x=map_sel,
y=model_map_sel).coefficient()
model_map_mean = flex.mean(model_map_values)
n_above_max = n_below_min = None
if (min is not None):
n_below_min = (map_values < min).count(True)
if (max is not None):
n_above_max = (map_values > max).count(True)
return group_args(cc=cc,
min=flex.min(map_values),
max=flex.max(map_values),
mean=flex.mean(map_values),
n_below_min=n_below_min,
n_above_max=n_above_max,
model_mean=model_map_mean)
def __init__(self, f_obs, asu_contents, e_statistics=False):
assert f_obs.is_real_array()
self.info = f_obs.info()
f_obs_selected = f_obs.select(f_obs.data() > 0)
f_obs_selected.use_binning_of(f_obs)
# compute <fobs^2> in resolution shells
self.mean_fobs_sq = f_obs_selected.mean_sq(
use_binning=True,
use_multiplicities=True).data[1:-1]
n_none = self.mean_fobs_sq.count(None)
if (n_none > 0):
error_message = "wilson_plot error: %d empty bin%s:" % plural_s(n_none)
if (self.info is not None):
error_message += "\n Info: " + str(self.info)
error_message += "\n Number of bins: %d" % len(self.mean_fobs_sq)
error_message += "\n Number of f_obs > 0: %d" % (
f_obs_selected.indices().size())
error_message += "\n Number of f_obs <= 0: %d" % (
f_obs.indices().size() - f_obs_selected.indices().size())
raise RuntimeError(error_message)
self.mean_fobs_sq = flex.double(self.mean_fobs_sq)
# compute <s^2> = <(sin(theta)/lambda)^2> in resolution shells
stol_sq = f_obs_selected.sin_theta_over_lambda_sq()
stol_sq.use_binner_of(f_obs_selected)
self.mean_stol_sq = flex.double(stol_sq.mean(
use_binning=True,
use_multiplicities=True).data[1:-1])
# cache scattering factor info
gaussians = {}
for chemical_type in asu_contents.keys():
gaussians[chemical_type] = eltbx.xray_scattering.wk1995(
chemical_type).fetch()
# compute expected f_calc^2 in resolution shells
self.expected_f_sq = flex.double()
for stol_sq in self.mean_stol_sq:
sum_fj_sq = 0
for chemical_type, n_atoms in asu_contents.items():
f0 = gaussians[chemical_type].at_stol_sq(stol_sq)
sum_fj_sq += f0 * f0 * n_atoms
self.expected_f_sq.append(sum_fj_sq)
self.expected_f_sq *= f_obs_selected.space_group().order_z() \
* f_obs_selected.space_group().n_ltr()
# fit to straight line
self.x = self.mean_stol_sq
self.y = flex.log(self.mean_fobs_sq / self.expected_f_sq)
fit = flex.linear_regression(self.x, self.y)
assert fit.is_well_defined()
self.fit_y_intercept = fit.y_intercept()
self.fit_slope = fit.slope()
self.wilson_intensity_scale_factor = math.exp(self.fit_y_intercept) # intensity scale factor
self.wilson_k = math.sqrt(self.wilson_intensity_scale_factor) # conversion to amplitude scale factor
self.wilson_b = -self.fit_slope / 2
self.fit_correlation = flex.linear_correlation(self.x,self.y).coefficient()
if e_statistics:
normalised = f_obs_selected.normalised_amplitudes(asu_contents, self)
self.normalised_f_obs = normalised.array()
self.mean_e_sq_minus_1 = normalised.mean_e_sq_minus_1()
self.percent_e_sq_gt_2 = normalised.percent_e_sq_gt_2()
def calc_cc(a1, a2):
a1, a2 = a1.common_sets(a2, assert_is_similar_symmetry=False)
corr = flex.linear_correlation(a1.data(), a2.data())
if corr.is_well_defined():# and a1.size() > 20:
return corr.coefficient()
else:
return float("nan")
def new_cc(a, b):
'''Compute CC between miller arrays a and b.'''
assert a.is_real_array()
assert b.is_real_array()
_a, _b = a.common_sets(other = b, assert_is_similar_symmetry = True)
return flex.linear_correlation(_a.data(), _b.data())
def cc_r1(fo, fc):
lc = flex.linear_correlation(fo.data(), fc.data())
assert lc.is_well_defined()
cc = lc.coefficient()
from libtbx import Auto
r1 = f_obs.r1_factor(other=f_calc, scale_factor=Auto)
return cc, r1
def check_regression(x, y, label, min_correlation=0, verbose=0):
xy_regr = flex.linear_regression(x, y)
xy_corr = flex.linear_correlation(x, y)
assert xy_regr.is_well_defined()
if (0 or verbose):
print label, "correlation: %.4f slope: %.3f" % (
xy_corr.coefficient(), xy_regr.slope())
assert min_correlation == 0 or xy_corr.coefficient() >= min_correlation
def check_regression(x, y, label, min_correlation=0, verbose=0):
xy_regr = flex.linear_regression(x, y)
xy_corr = flex.linear_correlation(x, y)
assert xy_regr.is_well_defined()
if (0 or verbose):
print label, "correlation: %.4f slope: %.3f" % (xy_corr.coefficient(),
xy_regr.slope())
assert min_correlation == 0 or xy_corr.coefficient() >= min_correlation
def calc_cc(a1, a2):
a1, a2 = a1.common_sets(a2, assert_is_similar_symmetry=False)
corr = flex.linear_correlation(a1.data(), a2.data())
if corr.is_well_defined(): # and a1.size() > 20:
return corr.coefficient()
else:
return float("nan")
def __init__(
self,
fmodel,
coeffs):
# XXX see f_model.py: duplication! Consolidate.
self.fmodel = fmodel
self.coeffs = coeffs
crystal_gridding = fmodel.f_obs().crystal_gridding(
d_min = self.fmodel.f_obs().d_min(),
resolution_factor = 1./3)
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = self.coeffs)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
rho_atoms = flex.double()
for site_frac in self.fmodel.xray_structure.sites_frac():
rho_atoms.append(map_data.eight_point_interpolation(site_frac))
rho_mean = flex.mean_default(rho_atoms.select(rho_atoms>0.5), 0.5)
sel_exclude = rho_atoms > min(rho_mean/2., 1)
sites_cart = fmodel.xray_structure.sites_cart()
#
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = self.fmodel.f_model())
fft_map.apply_sigma_scaling()
map_data2 = fft_map.real_map_unpadded()
#
for i_seq, site_cart in enumerate(sites_cart):
selection = maptbx.grid_indices_around_sites(
unit_cell = self.coeffs.unit_cell(),
fft_n_real = map_data.focus(),
fft_m_real = map_data.all(),
sites_cart = flex.vec3_double([site_cart]),
site_radii = flex.double([1.5]))
cc = flex.linear_correlation(x=map_data.select(selection),
y=map_data2.select(selection)).coefficient()
if(cc<0.7): sel_exclude[i_seq] = False
#
del map_data, fft_map, rho_atoms
self.d_min = fmodel.f_obs().d_min()
cs = self.fmodel.f_obs().average_bijvoet_mates().complete_set(d_min=self.d_min)
self.complete_set = cs.array(data = flex.double(cs.indices().size(), 0))
self.xray_structure_cut = self.fmodel.xray_structure.select(sel_exclude)
self.missing_set = self.complete_set.common_set(self.coeffs)
#
self.f_calc_missing = self.complete_set.structure_factors_from_scatterers(
xray_structure = self.xray_structure_cut).f_calc()
self.ss_missing = 1./flex.pow2(self.f_calc_missing.d_spacings().data()) / 4.
mask_manager = mmtbx.masks.manager(
miller_array = self.f_calc_missing,
miller_array_twin = None,
mask_params = None)
self.f_mask_missing = mask_manager.shell_f_masks(
xray_structure = self.xray_structure_cut,
force_update = True)
self.zero_data = flex.complex_double(self.f_calc_missing.data().size(), 0)
def corr(x, y, centered):
s1 = x > 0
s2 = y > 0
s = s1 | s2
s = s.iselection()
x_ = x.select(s)
y_ = y.select(s)
return flex.linear_correlation(
x=x_, y=y_, subtract_mean=centered).coefficient()
def corr(x, y, centered):
s1 = x > 0
s2 = y > 0
s = s1 | s2
s = s.iselection()
x_ = x.select(s)
y_ = y.select(s)
return flex.linear_correlation(x = x_, y = y_,
subtract_mean = centered).coefficient()
def __init__(
self,
fmodel,
coeffs):
# XXX see f_model.py: duplication! Consolidate.
self.fmodel = fmodel
self.coeffs = coeffs
crystal_gridding = fmodel.f_obs().crystal_gridding(
d_min = self.fmodel.f_obs().d_min(),
resolution_factor = 1./3)
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = self.coeffs)
fft_map.apply_sigma_scaling()
map_data = fft_map.real_map_unpadded()
rho_atoms = flex.double()
for site_frac in self.fmodel.xray_structure.sites_frac():
rho_atoms.append(map_data.eight_point_interpolation(site_frac))
rho_mean = flex.mean_default(rho_atoms.select(rho_atoms>0.5), 0.5)
sel_exclude = rho_atoms > min(rho_mean/2., 1)
sites_cart = fmodel.xray_structure.sites_cart()
#
fft_map = miller.fft_map(
crystal_gridding = crystal_gridding,
fourier_coefficients = self.fmodel.f_model())
fft_map.apply_sigma_scaling()
map_data2 = fft_map.real_map_unpadded()
#
for i_seq, site_cart in enumerate(sites_cart):
selection = maptbx.grid_indices_around_sites(
unit_cell = self.coeffs.unit_cell(),
fft_n_real = map_data.focus(),
fft_m_real = map_data.all(),
sites_cart = flex.vec3_double([site_cart]),
site_radii = flex.double([1.5]))
cc = flex.linear_correlation(x=map_data.select(selection),
y=map_data2.select(selection)).coefficient()
if(cc<0.7): sel_exclude[i_seq] = False
#
del map_data, fft_map, rho_atoms
self.d_min = fmodel.f_obs().d_min()
cs = self.fmodel.f_obs().average_bijvoet_mates().complete_set(d_min=self.d_min)
self.complete_set = cs.array(data = flex.double(cs.indices().size(), 0))
self.xray_structure_cut = self.fmodel.xray_structure.select(sel_exclude)
self.missing_set = self.complete_set.common_set(self.coeffs)
#
self.f_calc_missing = self.complete_set.structure_factors_from_scatterers(
xray_structure = self.xray_structure_cut).f_calc()
self.ss_missing = 1./flex.pow2(self.f_calc_missing.d_spacings().data()) / 4.
mask_manager = mmtbx.masks.manager(
miller_array = self.f_calc_missing,
miller_array_twin = None,
mask_params = None)
self.f_mask_missing = mask_manager.shell_f_masks(
xray_structure = self.xray_structure_cut,
force_update = True)
self.zero_data = flex.complex_double(self.f_calc_missing.data().size(), 0)
def simple(fmodel, pdb_hierarchy, params=None, log=None, show_results=False):
if(params is None): params =master_params().extract()
if(log is None): log = sys.stdout
# compute map coefficients
e_map_obj = fmodel.electron_density_map()
coeffs_1 = e_map_obj.map_coefficients(
map_type = params.map_1.type,
fill_missing = params.map_1.fill_missing_reflections,
isotropize = params.map_1.isotropize)
coeffs_2 = e_map_obj.map_coefficients(
map_type = params.map_2.type,
fill_missing = params.map_2.fill_missing_reflections,
isotropize = params.map_2.isotropize)
# compute maps
fft_map_1 = coeffs_1.fft_map(resolution_factor = params.resolution_factor)
fft_map_1.apply_sigma_scaling()
map_1 = fft_map_1.real_map_unpadded()
fft_map_2 = miller.fft_map(
crystal_gridding = fft_map_1,
fourier_coefficients = coeffs_2)
fft_map_2.apply_sigma_scaling()
map_2 = fft_map_2.real_map_unpadded()
# compute cc
broadcast(m="Map correlation and map values", log=log)
overall_cc = flex.linear_correlation(x = map_1.as_1d(),
y = map_2.as_1d()).coefficient()
print >> log, " Overall map cc(%s,%s): %6.4f"%(params.map_1.type,
params.map_2.type, overall_cc)
detail, atom_radius = params.detail, params.atom_radius
detail, atom_radius = set_detail_level_and_radius(detail=detail,
atom_radius=atom_radius, d_min=fmodel.f_obs().d_min())
use_hydrogens = params.use_hydrogens
if(use_hydrogens is None):
if(params.scattering_table == "neutron" or fmodel.f_obs().d_min() <= 1.2):
use_hydrogens = True
else:
use_hydrogens = False
hydrogen_atom_radius = params.hydrogen_atom_radius
if(hydrogen_atom_radius is None):
if(params.scattering_table == "neutron"):
hydrogen_atom_radius = atom_radius
else:
hydrogen_atom_radius = 1
results = compute(
pdb_hierarchy = pdb_hierarchy,
unit_cell = fmodel.xray_structure.unit_cell(),
fft_n_real = map_1.focus(),
fft_m_real = map_1.all(),
map_1 = map_1,
map_2 = map_2,
detail = detail,
atom_radius = atom_radius,
use_hydrogens = use_hydrogens,
hydrogen_atom_radius = hydrogen_atom_radius)
if(show_results):
show(log=log, results=results, params=params, detail=detail)
return results
def analyze_map (self, map, model_map=None, min=None, max=None,
compare_at_sites_only=False) :
"""
Extract statistics for the given map, plus statistics for the model map
if given. The CC can either be calculated across grid points within the
given radius of the sites, or at the sites directly.
"""
assert (map.focus() == self.fft_n_real) and (map.all() == self.fft_m_real)
map_sel = map.select(self.map_sel)
map_values = flex.double()
model_map_sel = model_map_mean = model_map_values = None
if (model_map is not None) :
assert ((model_map.focus() == self.fft_n_real) and
(model_map.all() == self.fft_m_real))
model_map_sel = model_map.select(self.map_sel)
model_map_values = flex.double()
for site_frac in self.sites_frac:
map_values.append(map.eight_point_interpolation(site_frac))
if (model_map is not None) :
model_map_values.append(model_map.eight_point_interpolation(site_frac))
cc = None
if (model_map is not None) :
if (compare_at_sites_only) :
cc = flex.linear_correlation(x=map_values,
y=model_map_values).coefficient()
else :
cc = flex.linear_correlation(x=map_sel, y=model_map_sel).coefficient()
model_map_mean = flex.mean(model_map_values)
n_above_max = n_below_min = None
if (min is not None) :
n_below_min = (map_values < min).count(True)
if (max is not None) :
n_above_max = (map_values > max).count(True)
return group_args(
cc=cc,
min=flex.min(map_values),
max=flex.max(map_values),
mean=flex.mean(map_values),
n_below_min=n_below_min,
n_above_max=n_above_max,
model_mean=model_map_mean)
def is_refinement_needed(self, residue_group, residue, cc_limit, ignore_hd):
result = False
if([self.sa, self.r, self.rot].count(None)==0):
is_outlier, value = self.rot.evaluate_residue(residue_group)
if(is_outlier): return True
for atom in residue.atoms():
if(not atom.element.strip().lower() in ["h","d"]):
sel_map = self.select(sites_cart = flex.vec3_double([atom.xyz]))
m1 = self.target_map_data.select(sel_map)
m2 = self.model_map_data.select(sel_map)
cc = flex.linear_correlation(x = m1, y = m2).coefficient()
if(cc < cc_limit): return True
return result
def print_validation(log, results, debug, pdb_hierarchy_selected):
box_1 = results.box_1
box_2 = results.box_2
box_3 = results.box_3
sites_cart_box = box_1.xray_structure_box.sites_cart()
sel = maptbx.grid_indices_around_sites(
unit_cell=box_1.xray_structure_box.unit_cell(),
fft_n_real=box_1.map_box.focus(),
fft_m_real=box_1.map_box.all(),
sites_cart=sites_cart_box,
site_radii=flex.double(sites_cart_box.size(), 2.0))
b1 = box_1.map_box.select(sel).as_1d()
b2 = box_2.map_box.select(sel).as_1d()
b3 = box_3.map_box.select(sel).as_1d()
print >> log, "Map 1: calculated Fobs with ligand"
print >> log, "Map 2: calculated Fobs without ligand"
print >> log, "Map 3: real Fobs data"
cc12 = flex.linear_correlation(x=b1, y=b2).coefficient()
cc13 = flex.linear_correlation(x=b1, y=b3).coefficient()
cc23 = flex.linear_correlation(x=b2, y=b3).coefficient()
print >> log, "CC(1,2): %6.4f" % cc12
print >> log, "CC(1,3): %6.4f" % cc13
print >> log, "CC(2,3): %6.4f" % cc23
#### D-function
b1 = maptbx.volume_scale_1d(map=b1, n_bins=10000).map_data()
b2 = maptbx.volume_scale_1d(map=b2, n_bins=10000).map_data()
b3 = maptbx.volume_scale_1d(map=b3, n_bins=10000).map_data()
print >> log, "Peak CC:"
print >> log, "CC(1,2): %6.4f" % flex.linear_correlation(
x=b1, y=b2).coefficient()
print >> log, "CC(1,3): %6.4f" % flex.linear_correlation(
x=b1, y=b3).coefficient()
print >> log, "CC(2,3): %6.4f" % flex.linear_correlation(
x=b2, y=b3).coefficient()
cutoffs = flex.double([i / 10. for i in range(1, 10)] +
[i / 100 for i in range(91, 100)])
d12 = maptbx.discrepancy_function(map_1=b1, map_2=b2, cutoffs=cutoffs)
d13 = maptbx.discrepancy_function(map_1=b1, map_2=b3, cutoffs=cutoffs)
d23 = maptbx.discrepancy_function(map_1=b2, map_2=b3, cutoffs=cutoffs)
print >> log, "q D(1,2) D(1,3) D(2,3)"
for c, d12_, d13_, d23_ in zip(cutoffs, d12, d13, d23):
print >> log, "%4.2f %6.4f %6.4f %6.4f" % (c, d12_, d13_, d23_)
###
if (debug):
#box_1.write_ccp4_map(file_name="box_1_polder.ccp4")
#box_2.write_ccp4_map(file_name="box_2_polder.ccp4")
#box_3.write_ccp4_map(file_name="box_3_polder.ccp4")
write_map_box(box=box_1, filename="box_1_polder.ccp4")
write_map_box(box=box_2, filename="box_2_polder.ccp4")
write_map_box(box=box_3, filename="box_3_polder.ccp4")
pdb_hierarchy_selected.adopt_xray_structure(box_1.xray_structure_box)
pdb_hierarchy_selected.write_pdb_file(
file_name="box_polder.pdb",
crystal_symmetry=box_1.box_crystal_symmetry)
#
print >> log, '*' * 79
message = result_message(cc12=cc12, cc13=cc13, cc23=cc23)
print >> log, message
return message
def __init__(self, f_obs, asu_contents, e_statistics=False):
assert f_obs.is_real_array()
self.info = f_obs.info()
f_obs_selected = f_obs.select(f_obs.data() > 0)
f_obs_selected.use_binning_of(f_obs)
# compute <fobs^2> in resolution shells
self.mean_fobs_sq = f_obs_selected.mean_sq(use_binning=True, use_multiplicities=True).data[1:-1]
n_none = self.mean_fobs_sq.count(None)
if n_none > 0:
error_message = "wilson_plot error: %d empty bin%s:" % plural_s(n_none)
if self.info is not None:
error_message += "\n Info: " + str(self.info)
error_message += "\n Number of bins: %d" % len(self.mean_fobs_sq)
error_message += "\n Number of f_obs > 0: %d" % (f_obs_selected.indices().size())
error_message += "\n Number of f_obs <= 0: %d" % (f_obs.indices().size() - f_obs_selected.indices().size())
raise RuntimeError(error_message)
self.mean_fobs_sq = flex.double(self.mean_fobs_sq)
# compute <s^2> = <(sin(theta)/lambda)^2> in resolution shells
stol_sq = f_obs_selected.sin_theta_over_lambda_sq()
stol_sq.use_binner_of(f_obs_selected)
self.mean_stol_sq = flex.double(stol_sq.mean(use_binning=True, use_multiplicities=True).data[1:-1])
# cache scattering factor info
gaussians = {}
for chemical_type in asu_contents.keys():
gaussians[chemical_type] = eltbx.xray_scattering.wk1995(chemical_type).fetch()
# compute expected f_calc^2 in resolution shells
self.expected_f_sq = flex.double()
for stol_sq in self.mean_stol_sq:
sum_fj_sq = 0
for chemical_type, n_atoms in asu_contents.items():
f0 = gaussians[chemical_type].at_stol_sq(stol_sq)
sum_fj_sq += f0 * f0 * n_atoms
self.expected_f_sq.append(sum_fj_sq)
self.expected_f_sq *= f_obs_selected.space_group().order_z() * f_obs_selected.space_group().n_ltr()
# fit to straight line
self.x = self.mean_stol_sq
self.y = flex.log(self.mean_fobs_sq / self.expected_f_sq)
fit = flex.linear_regression(self.x, self.y)
assert fit.is_well_defined()
self.fit_y_intercept = fit.y_intercept()
self.fit_slope = fit.slope()
self.wilson_intensity_scale_factor = math.exp(self.fit_y_intercept) # intensity scale factor
self.wilson_k = math.sqrt(self.wilson_intensity_scale_factor) # conversion to amplitude scale factor
self.wilson_b = -self.fit_slope / 2
self.fit_correlation = flex.linear_correlation(self.x, self.y).coefficient()
if e_statistics:
normalised = f_obs_selected.normalised_amplitudes(asu_contents, self)
self.normalised_f_obs = normalised.array()
self.mean_e_sq_minus_1 = normalised.mean_e_sq_minus_1()
self.percent_e_sq_gt_2 = normalised.percent_e_sq_gt_2()
def eval_cc_internal(merged_iobs, merged, setno):
"""
merged_iobs: i_obs array where all data in xscale hkl were merged
merged: xscale hkl object
setno: iset number in interest
"""
merged_sel = copy.copy(merged)
merged_sel.remove_selection(merged.iset != setno)
iobs_set = merged_sel.i_obs(anomalous_flag=merged_iobs.anomalous_flag())
iobs_set_merged = iobs_set.merge_equivalents(
use_internal_variance=False).array()
n_all = iobs_set_merged.size()
m, i = merged_iobs.common_sets(iobs_set_merged,
assert_is_similar_symmetry=False)
n_common = m.size()
corr = flex.linear_correlation(m.data(), i.data())
cc = corr.coefficient() if corr.is_well_defined() else float("nan")
ret1 = (n_all, n_common, cc)
ret2 = []
for frame in xrange(min(merged_sel.iframe), max(merged_sel.iframe) + 1):
iobs = iobs_set.select(merged_sel.iframe == frame)
iobs = iobs.merge_equivalents(use_internal_variance=False).array()
n_all = iobs.size()
m, i = merged_iobs.common_sets(iobs, assert_is_similar_symmetry=False)
n_common = m.size()
corr = flex.linear_correlation(m.data(), i.data())
cc = corr.coefficient() if corr.is_well_defined() else float("nan")
ret2.append([frame, n_all, n_common, cc])
return ret1, ret2
def compute(pdb_hierarchy,
unit_cell,
map_1,
map_2,
map_3,
detail,
atom_radius) :
results = []
for chain in pdb_hierarchy.chains():
for residue_group in chain.residue_groups():
for conformer in residue_group.conformers():
for residue in conformer.residues():
r_id_str = "%2s %1s %3s %4s %1s"%(chain.id, conformer.altloc,
residue.resname, residue.resseq, residue.icode)
for atom in residue.atoms():
a_id_str = "%s %4s"%(r_id_str, atom.name)
rad = atom_radius
if not atom.element_is_hydrogen() :
map_value_1 = map_1.eight_point_interpolation(
unit_cell.fractionalize(atom.xyz))
map_value_2 = map_2.eight_point_interpolation(
unit_cell.fractionalize(atom.xyz))
map_value_3 = map_3.eight_point_interpolation(
unit_cell.fractionalize(atom.xyz))
sel = maptbx.grid_indices_around_sites(
unit_cell = unit_cell,
fft_n_real = map_1.focus(),
fft_m_real = map_1.all(),
sites_cart = flex.vec3_double([atom.xyz]),
site_radii = flex.double([atom_radius]))
cc = flex.linear_correlation(x=map_1.select(sel),
y=map_2.select(sel)).coefficient()
result = group_args(
chain_id = chain.id,
res_num = residue.resseq,
res_name = residue.resname,
atom_name = atom.name,
alt_loc = conformer.altloc,
ins_code = residue.icode,
atom = atom,
id_str = a_id_str,
cc = cc,
map_value_1 = map_value_1,
map_value_2 = map_value_2,
map_value_3 = map_value_3,
b = atom.b,
occupancy = atom.occ,
n_atoms = 1)
results.append(result)
return results
def is_refinement_needed(self, residue_group, residue, cc_limit,
ignore_hd):
result = False
if ([self.sa, self.r, self.rot].count(None) == 0):
is_outlier, value = self.rot.evaluate_residue(residue_group)
if (is_outlier): return True
for atom in residue.atoms():
if (not atom.element.strip().lower() in ["h", "d"]):
sel_map = self.select(sites_cart=flex.vec3_double([atom.xyz]))
m1 = self.target_map_data.select(sel_map)
m2 = self.model_map_data.select(sel_map)
cc = flex.linear_correlation(x=m1, y=m2).coefficient()
if (cc < cc_limit): return True
return result
def get_cc_anom(self):
cc_anom_acentric, nrefl_anom_acentric = (0, 0)
if self.miller_array_merge.anomalous_flag():
ma_anom_dif_even = self.miller_array_merge.customized_copy(
data=self.I_even).anomalous_differences()
ma_anom_dif_odd = self.miller_array_merge.customized_copy(
data=self.I_odd).anomalous_differences()
i_acentric = (ma_anom_dif_even.centric_flags().data() == False)
cc_anom_acentric = flex.linear_correlation(
ma_anom_dif_even.data().select(i_acentric),
ma_anom_dif_odd.data().select(i_acentric)).coefficient()
nrefl_anom_acentric = len(
ma_anom_dif_even.data().select(i_acentric))
return cc_anom_acentric, nrefl_anom_acentric
def run(mtz1, mtz2):
# read in mtz files
reflection_file_1 = reflection_file_reader.any_reflection_file(mtz1)
miller_arrays_1 = reflection_file_1.as_miller_arrays()
reflection_file_2 = reflection_file_reader.any_reflection_file(mtz2)
miller_arrays_2 = reflection_file_2.as_miller_arrays()
ma_anom_diff_1 = miller_arrays_1[0].anomalous_differences()
ma_anom_diff_2 = miller_arrays_2[0].anomalous_differences()
ma_anom_diff_1.show_summary()
ma_anom_diff_2.show_summary()
ma_anom_diff_1_cs = ma_anom_diff_1.common_set(ma_anom_diff_2)
ma_anom_diff_2_cs = ma_anom_diff_2.common_set(ma_anom_diff_1)
# Make sure the 2 arrays have the same size
cc_anom = flex.linear_correlation(ma_anom_diff_1_cs.data(),
ma_anom_diff_2_cs.data()).coefficient()
print('Value of CC_anom for the dataset is = ', cc_anom)
def get_cciso(self, miller_array_iso):
cciso, n_refl_cciso = (0, 0)
if miller_array_iso:
matches_iso = miller.match_multi_indices(
miller_indices_unique=miller_array_iso.indices(),
miller_indices=self.miller_array_merge.indices())
I_iso = flex.double([
miller_array_iso.data()[pair[0]]
for pair in matches_iso.pairs()
])
I_merge_match_iso = flex.double(
[self.I_merge[pair[1]] for pair in matches_iso.pairs()])
n_refl_cciso = len(matches_iso.pairs())
if len(matches_iso.pairs()) > 0:
cciso = flex.linear_correlation(I_merge_match_iso,
I_iso).coefficient()
return cciso, n_refl_cciso
def optimize(d_min_min, d_min_max, step):
d_min = d_min_min
d_min_result = None
cc_best = -1.e6
while d_min < d_min_max + 1.e-6:
f_calc = xrs.structure_factors(d_min=d_min).f_calc()
fft_map = miller.fft_map(crystal_gridding=cg,
fourier_coefficients=f_calc)
map_data_ = fft_map.real_map_unpadded()
cc = flex.linear_correlation(
x=map_data_selected_as_1d,
y=map_data_.select(sel).as_1d()).coefficient()
if (cc > cc_best):
cc_best = cc
d_min_result = d_min
d_min += step
return d_min_result
def calc_sfdist(ari, arj):
size_ari = ari.size()
size_arj = arj.size()
ari, arj = ari.common_sets(arj, assert_is_similar_symmetry=False)
size_penalty = size_ari + size_arj - 2 * ari.size()
corr = flex.linear_correlation(ari.data(), arj.data())
if corr.is_well_defined():
distsq = 2. - 2. * corr.coefficient()
distsq = (ari.size() / (ari.size() + size_penalty)) * distsq + (
size_penalty / (ari.size() + size_penalty))
if (distsq > 2.):
distsq = 2.
if (distsq < 0.):
distsq = 0.
return ((2. - distsq) / 2.), ari.size()
else:
return float("nan"), ari.size()
def compute_map(self):
density_modification.density_modification.compute_map(self)
if self.model_map_coeffs is not None:
model_coeffs, dm_coeffs = self.model_map_coeffs.common_sets(self.map_coeffs)
fft_map = model_coeffs.fft_map(
resolution_factor=self.params.grid_resolution_factor).apply_sigma_scaling()
dm_map = dm_coeffs.fft_map(
resolution_factor=self.params.grid_resolution_factor).apply_sigma_scaling()
print
corr = flex.linear_correlation(
fft_map.real_map_unpadded().as_1d(), dm_map.real_map_unpadded().as_1d())
print "dm/model correlation:"
corr.show_summary()
self.correlation_coeffs.append(corr.coefficient())
self.mean_phase_errors.append(flex.mean(phase_error(
flex.arg(model_coeffs.data()),
flex.arg(dm_coeffs.data())))/density_modification.pi_180)
def exercise_fft_map_as_xplor_map(space_group_info, n_elements=10, d_min=3):
structure = random_structure.xray_structure(
space_group_info,
elements=["Si"]*n_elements,
volume_per_atom=1000,
min_distance=3.,
general_positions_only=False)
f_calc = structure.structure_factors(
d_min=d_min, anomalous_flag=False).f_calc()
fft_map = f_calc.fft_map()
fft_map.as_xplor_map(
file_name="tmp.map",
gridding_last=[n-1 for n in fft_map.n_real()])
read = iotbx.xplor.map.reader(file_name="tmp.map")
assert read.title_lines == ["cctbx.miller.fft_map"]
assert read.gridding.n == fft_map.n_real()
assert approx_equal(flex.linear_correlation(
read.data.as_1d(),
fft_map.real_map_unpadded(in_place=False).as_1d()).coefficient(), 1)
for first,last in [[(0,0,0),(3,5,6)],
[(-1,-3,4),(6,4,5)],
[(-2,3,0),(-2,3,0)],
[(-2,3,0),(-2,3,3)],
[(-2,3,0),(-2,8,0)],
[(-2,3,0),(-2,9,0)],
[(-2,3,0),(3,3,0)],
[(-2,3,0),(4,3,0)]]:
fft_map.as_xplor_map(
file_name="tmp.map",
gridding_first=first,
gridding_last=last)
read = iotbx.xplor.map.reader(file_name="tmp.map")
assert read.title_lines == ["cctbx.miller.fft_map"]
assert read.gridding.n == fft_map.n_real()
assert read.gridding.first == first
assert read.gridding.last == last
real_map = fft_map.real_map()
first_p1 = [i%n for i,n in zip(first, fft_map.n_real())]
assert eps_eq(read.data[first], real_map[first_p1], eps=1.e-4)
last_p1 = [i%n for i,n in zip(last, fft_map.n_real())]
assert eps_eq(read.data[last], real_map[last_p1], eps=1.e-4)
for x in xrange(1,10):
point = [iround(f+(l-f)*x/10.) for f,l in zip(first,last)]
point_p1 = [i%n for i,n in zip(point, fft_map.n_real())]
assert eps_eq(read.data[point], real_map[point_p1], eps=1.e-4)
def optimize(d_min_min, d_min_max, step):
d_min = d_min_min
d_min_result = None
cc_best=-1.e6
while d_min < d_min_max+1.e-6:
f_calc = xrs.structure_factors(d_min=d_min).f_calc()
fft_map = miller.fft_map(
crystal_gridding = cg,
fourier_coefficients = f_calc)
map_data_ = fft_map.real_map_unpadded()
cc = flex.linear_correlation(
x=map_data_selected_as_1d,
y=map_data_.select(sel).as_1d()).coefficient()
if(cc > cc_best):
cc_best = cc
d_min_result = d_min
d_min += step
return d_min_result
def exercise_packed(structure_ideal, f_obs,
anomalous_flag,
verbose=0):
sh, ls = shift_all(structure_ideal, f_obs, anomalous_flag)
flag = (random.random() > 0.5)
gradient_flags = randomize_gradient_flags(
xray.structure_factors.gradient_flags(site=flag, u=not flag),
f_obs.anomalous_flag(),
thresholds=(1/2.,0))
u_iso_refinable_params = flex.double()
for scatterer in sh.structure_shifted.scatterers():
scatterer.flags.set_grad_site(gradient_flags.site)
scatterer.flags.set_grad_u_iso(gradient_flags.u_iso)
scatterer.flags.set_grad_u_aniso(gradient_flags.u_aniso)
scatterer.flags.set_grad_occupancy(gradient_flags.occupancy)
scatterer.flags.set_grad_fp(gradient_flags.fp)
scatterer.flags.set_grad_fdp(gradient_flags.fdp)
scatterer.flags.set_tan_u_iso(True)
param = random.randint(90,120)
scatterer.flags.param= param
value = math.tan(scatterer.u_iso*math.pi/adptbx.b_as_u(param)-math.pi/2)
u_iso_refinable_params.append(value)
n_parameters = xray.scatterer_grad_flags_counts(
sh.structure_shifted.scatterers()).n_parameters()
assert n_parameters == sh.structure_shifted.n_parameters()
if (n_parameters > 0):
sfd = xray.structure_factors.gradients_direct(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=u_iso_refinable_params,
miller_set=f_obs,
d_target_d_f_calc=ls.derivatives(),
n_parameters=n_parameters)
assert sfd.packed().size() == n_parameters
re = resampling(miller_set=f_obs)
map0 = re(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=u_iso_refinable_params,
dp=miller.array(miller_set=f_obs, data=ls.derivatives()),
n_parameters=n_parameters,
verbose=verbose)
assert map0.packed().size() == n_parameters
correlation = flex.linear_correlation(sfd.packed(), map0.packed())
assert correlation.is_well_defined()
assert correlation.coefficient() > 0.999
