def _get_Is_from_Imidval(reflections, Imid):
"""Interpret the Imid value to extract and return the Icomb and Vcomb values."""
if Imid == 0: # special value to trigger prf
Int = reflections["intensity.prf.value"]
Var = reflections["intensity.prf.variance"]
elif Imid == 1: # special value to trigger sum
if "partiality" in reflections:
Int = reflections["intensity.sum.value"] / reflections["partiality"]
Var = reflections["intensity.sum.variance"] / flex.pow2(
reflections["partiality"])
else:
Int = reflections["intensity.sum.value"]
Var = reflections["intensity.sum.variance"]
else:
if "partiality" in reflections:
Int, Var = _calculate_combined_raw_intensities(
reflections["intensity.prf.value"],
reflections["intensity.sum.value"] / reflections["partiality"],
reflections["intensity.prf.variance"],
reflections["intensity.sum.variance"] /
flex.pow2(reflections["partiality"]),
Imid,
)
else:
Int, Var = _calculate_combined_raw_intensities(
reflections["intensity.prf.value"],
reflections["intensity.sum.value"],
reflections["intensity.prf.variance"],
reflections["intensity.sum.variance"],
Imid,
)
return Int, Var
def calculate_gradients(self, apm):
"calculate the gradient vector"
a = self.error_model.components["a"].parameters[0]
b = apm.x[0]
Ih_table = self.error_model.binner.Ih_table
I_hl = Ih_table.intensities
g_hl = Ih_table.inverse_scale_factors
weights = self.error_model.binner.weights
bin_vars = self.error_model.binner.bin_variances
sum_matrix = self.error_model.binner.summation_matrix
bin_counts = self.error_model.binner.binning_info["refl_per_bin"]
dsig_dc = (b * flex.pow2(I_hl) * (a**2) /
(self.error_model.binner.sigmaprime * flex.pow2(g_hl)))
ddelta_dsigma = (-1.0 * self.error_model.binner.delta_hl /
self.error_model.binner.sigmaprime)
deriv = ddelta_dsigma * dsig_dc
dphi_by_dvar = -2.0 * (flex.double(bin_vars.size(), 0.5) - bin_vars +
(1.0 / (2.0 * flex.pow2(bin_vars))))
term1 = 2.0 * self.error_model.binner.delta_hl * deriv * sum_matrix
term2a = self.error_model.binner.delta_hl * sum_matrix
term2b = deriv * sum_matrix
grad = dphi_by_dvar * ((term1 / bin_counts) -
(2.0 * term2a * term2b / flex.pow2(bin_counts)))
gradients = flex.double([flex.sum(grad * weights) / flex.sum(weights)])
return gradients
def calculate_bin_variances(self):
"""Calculate the variance of each bin."""
sum_deltasq = flex.pow2(self.delta_hl) * self.summation_matrix
sum_delta_sq = flex.pow2(self.delta_hl * self.summation_matrix)
bin_vars = (sum_deltasq / self.binning_info["refl_per_bin"]) - (
sum_delta_sq / flex.pow2(self.binning_info["refl_per_bin"]))
self.binning_info["bin_variances"] = bin_vars
return bin_vars
def get_rmsds_obs_pred(self, observations, experiment):
reflections = observations.select(
observations.get_flags(observations.flags.used_in_refinement))
assert len(reflections) > 0
obs_x, obs_y, obs_z = reflections['xyzobs.mm.value'].parts()
calc_x, calc_y, calc_z = reflections['xyzcal.mm'].parts()
rmsd_x = flex.mean(flex.pow2(obs_x - calc_x))**0.5
rmsd_y = flex.mean(flex.pow2(obs_y - calc_y))**0.5
rmsd_z = flex.mean(flex.pow2(obs_z - calc_z))**0.5
return (rmsd_x, rmsd_y, rmsd_z)
def calculate_regression_x_y(Ih_table):
"""Calculate regression data points."""
n = Ih_table.group_multiplicities() - 1.0
group_variances = (
flex.pow2(Ih_table.intensities -
(Ih_table.inverse_scale_factors * Ih_table.Ih_values)) *
Ih_table.h_index_matrix) / n
sigmasq_obs = group_variances * Ih_table.h_expand_matrix
isq = flex.pow2(Ih_table.intensities)
y = sigmasq_obs / isq
x = Ih_table.variances / isq
return x, y
def _xl_unit_cell_derivatives(self, isel, parameterisation=None, reflections=None):
# Get required data
h = self._h.select(isel)
B = self._B.select(isel)
wl = self._wavelength.select(isel)
# get derivatives of the B matrix wrt the parameters
dB_dxluc_p = [
None if der is None else flex.mat3_double(len(isel), der.elems)
for der in parameterisation.get_ds_dp(use_none_as_null=True)
]
d2theta_dp = []
# loop through the parameters
for der in dB_dxluc_p:
if der is None:
d2theta_dp.append(None)
continue
r0 = B * h
dr0 = der * h
r0len = r0.norms()
dr0len = dr0.dot(r0) / r0len
# 2theta = 2 * arcsin( |r0| / (2 * |s0| ) )
sintheta = 0.5 * r0len * wl
fac = 1.0 / flex.sqrt(flex.double(len(wl), 1.0) - flex.pow2(sintheta))
val = fac * wl * dr0len
d2theta_dp.append(val)
return d2theta_dp
def predict(self):
"""perform reflection prediction for the working reflections and update the
reflection manager"""
# get the matches
reflections = self._reflection_manager.get_obs()
# reset the 'use' flag for all observations
self._reflection_manager.reset_accepted_reflections()
# set twotheta in place
self._reflection_predictor(reflections)
# calculate residuals
reflections["2theta_resid"] = (reflections["2theta_cal.rad"] -
reflections["2theta_obs.rad"])
reflections["2theta_resid2"] = flex.pow2(reflections["2theta_resid"])
# set used_in_refinement flag to all those that had predictions
mask = reflections.get_flags(reflections.flags.predicted)
reflections.set_flags(mask, reflections.flags.used_in_refinement)
# collect the matches
self.update_matches(force=True)
return
def _calculate_suitable_combined_intensities(scaler, max_key):
reflections = scaler.reflection_table
suitable = scaler.suitable_refl_for_scaling_sel
suitable_conv = reflections["prescaling_correction"].select(suitable)
Isum = reflections["intensity.sum.value"].select(suitable)
Vsum = reflections["intensity.sum.variance"].select(suitable)
if "partiality" in reflections:
inv_p = _determine_inverse_partiality(reflections)
inv_p = inv_p.select(suitable)
if max_key == 1:
if "partiality" in reflections:
intensity = Isum * suitable_conv * inv_p
variance = Vsum * flex.pow2(suitable_conv * inv_p)
else:
intensity = Isum * suitable_conv
variance = Vsum * suitable_conv * suitable_conv
else:
Ipr = reflections["intensity.prf.value"].select(suitable)
Vpr = reflections["intensity.prf.variance"].select(suitable)
if max_key == 0:
intensity = Ipr * suitable_conv
variance = Vpr * suitable_conv * suitable_conv
else:
if "partiality" in reflections:
Int, Var = _calculate_combined_raw_intensities(
Ipr, Isum * inv_p, Vpr, Vsum * inv_p * inv_p, max_key)
else:
Int, Var = _calculate_combined_raw_intensities(
Ipr, Isum, Vpr, Vsum, max_key)
intensity = Int * suitable_conv
variance = Var * suitable_conv * suitable_conv
return intensity, variance
def calculate_gradients(Ih_table):
"""Return a gradient vector on length len(self.apm.x)."""
gsq = flex.pow2(Ih_table.inverse_scale_factors) * Ih_table.weights
sumgsq = gsq * Ih_table.h_index_matrix
prefactor = (
-2.0
* Ih_table.weights
* (
Ih_table.intensities
- (Ih_table.Ih_values * Ih_table.inverse_scale_factors)
)
)
dIh = (
Ih_table.intensities
- (Ih_table.Ih_values * 2.0 * Ih_table.inverse_scale_factors)
) * Ih_table.weights
dIh_by_dpi = calc_dIh_by_dpi(
dIh, sumgsq, Ih_table.h_index_matrix, Ih_table.derivatives.transpose()
)
term_1 = (prefactor * Ih_table.Ih_values) * Ih_table.derivatives
term_2 = (
prefactor * Ih_table.inverse_scale_factors * Ih_table.h_index_matrix
) * dIh_by_dpi
gradient = term_1 + term_2
return gradient
def resolution_histogram(reflections, imageset, plot_filename=None):
d_star_sq = flex.pow2(reflections['rlp'].norms())
hist = get_histogram(d_star_sq)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.bar(hist.slot_centers()-0.5*hist.slot_width(), hist.slots(),
width=hist.slot_width())
ax.set_xlabel("d_star_sq")
ax.set_ylabel("Frequency")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
def resolution_histogram(reflections, imageset, plot_filename=None):
d_star_sq = flex.pow2(reflections['rlp'].norms())
hist = get_histogram(d_star_sq)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.bar(hist.slot_centers() - 0.5 * hist.slot_width(),
hist.slots(),
width=hist.slot_width())
ax.set_xlabel("d_star_sq")
ax.set_ylabel("Frequency")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
def test_error_model_target(large_reflection_table, test_sg):
"""Test the error model target."""
Ih_table = IhTable([large_reflection_table], test_sg, nblocks=1)
block = Ih_table.blocked_data_list[0]
em = BasicErrorModel
em.min_reflections_required = 1
params = generated_param()
params.weighting.error_model.basic.n_bins = 2
params.weighting.error_model.basic.min_Ih = 1.0
error_model = em(basic_params=params.weighting.error_model.basic)
error_model.configure_for_refinement(block)
error_model.parameters = [1.0, 0.05]
parameterisation = ErrorModelB_APM(error_model)
target = ErrorModelTargetB(error_model)
target.predict(parameterisation)
# Test residual calculation
residuals = target.calculate_residuals(parameterisation)
assert residuals == (
flex.double(2, 1.0) -
flex.pow2(error_model.binner.binning_info["bin_variances"]))
# Test gradient calculation against finite differences.
gradients = target.calculate_gradients(parameterisation)
gradient_fd = calculate_gradient_fd(target, parameterisation)
assert list(gradients) == pytest.approx(list(gradient_fd))
# Test the method calls
r, g = target.compute_functional_gradients(parameterisation)
assert r == residuals
assert list(gradients) == pytest.approx(list(g))
r, g = target.compute_functional_gradients(parameterisation)
assert r == residuals
assert list(gradients) == pytest.approx(list(g))
def generate_reflections_in_sg(space_group, id_=0, assign_id=False):
"""Generate reflections with intensities consistent with space group"""
sgi = sgtbx.space_group_info(symbol=space_group)
cs = sgi.any_compatible_crystal_symmetry(volume=3000)
cs = cs.best_cell()
cs = cs.minimum_cell()
intensities = (
generate_intensities(cs, d_min=2.0)
.generate_bijvoet_mates()
.set_observation_type_xray_intensity()
)
intensities = intensities.expand_to_p1()
# needed to give vaguely sensible E_cc_true values
reflections = flex.reflection_table()
reflections["intensity.sum.value"] = intensities.data()
reflections["intensity.sum.variance"] = flex.pow2(intensities.sigmas())
reflections["miller_index"] = intensities.indices()
reflections["d"] = intensities.d_spacings().data()
reflections["id"] = flex.int(reflections.size(), id_)
if assign_id:
reflections.experiment_identifiers()[id_] = str(id_)
reflections.set_flags(
flex.bool(reflections.size(), True), reflections.flags.integrated
)
return reflections
def test_target_fixedIh():
"""Test the target function for targeted scaling (where Ih is fixed)."""
target = ScalingTargetFixedIH()
Ih_table = mock_Ih_table().blocked_data_list[0]
R, _ = target.compute_residuals(Ih_table)
expected_residuals = flex.double([-1.0, 0.0, 1.0])
assert list(R) == pytest.approx(list(expected_residuals))
_, G = target.compute_functional_gradients(Ih_table)
assert list(G) == pytest.approx([-44.0])
# Add in finite difference check
Ih_table = mock_Ih_table().blocked_data_list[0]
J = target.calculate_jacobian(Ih_table)
assert J.n_cols == 1
assert J.n_rows == 3
assert J.non_zeroes == 3
assert J[0, 0] == pytest.approx(-11.0)
assert J[1, 0] == pytest.approx(-22.0)
assert J[2, 0] == pytest.approx(-33.0)
expected_rmsd = (flex.sum(flex.pow2(expected_residuals)) /
len(expected_residuals))**0.5
assert target._rmsds is None
target.param_restraints = False # don't try to use apm to get restraints
assert target.rmsds(mock_Ih_table(), [])
assert target._rmsds == pytest.approx([expected_rmsd])
def make_test_data_thaumatin_41c():
"""Real thaumatin data (P41212). Nice example of 41 screw axis,
in trouble if we can't get this one right!"""
r = flex.reflection_table()
miller_ax_vals = list(range(1, 87))
i = [-0.006, -0.027, -0.016, 0.094, 0.012, 0.041, 0.039, 605.708, -0.01]
i += [0.058, 0.005, 406.319, 0.047, 0.043, 0.082, 0.754, 0.101, 0.126]
i += [-0.17, 1.381, 0.149, -0.177, 0.175, 25.368, 0.007, 0.442, -0.753]
i += [2944.402, 0.02, -0.41, 0.399, 1334.451, 0.35, 0.028, -0.353, 24.594]
i += [0.459, 0.093, -0.666, 1068.914, -1.048, 0.882, 0.26, 391.129, 0.771]
i += [-0.605, 1.923, 79.672, 0.539, 0.342, 0.673, 570.054, -0.624, -0.388]
i += [
-0.572, 1175.132, -0.764, 0.006, 1.027, 459.19, -0.116, 0.098, -0.186
]
i += [319.29, 0.591, 0.874, 0.265, 4.021, 0.246, 0.262, 0.552, 14.901]
i += [-1.647, -1.776, 0.01, 57.969, 1.067, -0.751, 0.438, 0.98, 0.277]
i += [0.317, -0.888, 11.128, 0.332, -0.608]
s = [0.005, 0.016, 0.023, 0.055, 0.072, 0.088, 0.112, 12.797, 0.125]
s += [0.148, 0.17, 8.9, 0.199, 0.274, 0.297, 0.385, 0.378, 0.345, 0.287]
s += [0.371, 0.291, 0.343, 0.334, 1.276, 0.358, 0.574, 0.724, 61.67, 0.46]
s += [0.475, 0.652, 40.835, 0.65, 0.674, 0.796, 1.533, 0.587, 0.663, 0.692]
s += [23.811, 0.794, 0.707, 0.765, 9.989, 0.833, 0.818, 1.21, 3.354, 0.808]
s += [
0.836, 0.836, 14.031, 0.894, 0.914, 0.992, 26.717, 0.937, 0.88, 0.904
]
s += [
12.025, 0.879, 0.886, 0.981, 9.202, 1.012, 1.478, 1.413, 1.596, 1.453
]
s += [1.411, 0.861, 2.561, 1.481, 1.162, 0.918, 3.367, 0.93, 1.03, 0.875]
s += [1.0, 0.925, 0.918, 0.983, 1.631, 0.821, 1.35]
r["miller_index"] = flex.miller_index([(0, 0, j) for j in miller_ax_vals])
r["variance"] = flex.pow2(flex.double(s))
r["intensity"] = flex.double(i)
return r
def test_correct_correction(dials_data):
"""Test that the anvil absorption correction is producing expected values."""
data_dir = dials_data("centroid_test_data")
# We'll need an integrated reflection table and an experiment list.
reflections_file = data_dir.join("integrated.pickle")
experiments_file = data_dir.join("experiments.json")
# We need only test with the first ten reflections.
reflections = flex.reflection_table.from_file(reflections_file)
reflections = reflections.select(flex.size_t_range(10))
experiment = ExperimentList.from_file(experiments_file)[0]
# Test the correction that would be applied to a DAC with 1.5mm-thick anvils,
# aligned along the z-axis at goniometer zero-datum.
old_reflections = copy.deepcopy(reflections)
correct_intensities_for_dac_attenuation(experiment, reflections, (0, 0, 1),
1.5)
cases = {
"intensity.sum.value": reflections.flags.integrated_sum,
"intensity.sum.variance": reflections.flags.integrated_sum,
"intensity.prf.value": reflections.flags.integrated_prf,
"intensity.prf.variance": reflections.flags.integrated_prf,
}
corrections = flex.double([
0,
6.653068275094517,
6.522657529202368,
6.3865190053761,
6.587270967838122,
6.43403642876391,
6.39216742203502,
0,
6.152148372872684,
6.0474840161407375,
])
for case, flag in cases.items():
flagged = reflections.get_flags(flag)
target_correction = corrections.select(flagged)
if "variance" in case:
target_correction = flex.pow2(target_correction)
intensity_correction = (reflections[case] /
old_reflections[case]).select(flagged)
# Check that the un-integrated reflections are unchanged.
assert pytest.approx(reflections[case].select(
~flagged)) == old_reflections[case].select(~flagged), (
"Un-integrated reflections have been erroneously "
"'corrected'.")
# Check that the applied corrections are correct.
assert pytest.approx(
intensity_correction, rel=1e-5
) == list(target_correction), (
"The applied intensity correction to %s doesn't seem to be correct."
% case)
def configure_components(self, reflection_table, experiment, params):
"""Add the required reflection table data to the model components."""
xyz = reflection_table["xyzobs.px.value"].parts()
norm_time = xyz[2] * self.configdict["time_norm_fac"]
if "decay" in self.components:
d = reflection_table["d"]
norm_res = (
(1.0 / flex.pow2(d)) -
self.configdict["resmin"]) / self.configdict["res_bin_width"]
self.components["decay"].data = {"x": norm_res, "y": norm_time}
if "absorption" in self.components:
norm_x_abs = (xyz[0] - self.configdict["xmin"]
) / self.configdict["x_bin_width"]
norm_y_abs = (xyz[1] - self.configdict["ymin"]
) / self.configdict["y_bin_width"]
self.components["absorption"].data = {
"x": norm_x_abs,
"y": norm_y_abs,
"z": norm_time,
}
if "modulation" in self.components:
norm_x_det = (xyz[0] - self.configdict["xmin"]
) / self.configdict["x_det_bin_width"]
norm_y_det = (xyz[1] - self.configdict["ymin"]
) / self.configdict["y_det_bin_width"]
self.components["modulation"].data = {
"x": norm_x_det,
"y": norm_y_det
}
def compute_overall_stats(self):
# Create lookups for elements by miller index
index_lookup = defaultdict(list)
for i, h in enumerate(self.reflection_table["miller_index"]):
index_lookup[h].append(i)
# Compute the Overall Sum(X) and Sum(X^2) for each unique reflection
for h in index_lookup:
sel = flex.size_t(index_lookup[h])
intensities = self.reflection_table["intensity"].select(sel)
n = intensities.size()
sum_x = flex.sum(intensities)
sum_x2 = flex.sum(flex.pow2(intensities))
self.reflection_sums[h] = ReflectionSum(sum_x, sum_x2, n)
# Compute some numbers
self._num_datasets = len(set(self.reflection_table["dataset"]))
self._num_groups = len(set(self.reflection_table["group"]))
self._num_reflections = self.reflection_table.size()
self._num_unique = len(self.reflection_sums)
logger.info(
"""
Summary of input data:
# Datasets: %s
# Groups: %s
# Reflections: %s
# Unique reflections: %s""",
self._num_datasets,
self._num_groups,
self._num_reflections,
self._num_unique,
)
def resolution_histogram(reflections, imageset, plot_filename=None):
d_star_sq = flex.pow2(reflections["rlp"].norms())
hist = get_histogram(d_star_sq)
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.bar(
hist.slot_centers() - 0.5 * hist.slot_width(),
hist.slots(),
width=hist.slot_width(),
)
ax.set_xlabel("d_star_sq")
ax.set_ylabel("Frequency")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
pyplot.savefig(plot_filename)
pyplot.close()
def stats_single_image(imageset, reflections, i=None, resolution_analysis=True,
plot=False):
reflections = map_to_reciprocal_space(reflections, imageset)
if plot and i is not None:
filename = "i_over_sigi_vs_resolution_%d.png" %(i+1)
hist_filename = "spot_count_vs_resolution_%d.png" %(i+1)
extra_filename = "log_sum_i_sigi_vs_resolution_%d.png" %(i+1)
distl_method_1_filename = "distl_method_1_%d.png" %(i+1)
distl_method_2_filename = "distl_method_2_%d.png" %(i+1)
else:
filename = None
hist_filename = None
extra_filename = None
distl_method_1_filename = None
distl_method_2_filename = None
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
#plot_ordered_d_star_sq(reflections, imageset)
reflections_all = reflections
ice_sel = ice_rings_selection(reflections_all)
reflections_no_ice = reflections_all.select(~ice_sel)
n_spots_total = len(reflections_all)
n_spots_no_ice = len(reflections_no_ice)
n_spot_4A = (d_spacings > 4).count(True)
intensities = reflections_no_ice['intensity.sum.value']
total_intensity = flex.sum(intensities)
#print i
if hist_filename is not None:
resolution_histogram(
reflections, imageset, plot_filename=hist_filename)
if extra_filename is not None:
log_sum_i_sigi_vs_resolution(
reflections, imageset, plot_filename=extra_filename)
if resolution_analysis and n_spots_no_ice > 10:
estimated_d_min = estimate_resolution_limit(
reflections_all, imageset, ice_sel=ice_sel, plot_filename=filename)
d_min_distl_method_1, noisiness_method_1 \
= estimate_resolution_limit_distl_method1(
reflections_all, imageset, ice_sel, plot_filename=distl_method_1_filename)
d_min_distl_method_2, noisiness_method_2 = \
estimate_resolution_limit_distl_method2(
reflections_all, imageset, ice_sel, plot_filename=distl_method_2_filename)
else:
estimated_d_min = -1.0
d_min_distl_method_1 = -1.0
noisiness_method_1 = -1.0
d_min_distl_method_2 = -1.0
noisiness_method_2 = -1.0
return group_args(n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=n_spot_4A,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
noisiness_method_1=noisiness_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_2=noisiness_method_2)
def estimate_resolution_limit_distl_method2(reflections, plot_filename=None):
# Implementation of Method 2 (section 2.4.4) of:
# Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
# J. Appl. Cryst. (2006). 39, 112-119
# https://doi.org/10.1107/S0021889805040677
variances = reflections["intensity.sum.variance"]
sel = variances > 0
reflections = reflections.select(sel)
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
binner = binner_d_star_cubed(d_spacings)
bin_counts = flex.size_t()
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = sel_all
bin_counts.append(sel.count(True))
# print list(bin_counts)
t0 = (bin_counts[0] + bin_counts[1]) / 2
mu = 0.15
for i in range(len(bin_counts) - 1):
tj = bin_counts[i]
tj1 = bin_counts[i + 1]
if (tj < (mu * t0)) and (tj1 < (mu * t0)):
break
d_min = binner.bins[i].d_min
noisiness = 0
m = len(bin_counts)
for i in range(m):
for j in range(i + 1, m):
if bin_counts[i] <= bin_counts[j]:
noisiness += 1
noisiness /= 0.5 * m * (m - 1)
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(range(len(bin_counts)), bin_counts)
ax.set_ylabel("number of spots in shell")
xlim = pyplot.xlim()
ylim = pyplot.ylim()
ax.vlines(i, ylim[0], ylim[1], colors="red")
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.savefig(plot_filename)
pyplot.close()
return d_min, noisiness
def scale_factor(self,
this,
other,
weights=None,
cutoff_factor=None,
use_binning=False):
"""
The analytical expression for the least squares scale factor.
K = sum(w * yo * yc) / sum(w * yc^2)
If the optional cutoff_factor argument is provided, only the reflections
whose magnitudes are greater than cutoff_factor * max(yo) will be included
in the calculation.
"""
assert not use_binning or this.binner() is not None
if use_binning: assert cutoff_factor is None
assert other.size() == this.data().size()
if not use_binning:
if this.data().size() == 0: return None
obs = this.data()
calc = other.data()
if cutoff_factor is not None:
assert cutoff_factor < 1
sel = obs >= flex.max(this.data()) * cutoff_factor
obs = obs.select(sel)
calc = calc.select(sel)
if weights is not None:
weights = weights.select(sel)
if weights is None:
return flex.sum(obs * calc) / flex.sum(flex.pow2(calc))
else:
return flex.sum(weights * obs * calc) \
/ flex.sum(weights * flex.pow2(calc))
results = []
for i_bin in this.binner().range_all():
sel = this.binner().selection(i_bin)
weights_sel = None
if weights is not None:
weights_sel = weights.select(sel)
results.append(
self.scale_factor(this.select(sel), other.select(sel),
weights_sel))
return binned_data(binner=this.binner(),
data=results,
data_fmt="%7.4f")
def plot_ordered_d_star_sq(reflections, imageset):
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
d_star_sq = flex.pow2(reflections['rlp'].norms())
perm = flex.sort_permutation(d_star_sq)
pyplot.scatter(list(range(len(perm))), list(d_star_sq.select(perm)), marker='+')
pyplot.show()
def calculate_residuals(self, _):
"""Return the residual vector"""
bin_vars = self.error_model.binner.bin_variances
R = ((flex.pow2(flex.double(bin_vars.size(), 0.5) - bin_vars) +
(1.0 / bin_vars) - flex.double(bin_vars.size(), 1.25)) *
self.error_model.binner.weights /
flex.sum(self.error_model.binner.weights))
return R
def compute_functional_gradients(cls, Ih_table):
"""Return the functional and gradients."""
resids = cls.calculate_residuals(Ih_table)
gradients = cls.calculate_gradients(Ih_table)
weights = Ih_table.weights
functional = flex.sum(flex.pow2(resids) * weights)
del Ih_table.derivatives
return functional, gradients
def plot_ordered_d_star_sq(reflections, imageset):
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
d_star_sq = flex.pow2(reflections['rlp'].norms())
perm = flex.sort_permutation(d_star_sq)
pyplot.scatter(list(range(len(perm))), list(d_star_sq.select(perm)), marker='+')
pyplot.show()
def calc_Ih(self):
"""Calculate the current best estimate for Ih for each reflection group."""
scale_factors = self.Ih_table["inverse_scale_factor"]
gsq = flex.pow2(scale_factors) * self.Ih_table["weights"]
sumgsq = gsq * self.h_index_matrix
gI = (scale_factors * self.Ih_table["intensity"]) * self.Ih_table["weights"]
sumgI = gI * self.h_index_matrix
Ih = sumgI / sumgsq
self.Ih_table["Ih_values"] = Ih * self.h_expand_matrix
def plot_ordered_d_star_sq(reflections, imageset):
from matplotlib import pyplot
d_star_sq = flex.pow2(reflections["rlp"].norms())
perm = flex.sort_permutation(d_star_sq)
pyplot.scatter(list(range(len(perm))),
list(d_star_sq.select(perm)),
marker="+")
pyplot.show()
def __call__(self, d):
"""
True if within powder ring.
:param d: The resolution
:return: True/False in powder ring
"""
result = flex.bool(len(d), False)
d2 = 1.0 / flex.pow2(d)
for ice_ring in self.ice_rings:
result = result | (d2 >= ice_ring[0]) & (d2 <= ice_ring[1])
return result
def spot_resolution_shells(experiments, reflections, params):
from dials.array_family import flex
reflections.centroid_px_to_mm(experiments)
reflections.map_centroids_to_reciprocal_space(experiments)
two_theta_array = reflections["rlp"].norms()
h0 = flex.weighted_histogram(flex.pow2(two_theta_array), n_slots=params.shells)
n = h0.slots()
d = 1.0 / flex.sqrt(h0.slot_centers())
for j in range(params.shells):
print("%d %f %d" % (j, d[j], n[j]))
def apply_scaling_factors(reflection_table):
"""Apply the inverse scale factor to the scale intensities."""
if "partiality" in reflection_table:
reflection_table = reflection_table.select(
reflection_table["partiality"] > 0.0)
assert "inverse_scale_factor" in reflection_table
reflection_table["intensity.scale.value"] /= reflection_table[
"inverse_scale_factor"]
reflection_table["intensity.scale.variance"] /= flex.pow2(
reflection_table["inverse_scale_factor"])
return reflection_table
def correct_intensities_for_dac_attenuation(
experiment: Experiment,
reflections: flex.reflection_table,
dac_norm: Vector,
thickness: float,
density: float = 3.51,
) -> None:
"""
Boost integrated intensities to account for attenuation by a diamond anvil cell.
Take an experiment object and reflection table containing integrated but unscaled
diffraction data, boost the integrated reflection intensities to correct for the
estimated attenuation due to the passage of the incident and diffracted beams
through the diamond anvils.
Args:
experiment: An experiment from integrated data.
reflections: A table of reflections from integrated data.
dac_norm: A 3-vector representing the normal to the anvil surfaces in the
laboratory frame when the goniometer is at zero datum, i.e. the axes
are all at 0°. The vector is assumed to be normalised.
thickness: The thickness of each diamond anvil (assumed equal).
density: The density of the anvil material in g·cm⁻³
(units chosen for consistency with the NIST tables of X-ray mass
attenuation coefficients, https://dx.doi.org/10.18434/T4D01F).
Defaults to a typical value for synthetic diamond.
"""
# Select only those reflections whose intensities after integration ought to be
# meaningful.
sel = reflections.get_flags(reflections.flags.integrated, all=False)
sel = sel.iselection()
refls_sel = reflections.select(sel)
correction = attenuation_correction(experiment, refls_sel, dac_norm,
thickness, density)
# We need only correct non-null values for each integration method.
prf_subsel = refls_sel.get_flags(refls_sel.flags.integrated_prf)
sum_subsel = refls_sel.get_flags(refls_sel.flags.integrated_sum)
# Correct the measured intensities and variances for this attenuation.
methods = {"prf": prf_subsel, "sum": sum_subsel}
quantities = {"value": correction, "variance": flex.pow2(correction)}
for method, subsel in methods.items():
setting_subsel = sel.select(subsel)
for quantity, factor in quantities.items():
col = f"intensity.{method}.{quantity}"
corrected = (refls_sel[col] * factor).select(subsel)
try:
reflections[col].set_selected(setting_subsel, corrected)
except KeyError:
pass
def make_test_data_LCY_21c():
"""Real data from LCY (P212121)."""
r = flex.reflection_table()
miller_ax_vals = list(range(2, 17))
i = [243.42, 0.95, 841.09, 4.08, 6780.07, 0.12, 19517.36, 0.10]
i += [45.17, 1.60, 1056.85, 5.88, 7054.45, 7.77, 284.67]
# separate to avoid black formatting
s = [8.64, 0.80, 28.70, 1.87, 248.14, 1.30, 637.42, 1.57, 4.49, 2.05]
s += [50.27, 3.37, 520.39, 4.84, 24.14]
r["miller_index"] = flex.miller_index([(0, 0, j) for j in miller_ax_vals])
r["variance"] = flex.pow2(flex.double(s))
r["intensity"] = flex.double(i)
return r
def ice_rings_selection(reflections):
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
from dials.algorithms.integration import filtering
unit_cell = uctbx.unit_cell((4.498,4.498,7.338,90,90,120))
space_group = sgtbx.space_group_info(number=194).group()
width = 0.06
ice_filter = filtering.PowderRingFilter(
unit_cell, space_group, flex.min(d_spacings)-width, width)
ice_sel = ice_filter(d_spacings)
return ice_sel
def log_sum_i_sigi_vs_resolution(reflections, imageset, plot_filename=None):
d_star_sq = flex.pow2(reflections['rlp'].norms())
hist = get_histogram(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = intensities.select(sel)
i_over_sigi = intensities/flex.sqrt(variances)
#log_i_over_sigi = flex.log(i_over_sigi)
slots = []
for slot in hist.slot_infos():
sel = (d_star_sq > slot.low_cutoff) & (d_star_sq < slot.high_cutoff)
if sel.count(True) > 0:
slots.append(math.log(flex.sum(i_over_sigi.select(sel))))
else:
slots.append(0)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
#ax.bar(hist.slot_centers()-0.5*hist.slot_width(), hist.slots(),
ax.scatter(hist.slot_centers()-0.5*hist.slot_width(), slots, s=20, color='blue', marker='o', alpha=0.5)
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(sum(I/sigI))")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
def estimate_resolution_limit(reflections, imageset, ice_sel=None,
plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities/flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(
d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(
d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1-low_percentile_limit
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
#sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#print "%.2f" %(sel.count(True)/sel_all.count(True))
if sel.count(True) == 0:
#outliers_all.set_selected(sel_all & ice_sel, True)
continue
#if i_slot > i_slot_max:
#break
#else:
#continue
outliers = wilson_outliers(
reflections.select(sel_all), ice_sel=ice_sel.select(sel_all))
#print "rejecting %d wilson outliers" %outliers.count(True)
outliers_all.set_selected(sel_all, outliers)
#if sel.count(True)/sel_all.count(True) < 0.25:
#outliers_all.set_selected(sel_all & ice_sel, True)
#from scitbx.math import median_statistics
#intensities_sel = intensities.select(sel)
#stats = median_statistics(intensities_sel)
#z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
#outliers = z_score > 3.5
#perm = flex.sort_permutation(intensities_sel)
##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
##print ' '.join('%.2f' %v for v in z_score.select(perm))
##print
isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
#fit_upper.show_summary()
#fit_lower.show_summary()
if m_upper == m_lower:
intersection = (-1,-1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
intersection = (
(c_lower-c_upper)/(m_upper-m_lower),
(m_upper*c_lower-m_lower*c_upper)/(m_upper-m_lower))
a = m_upper
c_ = c_upper
b = m_lower
d = c_lower
assert intersection == ((d-c_)/(a-b), (a*d-b*c_)/(a-b))
#inside = points_inside_envelope(
#d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper)
inside = inside & ~outliers_all
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
#d_star_sq_estimate = intersection[0]
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
#resolution_estimate = max(resolution_estimate, flex.min(d_spacings))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
ax.scatter(d_star_sq.select(inside), log_i_over_sigi.select(inside),
marker='+', color='green')
ax.scatter(d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker='+', color='black')
ax.scatter(d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker='+', color='grey')
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')
if (intersection[0] <= ax.get_xlim()[1] and
intersection[1] <= ax.get_ylim()[1]):
ax.scatter([intersection[0]], [intersection[1]], marker='x', s=50, color='b')
#ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
ax.set_xlabel('d_star_sq')
ax.set_ylabel('ln(I/sigI)')
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax.vlines(uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def run(args):
import libtbx.load_env
usage = "%s [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
check_format=False,
epilog=help_message)
params, options, args = parser.parse_args(show_diff_phil=True,
return_unhandled=True)
assert len(args) == 2
from iotbx.reflection_file_reader import any_reflection_file
xyz = []
intensities = []
lp_corrections = []
for f in args:
xdet = None
ydet = None
rot = None
i_sigi = None
lp = None
arrays = any_reflection_file(f).as_miller_arrays(merge_equivalents=False)
for ma in arrays:
print ma.info().labels
if ma.info().labels[0] == 'XDET':
xdet = ma
elif ma.info().labels[0] == 'YDET':
ydet = ma
elif ma.info().labels[0] == 'ROT':
rot = ma
elif ma.info().labels == ['I', 'SIGI']:
i_sigi = ma
elif ma.info().labels[0] == 'LP':
lp = ma
assert [xdet, ydet, rot, i_sigi, lp].count(None) == 0
xyz.append(flex.vec3_double(xdet.data(), ydet.data(), rot.data()))
intensities.append(i_sigi)
lp_corrections.append(lp)
xyz1, xyz2 = xyz
xyz2 += (1e-3,1e-3,1e-3)
intensities1, intensities2 = intensities
lp1, lp2 = lp_corrections
# Do the nn match
from annlib_ext import AnnAdaptor as ann_adaptor
ann = ann_adaptor(xyz1.as_double().as_1d(), 3)
ann.query(xyz2.as_double().as_1d())
distances = flex.sqrt(ann.distances)
matches = distances < 2 #pixels
index1 = flex.size_t(list(ann.nn.select(matches)))
index2 = flex.size_t(list(matches.iselection()))
intensities1 = intensities1.select(index1)
intensities2 = intensities2.select(index2)
isigi1 = intensities1.data()/intensities1.sigmas()
isigi2 = intensities2.data()/intensities2.sigmas()
lp1 = lp1.select(index1)
lp2 = lp2.select(index2)
##differences = intensities1.data() - intensities2.data()
##sums = intensities1.data() + intensities2.data()
#differences = isigi1 - isigi2
#sums = isigi1 + isigi2
#assert sums.all_ne(0)
#dos = differences/sums
#mean_dos = []
#binner = intensities1.setup_binner_d_star_sq_step(d_star_sq_step=0.01)
#d_spacings = intensities1.d_spacings().data()
#for i in range(binner.n_bins_used()):
#d_max, d_min = binner.bin_d_range(i+1)
#bin_sel = (d_spacings > d_min) & (d_spacings <= d_max)
#mean_dos.append(flex.mean(dos.select(bin_sel)))
# set backend before importing pyplot
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
pyplot.scatter(intensities1.data(), intensities2.data(), marker='+', alpha=0.5)
m = max(pyplot.xlim()[1], pyplot.ylim()[1])
pyplot.plot((0,m), (0, m), c='black')
pyplot.savefig('scatter_intensities.png')
pyplot.clf()
pyplot.scatter(intensities1.sigmas(), intensities2.sigmas(), marker='+', alpha=0.5)
m = max(pyplot.xlim()[1], pyplot.ylim()[1])
pyplot.plot((0,m), (0, m), c='black')
pyplot.savefig('scatter_sigmas.png')
pyplot.clf()
pyplot.scatter(
flex.pow2(intensities1.sigmas()), flex.pow2(intensities2.sigmas()),
marker='+', alpha=0.5)
m = max(pyplot.xlim()[1], pyplot.ylim()[1])
pyplot.plot((0,m), (0, m), c='black')
pyplot.savefig('scatter_variances.png')
pyplot.clf()
pyplot.scatter(isigi1, isigi2, marker='+', alpha=0.5)
m = max(pyplot.xlim()[1], pyplot.ylim()[1])
pyplot.plot((0,m), (0, m), c='black')
pyplot.savefig('scatter_i_sig_i.png')
pyplot.clf()
pyplot.scatter(lp1.data(), lp2.data(), marker='+', alpha=0.5)
m = max(pyplot.xlim()[1], pyplot.ylim()[1])
pyplot.plot((0,m), (0, m))
pyplot.savefig('scatter_LP.png')
pyplot.clf()
#from cctbx import uctbx
#pyplot.scatter(uctbx.d_star_sq_as_d(binner.bin_centers(2)), mean_dos)
#pyplot.savefig('mean_dos.png')
#pyplot.clf()
return
def estimate_resolution_limit_distl_method1(
reflections, imageset, ice_sel=None, plot_filename=None):
# Implementation of Method 1 (section 2.4.4) of:
# Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
# J. Appl. Cryst. (2006). 39, 112-119
# http://dx.doi.org/10.1107/S0021889805040677
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = reflections['intensity.sum.value']
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
reflections = reflections.select(sel)
intensities = reflections['intensity.sum.value']
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
d_star_cubed = flex.pow(reflections['rlp'].norms(), 3)
step = 2
while len(reflections)/step > 40:
step += 1
order = flex.sort_permutation(d_spacings, reverse=True)
ds3_subset = flex.double()
d_subset = flex.double()
for i in range(len(reflections)//step):
ds3_subset.append(d_star_cubed[order[i*step]])
d_subset.append(d_spacings[order[i*step]])
x = flex.double(range(len(ds3_subset)))
# (i)
# Usually, Pm is the last point, that is, m = n. But m could be smaller than
# n if an unusually high number of spots are detected around a certain
# intermediate resolution. In that case, our search for the image resolution
# does not go outside the spot 'bump;. This is particularly useful when
# ice-rings are present.
slopes = (ds3_subset[1:] - ds3_subset[0])/(x[1:]-x[0])
skip_first = 3
p_m = flex.max_index(slopes[skip_first:]) + 1 + skip_first
# (ii)
from scitbx import matrix
x1 = matrix.col((0, ds3_subset[0]))
x2 = matrix.col((p_m, ds3_subset[p_m]))
gaps = flex.double([0])
v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize()
for i in range(1, p_m):
x0 = matrix.col((i, ds3_subset[i]))
r = x1 - x0
g = abs(v.dot(r))
gaps.append(g)
mv = flex.mean_and_variance(gaps)
s = mv.unweighted_sample_standard_deviation()
# (iii)
p_k = flex.max_index(gaps)
g_k = gaps[p_k]
p_g = p_k
for i in range(p_k+1, len(gaps)):
g_i = gaps[i]
if g_i > (g_k - 0.5 * s):
p_g = i
ds3_g = ds3_subset[p_g]
d_g = d_subset[p_g]
noisiness = 0
n = len(ds3_subset)
for i in range(n-1):
for j in range(i+1, n-1):
if slopes[i] >= slopes[j]:
noisiness += 1
noisiness /= ((n-1)*(n-2)/2)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(range(len(ds3_subset)), ds3_subset)
#ax.set_xlabel('')
ax.set_ylabel('D^-3')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
ax.vlines(p_g, ylim[0], ylim[1], colors='red')
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.savefig(plot_filename)
pyplot.close()
return d_g, noisiness
def run(self, method):
from xia2.Handlers.Streams import Debug
Debug.write('Running dials.index')
self.clear_command_line()
for f in self._sweep_filenames:
self.add_command_line(f)
for f in self._spot_filenames:
self.add_command_line(f)
self.add_command_line('indexing.method=%s' % method)
nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc
self.set_cpu_threads(nproc)
self.add_command_line('indexing.nproc=%i' % nproc)
if PhilIndex.params.xia2.settings.small_molecule == True:
self.add_command_line('filter_ice=false')
if self._reflections_per_degree is not None:
self.add_command_line(
'reflections_per_degree=%i' %self._reflections_per_degree)
if self._fft3d_n_points is not None:
self.add_command_line(
'fft3d.reciprocal_space_grid.n_points=%i' %self._fft3d_n_points)
if self._close_to_spindle_cutoff is not None:
self.add_command_line(
'close_to_spindle_cutoff=%f' %self._close_to_spindle_cutoff)
if self._outlier_algorithm:
self.add_command_line('outlier.algorithm=%s' % self._outlier_algorithm)
if self._max_cell:
self.add_command_line('max_cell=%d' % self._max_cell)
if self._min_cell:
self.add_command_line('min_cell=%d' % self._min_cell)
if self._histogram_binning is not None:
self.add_command_line('max_cell_estimation.histogram_binning=%s' %self._histogram_binning)
if self._d_min_start:
self.add_command_line('d_min_start=%f' % self._d_min_start)
if self._indxr_input_lattice is not None:
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
self._symm = lattice_to_spacegroup_number(
self._indxr_input_lattice)
self.add_command_line('known_symmetry.space_group=%s' % self._symm)
if self._indxr_input_cell is not None:
self.add_command_line(
'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' %self._indxr_input_cell)
if self._maximum_spot_error:
self.add_command_line('maximum_spot_error=%.f' %
self._maximum_spot_error)
if self._detector_fix:
self.add_command_line('detector.fix=%s' % self._detector_fix)
if self._beam_fix:
self.add_command_line('beam.fix=%s' % self._beam_fix)
if self._phil_file is not None:
self.add_command_line("%s" %self._phil_file)
self._experiment_filename = os.path.join(
self.get_working_directory(), '%d_experiments.json' %self.get_xpid())
self._indexed_filename = os.path.join(
self.get_working_directory(), '%d_indexed.pickle' %self.get_xpid())
self.add_command_line("output.experiments=%s" %self._experiment_filename)
self.add_command_line("output.reflections=%s" %self._indexed_filename)
self.start()
self.close_wait()
self.check_for_errors()
from dials.array_family import flex
from dxtbx.serialize import load
self._experiment_list = load.experiment_list(self._experiment_filename)
self._reflections = flex.reflection_table.from_pickle(
self._indexed_filename)
crystal = self._experiment_list.crystals()[0]
self._p1_cell = crystal.get_unit_cell().parameters()
refined_sel = self._reflections.get_flags(self._reflections.flags.used_in_refinement)
refl = self._reflections.select(refined_sel)
xc, yc, zc = refl['xyzcal.px'].parts()
xo, yo, zo = refl['xyzobs.px.value'].parts()
import math
self._nref = refl.size()
self._rmsd_x = math.sqrt(flex.mean(flex.pow2(xc - xo)))
self._rmsd_y = math.sqrt(flex.mean(flex.pow2(yc - yo)))
self._rmsd_z = math.sqrt(flex.mean(flex.pow2(zc - zo)))
return
def generate_integration_report(experiment, reflections, n_resolution_bins=20):
'''
Generate the integration report
'''
from collections import OrderedDict
from dials.algorithms.statistics import pearson_correlation_coefficient
from dials.algorithms.statistics import spearman_correlation_coefficient
from cctbx import miller, crystal
def overall_report(data):
# Start by adding some overall numbers
report = OrderedDict()
report['n'] = len(reflections)
report['n_full'] = data['full'].count(True)
report['n_partial'] = data['full'].count(False)
report['n_overload'] = data['over'].count(True)
report['n_ice'] = data['ice'].count(True)
report['n_summed'] = data['sum'].count(True)
report['n_fitted'] = data['prf'].count(True)
report['n_integated'] = data['int'].count(True)
report['n_invalid_bg'] = data['ninvbg'].count(True)
report['n_invalid_fg'] = data['ninvfg'].count(True)
report['n_failed_background'] = data['fbgd'].count(True)
report['n_failed_summation'] = data['fsum'].count(True)
report['n_failed_fitting'] = data['fprf'].count(True)
# Compute mean background
try:
report['mean_background'] = flex.mean(
data['background.mean'].select(data['int']))
except Exception:
report['mean_background'] = 0.0
# Compute mean I/Sigma summation
try:
report['ios_sum'] = flex.mean(
data['intensity.sum.ios'].select(data['sum']))
except Exception:
report['ios_sum'] = 0.0
# Compute mean I/Sigma profile fitting
try:
report['ios_prf'] = flex.mean(
data['intensity.prf.ios'].select(data['prf']))
except Exception:
report['ios_prf'] = 0.0
# Compute the mean profile correlation
try:
report['cc_prf'] = flex.mean(
data['profile.correlation'].select(data['prf']))
except Exception:
report['cc_prf'] = 0.0
# Compute the correlations between summation and profile fitting
try:
mask = data['sum'] & data['prf']
Isum = data['intensity.sum.value'].select(mask)
Iprf = data['intensity.prf.value'].select(mask)
report['cc_pearson_sum_prf'] = pearson_correlation_coefficient(Isum, Iprf)
report['cc_spearman_sum_prf'] = spearman_correlation_coefficient(Isum, Iprf)
except Exception:
report['cc_pearson_sum_prf'] = 0.0
report['cc_spearman_sum_prf'] = 0.0
# Return the overall report
return report
def binned_report(binner, index, data):
# Create the indexers
indexer_all = binner.indexer(index)
indexer_sum = binner.indexer(index.select(data['sum']))
indexer_prf = binner.indexer(index.select(data['prf']))
indexer_int = binner.indexer(index.select(data['int']))
# Add some stats by resolution
report = OrderedDict()
report['bins'] = list(binner.bins())
report['n_full'] = list(indexer_all.sum(data['full']))
report['n_partial'] = list(indexer_all.sum(~data['full']))
report['n_overload'] = list(indexer_all.sum(data['over']))
report['n_ice'] = list(indexer_all.sum(data['ice']))
report['n_summed'] = list(indexer_all.sum(data['sum']))
report['n_fitted'] = list(indexer_all.sum(data['prf']))
report['n_integrated'] = list(indexer_all.sum(data['int']))
report['n_invalid_bg'] = list(indexer_all.sum(data['ninvbg']))
report['n_invalid_fg'] = list(indexer_all.sum(data['ninvfg']))
report['n_failed_background'] = list(indexer_all.sum(data['fbgd']))
report['n_failed_summation'] = list(indexer_all.sum(data['fsum']))
report['n_failed_fitting'] = list(indexer_all.sum(data['fprf']))
# Compute mean background
try:
report['mean_background'] = list(indexer_int.mean(
data['background.mean'].select(data['int'])))
except Exception:
report['mean_background'] = [0.0] * len(binner)
# Compute mean I/Sigma summation
try:
report['ios_sum'] = list(indexer_sum.mean(
data['intensity.sum.ios'].select(data['sum'])))
except Exception:
report['ios_sum'] = [0.0] * len(binner)
# Compute mean I/Sigma profile fitting
try:
report['ios_prf'] = list(indexer_prf.mean(
data['intensity.prf.ios'].select(data['prf'])))
except Exception:
report['ios_prf'] = [0.0] * len(binner)
# Compute the mean profile correlation
try:
report['cc_prf'] = list(indexer_prf.mean(
data['profile.correlation'].select(data['prf'])))
except Exception:
report['cc_prf'] = [0.0] * len(binner)
try:
report['rmsd_xy'] = list(indexer_sum.mean(
data['xyz.rmsd'].select(data['int'])))
except Exception:
report['rmsd_xy'] = [0.0] * len(binner)
# Return the binned report
return report
def resolution_bins(experiment, hkl, nbins):
# Create the crystal symmetry object
cs = crystal.symmetry(
space_group=experiment.crystal.get_space_group(),
unit_cell=experiment.crystal.get_unit_cell())
# Create the resolution binner object
ms = miller.set(cs, hkl)
ms.setup_binner(n_bins=nbins)
binner = ms.binner()
brange = list(binner.range_used())
bins = [binner.bin_d_range(brange[0])[0]]
for i in brange:
bins.append(binner.bin_d_range(i)[1])
return flex.double(reversed(bins))
def select(data, indices):
# Select rows from columns
result = {}
for key, value in data.iteritems():
result[key] = value.select(indices)
return result
# Check the required columns are there
assert("miller_index" in reflections)
assert("d" in reflections)
assert("flags" in reflections)
assert("bbox" in reflections)
assert("xyzcal.px" in reflections)
assert("partiality" in reflections)
assert("intensity.sum.value" in reflections)
assert("intensity.sum.variance" in reflections)
# Get the flag enumeration
flags = flex.reflection_table.flags
# Get some keys from the data
data = {}
for key in ['miller_index',
'xyzcal.px',
'xyzobs.px.value',
'd',
'bbox',
'background.mean',
'partiality',
'intensity.sum.value',
'intensity.sum.variance',
'intensity.prf.value',
'intensity.prf.variance',
'profile.correlation']:
if key in reflections:
data[key] = reflections[key]
# Compute some flag stuff
data["full"] = data['partiality'] > 0.997300203937
data["over"] = reflections.get_flags(flags.overloaded)
data["ice"] = reflections.get_flags(flags.in_powder_ring)
data["sum"] = reflections.get_flags(flags.integrated_sum)
data["prf"] = reflections.get_flags(flags.integrated_prf)
data["int"] = reflections.get_flags(flags.integrated, all=False)
data["ninvbg"] = reflections.get_flags(flags.background_includes_bad_pixels)
data["ninvfg"] = reflections.get_flags(flags.foreground_includes_bad_pixels)
data["fbgd"] = reflections.get_flags(flags.failed_during_background_modelling)
data["fsum"] = reflections.get_flags(flags.failed_during_summation)
data["fprf"] = reflections.get_flags(flags.failed_during_profile_fitting)
# Try to calculate the i over sigma for summation
data['intensity.sum.ios'] = flex_ios(
data['intensity.sum.value'],
data['intensity.sum.variance'])
# Try to calculate the i over sigma for profile fitting
try:
data['intensity.prf.ios'] = flex_ios(
data['intensity.prf.value'],
data['intensity.prf.variance'])
except Exception:
pass
# Try to calculate the rmsd between observation and prediction
try:
xcal, ycal, zcal = data['xyzcal.px'].parts()
xobs, yobs, zobs = data['xyzobs.px.value'].parts()
data['xyz.rmsd'] = flex.sqrt(
flex.pow2(xcal - xobs) +
flex.pow2(ycal - yobs))
except Exception:
pass
# Create the resolution binner
resolution_binner = Binner(
resolution_bins(
experiment,
data['miller_index'],
n_resolution_bins))
# Create the frame binner object
try:
array_range = experiment.imageset.get_array_range()
except:
array_range = (0, len(experiment.imageset))
frame_binner = Binner(flex.int(range(
array_range[0],
array_range[1]+1)).as_double())
# Create the overall report
overall = overall_report(data)
# Create high/low resolution reports
hl_binner = resolution_binner.indexer(data['d'])
high_summary = overall_report(select(data, hl_binner.indices(0)))
low_summary = overall_report(select(data, hl_binner.indices(n_resolution_bins-1)))
high_summary['dmin'] = resolution_binner.bins()[0]
high_summary['dmax'] = resolution_binner.bins()[1]
low_summary['dmin'] = resolution_binner.bins()[n_resolution_bins-1]
low_summary['dmax'] = resolution_binner.bins()[n_resolution_bins]
overall['dmin'] = high_summary['dmin']
overall['dmax'] = low_summary['dmax']
# Create the overall report
summary = OrderedDict([
('overall', overall),
('low', low_summary),
('high', high_summary)
])
# Create a report binned by resolution
resolution = binned_report(resolution_binner, data['d'], data)
# Create the report binned by image
image = binned_report(frame_binner, data['xyzcal.px'].parts()[2], data)
# Return the report
return OrderedDict([
("summary", summary),
("resolution", resolution),
("image", image)])
def residual(two_thetas_obs, miller_indices, wavelength, unit_cell): two_thetas_calc = unit_cell.two_theta(miller_indices, wavelength, deg=True) return flex.sum(flex.pow2(two_thetas_obs - two_thetas_calc))
def estimate_resolution_limit_distl_method2(
reflections, imageset, ice_sel=None, plot_filename=None):
# Implementation of Method 2 (section 2.4.4) of:
# Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
# J. Appl. Cryst. (2006). 39, 112-119
# http://dx.doi.org/10.1107/S0021889805040677
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = reflections['intensity.sum.value']
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
reflections = reflections.select(sel)
intensities = reflections['intensity.sum.value']
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
d_star_cubed = flex.pow(reflections['rlp'].norms(), 3)
binner = binner_d_star_cubed(d_spacings)
bin_counts = flex.size_t()
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#sel = ~(ice_sel) & sel_all
sel = sel_all
bin_counts.append(sel.count(True))
#print list(bin_counts)
t0 = (bin_counts[0] + bin_counts[1])/2
mu = 0.15
for i in range(len(bin_counts)-1):
tj = bin_counts[i]
tj1 = bin_counts[i+1]
if (tj < (mu * t0)) and (tj1 < (mu * t0)):
break
d_min = binner.bins[i].d_min
noisiness = 0
m = len(bin_counts)
for i in range(m):
for j in range(i+1, m):
if bin_counts[i] <= bin_counts[j]:
noisiness += 1
noisiness /= (0.5 * m * (m-1))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(range(len(bin_counts)), bin_counts)
#ax.set_xlabel('')
ax.set_ylabel('number of spots in shell')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
ax.vlines(i, ylim[0], ylim[1], colors='red')
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.savefig(plot_filename)
pyplot.close()
return d_min, noisiness