def setup_binner(
self,
unit_cell: uctbx.unit_cell,
space_group: sgtbx.space_group,
n_resolution_bins: int,
) -> None:
"""Create a binner for the reflections contained in the table."""
ma = _reflection_table_to_iobs(self.as_reflection_table(), unit_cell,
space_group)
# need d star sq step
d_star_sq = ma.d_star_sq().data()
d_star_sq_min = flex.min(d_star_sq)
d_star_sq_max = flex.max(d_star_sq)
span = d_star_sq_max - d_star_sq_min
relative_tolerance = 1e-6
d_star_sq_max += span * relative_tolerance
d_star_sq_min -= span * relative_tolerance
# Avoid a zero-size step that would otherwise anger the d_star_sq_step binner.
step = max((d_star_sq_max - d_star_sq_min) / n_resolution_bins, 0.004)
self.binner = ma.setup_binner_d_star_sq_step(
auto_binning=False,
d_max=uctbx.d_star_sq_as_d(d_star_sq_max),
d_min=uctbx.d_star_sq_as_d(d_star_sq_min),
d_star_sq_step=step,
)
def d_star_sq_to_d_ticks(d_star_sq, nticks):
from cctbx import uctbx
d_spacings = uctbx.d_star_sq_as_d(flex.double(d_star_sq))
min_d_star_sq = min(d_star_sq)
dstep = (max(d_star_sq) - min_d_star_sq) / nticks
tickvals = list(min_d_star_sq + (i * dstep) for i in range(nticks))
ticktext = ['%.2f' % (uctbx.d_star_sq_as_d(dsq)) for dsq in tickvals]
return tickvals, ticktext
def d_star_sq_to_d_ticks(d_star_sq, nticks): from cctbx import uctbx d_spacings = uctbx.d_star_sq_as_d(flex.double(d_star_sq)) min_d_star_sq = min(d_star_sq) dstep = (max(d_star_sq) - min_d_star_sq)/nticks tickvals = list(min_d_star_sq + (i*dstep) for i in range(nticks)) ticktext = ['%.2f' %(uctbx.d_star_sq_as_d(dsq)) for dsq in tickvals] return tickvals, ticktext
def count_ice_rings(self, width=0.002, verbose=False):
""" A function to find and count ice rings (modeled after
dials.algorithms.integration.filtering.PowderRingFilter, with some alterations:
1. Hard-coded with ice unit cell / space group
2. Returns spot counts vs. water diffraction resolution "bin"
Rather than finding ice rings themselves (which may be laborious and time
consuming), this method relies on counting the number of found spots that land
in regions of water diffraction. A weakness of this approach is that if any spot
filtering or spot-finding parameters are applied by prior methods, not all ice
rings may be found. This is acceptable, since the purpose of this method is to
determine if water and protein diffraction occupy the same resolutions.
"""
ice_start = time.time()
unit_cell = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
space_group = sgtbx.space_group_info(number=194).group()
# Correct unit cell
unit_cell = space_group.average_unit_cell(unit_cell)
half_width = width / 2
d_min = uctbx.d_star_sq_as_d(uctbx.d_as_d_star_sq(self.d_min) + half_width)
# Generate a load of indices
generator = index_generator(unit_cell, space_group.type(), False, d_min)
indices = generator.to_array()
# Compute d spacings and sort by resolution
d_star_sq = flex.sorted(unit_cell.d_star_sq(indices))
d = uctbx.d_star_sq_as_d(d_star_sq)
dd = list(zip(d_star_sq, d))
# identify if spots fall within ice ring areas
results = []
for ds2, d_res in dd:
result = [i for i in (flex.abs(self.ref_d_star_sq - ds2) < half_width) if i]
results.append((d_res, len(result)))
possible_ice = [r for r in results if r[1] / len(self.observed) * 100 >= 5]
if verbose:
print(
"SCORER: ice ring time = {:.5f} seconds".format(time.time() - ice_start)
)
self.n_ice_rings = len(possible_ice) # output in info
return self.n_ice_rings
def resolution_histogram(reflections, imageset, plot_filename=None):
d_star_sq = flex.pow2(reflections['rlp'].norms())
hist = get_histogram(d_star_sq)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.bar(hist.slot_centers() - 0.5 * hist.slot_width(),
hist.slots(),
width=hist.slot_width())
ax.set_xlabel("d_star_sq")
ax.set_ylabel("Frequency")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
def resolution_histogram(reflections, imageset, plot_filename=None):
d_star_sq = flex.pow2(reflections["rlp"].norms())
hist = get_histogram(d_star_sq)
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.bar(
hist.slot_centers() - 0.5 * hist.slot_width(),
hist.slots(),
width=hist.slot_width(),
)
ax.set_xlabel("d_star_sq")
ax.set_ylabel("Frequency")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
pyplot.savefig(plot_filename)
pyplot.close()
def stats_single_image(imageset, reflections, i=None, resolution_analysis=True,
plot=False):
reflections = map_to_reciprocal_space(reflections, imageset)
if plot and i is not None:
filename = "i_over_sigi_vs_resolution_%d.png" %(i+1)
hist_filename = "spot_count_vs_resolution_%d.png" %(i+1)
extra_filename = "log_sum_i_sigi_vs_resolution_%d.png" %(i+1)
distl_method_1_filename = "distl_method_1_%d.png" %(i+1)
distl_method_2_filename = "distl_method_2_%d.png" %(i+1)
else:
filename = None
hist_filename = None
extra_filename = None
distl_method_1_filename = None
distl_method_2_filename = None
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
#plot_ordered_d_star_sq(reflections, imageset)
reflections_all = reflections
ice_sel = ice_rings_selection(reflections_all)
reflections_no_ice = reflections_all.select(~ice_sel)
n_spots_total = len(reflections_all)
n_spots_no_ice = len(reflections_no_ice)
n_spot_4A = (d_spacings > 4).count(True)
intensities = reflections_no_ice['intensity.sum.value']
total_intensity = flex.sum(intensities)
#print i
if hist_filename is not None:
resolution_histogram(
reflections, imageset, plot_filename=hist_filename)
if extra_filename is not None:
log_sum_i_sigi_vs_resolution(
reflections, imageset, plot_filename=extra_filename)
if resolution_analysis and n_spots_no_ice > 10:
estimated_d_min = estimate_resolution_limit(
reflections_all, imageset, ice_sel=ice_sel, plot_filename=filename)
d_min_distl_method_1, noisiness_method_1 \
= estimate_resolution_limit_distl_method1(
reflections_all, imageset, ice_sel, plot_filename=distl_method_1_filename)
d_min_distl_method_2, noisiness_method_2 = \
estimate_resolution_limit_distl_method2(
reflections_all, imageset, ice_sel, plot_filename=distl_method_2_filename)
else:
estimated_d_min = -1.0
d_min_distl_method_1 = -1.0
noisiness_method_1 = -1.0
d_min_distl_method_2 = -1.0
noisiness_method_2 = -1.0
return group_args(n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=n_spot_4A,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
noisiness_method_1=noisiness_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_2=noisiness_method_2)
def resolution_histogram(reflections, imageset, plot_filename=None):
d_star_sq = flex.pow2(reflections['rlp'].norms())
hist = get_histogram(d_star_sq)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.bar(hist.slot_centers()-0.5*hist.slot_width(), hist.slots(),
width=hist.slot_width())
ax.set_xlabel("d_star_sq")
ax.set_ylabel("Frequency")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
def __init__(self, unit_cell, space_group, d_min, width):
"""
Initialise the filter.
:param unit_cell: The unit_cell of the powder rings
:param space_group: The space group of the powder rings
:param d_min: The maximum resolution to filter to
:param width: The resolution width to filter around
"""
assert d_min > 0
assert width > 0
# Correct unit cell
unit_cell = space_group.average_unit_cell(unit_cell)
self.half_width = width / 2.0
d_min = uctbx.d_star_sq_as_d(
uctbx.d_as_d_star_sq(d_min) + self.half_width)
# Generate a load of indices
generator = index_generator(unit_cell, space_group.type(), False,
d_min)
indices = generator.to_array()
# Compute d spacings and sort by resolution
self.d_star_sq = flex.sorted(unit_cell.d_star_sq(indices))
def make_merging_stats_plots(script):
"""Make merging stats plots for HTML report"""
d = {
"scaling_tables": ([], []),
"resolution_plots": {},
"batch_plots": {},
"misc_plots": {},
"anom_plots": {},
"image_range_tables": [],
}
(
batches,
rvb,
isigivb,
svb,
batch_data,
) = reflection_tables_to_batch_dependent_properties( # pylint: disable=unbalanced-tuple-unpacking
script.reflections,
script.experiments,
script.scaled_miller_array,
)
bm = batch_manager(batches, batch_data)
image_range_tables = make_image_range_table(script.experiments, bm)
if script.merging_statistics_result:
stats = script.merging_statistics_result
anom_stats = script.anom_merging_statistics_result
is_centric = script.scaled_miller_array.space_group().is_centric()
# Now calculate batch data
plotter = ResolutionPlotsAndStats(stats, anom_stats, is_centric)
d["resolution_plots"].update(plotter.make_all_plots())
d["scaling_tables"] = plotter.statistics_tables()
d["batch_plots"].update(scale_rmerge_vs_batch_plot(bm, rvb, svb))
d["batch_plots"].update(i_over_sig_i_vs_batch_plot(bm, isigivb))
plotter = IntensityStatisticsPlots(
script.scaled_miller_array, run_xtriage_analysis=False
)
d["resolution_plots"].update(plotter.generate_resolution_dependent_plots())
if d["resolution_plots"]["cc_one_half"]["data"][2]:
cc_anom = d["resolution_plots"]["cc_one_half"]["data"][2]["y"]
significance = d["resolution_plots"]["cc_one_half"]["data"][3]["y"]
sig = flex.double(cc_anom) > flex.double(significance)
max_anom = 0
for i, v in enumerate(sig):
if v:
max_anom = i
else:
break
d_min = uctbx.d_star_sq_as_d(plotter.binner.limits())[max_anom + 1]
else:
d_min = 0.0
d["misc_plots"].update(plotter.generate_miscellanous_plots())
intensities_anom = script.scaled_miller_array.as_anomalous_array()
intensities_anom = intensities_anom.map_to_asu().customized_copy(
info=script.scaled_miller_array.info()
)
anom_plotter = AnomalousPlotter(intensities_anom, strong_cutoff=d_min)
d["anom_plots"].update(anom_plotter.make_plots())
d["image_range_tables"] = [image_range_tables]
return d
def estimate_resolution_limit_distl_method2(reflections, plot_filename=None):
# Implementation of Method 2 (section 2.4.4) of:
# Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
# J. Appl. Cryst. (2006). 39, 112-119
# https://doi.org/10.1107/S0021889805040677
variances = reflections["intensity.sum.variance"]
sel = variances > 0
reflections = reflections.select(sel)
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
binner = binner_d_star_cubed(d_spacings)
bin_counts = flex.size_t()
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = sel_all
bin_counts.append(sel.count(True))
# print list(bin_counts)
t0 = (bin_counts[0] + bin_counts[1]) / 2
mu = 0.15
for i in range(len(bin_counts) - 1):
tj = bin_counts[i]
tj1 = bin_counts[i + 1]
if (tj < (mu * t0)) and (tj1 < (mu * t0)):
break
d_min = binner.bins[i].d_min
noisiness = 0
m = len(bin_counts)
for i in range(m):
for j in range(i + 1, m):
if bin_counts[i] <= bin_counts[j]:
noisiness += 1
noisiness /= 0.5 * m * (m - 1)
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(range(len(bin_counts)), bin_counts)
ax.set_ylabel("number of spots in shell")
xlim = pyplot.xlim()
ylim = pyplot.ylim()
ax.vlines(i, ylim[0], ylim[1], colors="red")
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.savefig(plot_filename)
pyplot.close()
return d_min, noisiness
def test_powder_ring_filter():
unit_cell = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
space_group = sgtbx.space_group_info(number=194).group()
d_spacings = flex.double([1.9220704466392748])
d_min = flex.min(d_spacings)
ice_filter = filtering.PowderRingFilter(unit_cell, space_group, d_min, width=0.004)
assert min(uctbx.d_star_sq_as_d(ice_filter.d_star_sq)) < d_min
assert all(ice_filter(d_spacings))
def filter_by_resolution(self, refl, d_min, d_max):
d_min = float(d_min) if d_min is not None else 0.1
d_max = float(d_max) if d_max is not None else 99
d_star_sq = flex.pow2(refl["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
filter = flex.bool(len(d_spacings), False)
filter = filter | (d_spacings >= d_min) & (d_spacings <= d_max)
return filter
def setup_binner(self, unit_cell, space_group, n_resolution_bins):
ma = _reflection_table_to_iobs(self.Ih_table, unit_cell, space_group)
# need d star sq step
d_star_sq = ma.d_star_sq().data()
d_star_sq_min = flex.min(d_star_sq)
d_star_sq_max = flex.max(d_star_sq)
span = d_star_sq_max - d_star_sq_min
relative_tolerance = 1e-6
d_star_sq_max += span * relative_tolerance
d_star_sq_min -= span * relative_tolerance
step = (d_star_sq_max - d_star_sq_min) / n_resolution_bins
self.binner = ma.setup_binner_d_star_sq_step(
auto_binning=False,
d_max=uctbx.d_star_sq_as_d(d_star_sq_max),
d_min=uctbx.d_star_sq_as_d(d_star_sq_min),
d_star_sq_step=step,
)
def make_plots(self):
"""Generate tables of overall and resolution-binned merging statistics."""
d = {
"scaling_tables": ([], []),
"resolution_plots": OrderedDict(),
"batch_plots": OrderedDict(),
"misc_plots": OrderedDict(),
"anom_plots": OrderedDict(),
"image_range_tables": [],
}
if "statistics" in self.data:
plotter = ResolutionPlotsAndStats(
self.data["statistics"],
self.data["anomalous_statistics"],
is_centric=self.data["is_centric"],
)
d["resolution_plots"].update(plotter.make_all_plots())
d["scaling_tables"] = plotter.statistics_tables()
d["batch_plots"].update(
scale_rmerge_vs_batch_plot(
self.data["bm"],
self.data["r_merge_vs_batch"],
self.data["scale_vs_batch"],
))
d["batch_plots"].update(
i_over_sig_i_vs_batch_plot(self.data["bm"],
self.data["isigivsbatch"]))
plotter = IntensityStatisticsPlots(
self.data["scaled_miller_array"], run_xtriage_analysis=False)
d["resolution_plots"].update(
plotter.generate_resolution_dependent_plots())
if d["resolution_plots"]["cc_one_half"]["data"][2]:
cc_anom = d["resolution_plots"]["cc_one_half"]["data"][2]["y"]
significance = d["resolution_plots"]["cc_one_half"]["data"][3][
"y"]
sig = flex.double(cc_anom) > flex.double(significance)
max_anom = 0
for i, v in enumerate(sig):
if v:
max_anom = i
else:
break
d_min = uctbx.d_star_sq_as_d(
plotter.binner.limits())[max_anom + 1]
else:
d_min = 0.0
d["misc_plots"].update(plotter.generate_miscellanous_plots())
intensities_anom = self.data[
"scaled_miller_array"].as_anomalous_array()
intensities_anom = intensities_anom.map_to_asu().customized_copy(
info=self.data["scaled_miller_array"].info())
anom_plotter = AnomalousPlotter(intensities_anom,
strong_cutoff=d_min)
d["anom_plots"].update(anom_plotter.make_plots())
d["image_range_tables"] = self.data["image_range_tables"]
return d
def savefig(self, filename):
from cctbx import uctbx
xticks = self.ax.get_xticks()
xticks_d = [
'%.2f' %uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
self.ax.set_xticklabels(xticks_d)
self.ax.set_xlabel('Resolution (A)')
self.ax.set_ylabel(self.ylabel)
self.ax.legend(loc='best')
self.fig.savefig(filename)
def savefig(self, filename):
from cctbx import uctbx
xticks = self.ax.get_xticks()
xticks_d = [
'%.2f' %uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
self.ax.set_xticklabels(xticks_d)
self.ax.set_xlabel('Resolution (A)')
self.ax.set_ylabel(self.ylabel)
self.ax.legend(loc='best')
self.fig.savefig(filename)
def resolution_fit(d_star_sq, y_obs, model, limit, sel=None):
"""Estimate a resolution limit based on the input merging statistics
The function defined by `model` will be fit to the input `d_star_sq` and `y_obs`.
The estimated resolution limit is chosen as the `d_star_sq` value at which the
fitted function equals `limit`.
Args:
d_star_sq (scitbx.array_family.flex.double): The high resolution limits of the
resolution bins in units 1/d*2
y_obs (scitbx.array_family.flex.double): The statistic against which to fit the
function `model`
model: The function to fit against `y_obs`. Must be callable, taking as input x
(d_star_sq) and y (the metric to be fitted) values, returning the fitted
y(x) values.
limit (float): The resolution limit criterion.
sel (scitbx.array_family.flex.bool): An optional selection to apply to the
`d_star_sq` and `y_obs` values.
Returns: The estimated resolution limit in units of Å^-1
Raises:
RuntimeError: Raised if no `y_obs` values remain after application of the
selection `sel`
"""
if not sel:
sel = flex.bool(len(d_star_sq), True)
sel &= y_obs > 0
y_obs = y_obs.select(sel)
d_star_sq = d_star_sq.select(sel)
if not len(y_obs):
raise RuntimeError("No reflections left for fitting")
y_fit = model(d_star_sq, y_obs, 6)
logger.debug(
tabulate(
[("d*2", "d", "obs", "fit")]
+ [
(ds2, uctbx.d_star_sq_as_d(ds2), yo, yf)
for ds2, yo, yf in zip(d_star_sq, y_obs, y_fit)
],
headers="firstrow",
)
)
if flex.min(y_obs) > limit:
d_min = 1.0 / math.sqrt(flex.max(d_star_sq))
else:
try:
d_min = 1.0 / math.sqrt(interpolate_value(d_star_sq, y_fit, limit))
except RuntimeError as e:
logger.debug(f"Error interpolating value: {e}")
d_min = None
return ResolutionResult(d_star_sq, y_obs, y_fit, d_min)
def quasi_normalisation(intensities):
"""Quasi-normalisation of the input intensities.
Args:
intensities (cctbx.miller.array): The intensities to be normalised.
Returns:
cctbx.miller.array: The normalised intensities.
"""
# handle negative reflections to minimise effect on mean I values.
work = intensities.deep_copy()
work.data().set_selected(work.data() < 0.0, 0.0)
# set up binning objects
if work.size() > 20000:
n_refl_shells = 20
elif work.size() > 15000:
n_refl_shells = 15
else:
n_refl_shells = 10
d_star_sq = work.d_star_sq().data()
d_star_sq_max = flex.max(d_star_sq)
d_star_sq_min = flex.min(d_star_sq)
span = d_star_sq_max - d_star_sq_min
d_star_sq_max += span * 1e-6
d_star_sq_min -= span * 1e-6
d_star_sq_step = (d_star_sq_max - d_star_sq_min) / n_refl_shells
work.setup_binner_d_star_sq_step(
d_min=uctbx.d_star_sq_as_d(
d_star_sq_min), # cctbx/cctbx_project#588
d_max=uctbx.d_star_sq_as_d(
d_star_sq_max), # cctbx/cctbx_project#588
d_star_sq_step=d_star_sq_step,
auto_binning=False,
)
normalisations = work.intensity_quasi_normalisations()
return intensities.customized_copy(
data=(intensities.data() / normalisations.data()),
sigmas=(intensities.sigmas() / normalisations.data()),
)
def stats_for_reflection_table(reflections,
resolution_analysis=True,
filter_ice=True,
ice_rings_width=0.004):
assert "rlp" in reflections, "Reflections must have been mapped to reciprocal space"
reflections = reflections.select(reflections["rlp"].norms() > 0)
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
reflections_all = reflections
reflections_no_ice = reflections_all
ice_sel = None
if reflections.size() and filter_ice:
ice_sel = ice_rings_selection(reflections_all, width=ice_rings_width)
if ice_sel is not None:
reflections_no_ice = reflections_all.select(~ice_sel)
n_spots_total = len(reflections_all)
n_spots_no_ice = len(reflections_no_ice)
n_spot_4A = (d_spacings > 4).count(True)
intensities = reflections_no_ice["intensity.sum.value"]
total_intensity = flex.sum(intensities)
if resolution_analysis and n_spots_no_ice > 10:
estimated_d_min = estimate_resolution_limit(reflections_all,
ice_sel=ice_sel)
(
d_min_distl_method_1,
noisiness_method_1,
) = estimate_resolution_limit_distl_method1(reflections_all)
(
d_min_distl_method_2,
noisiness_method_2,
) = estimate_resolution_limit_distl_method2(reflections_all)
else:
estimated_d_min = -1.0
d_min_distl_method_1 = -1.0
noisiness_method_1 = -1.0
d_min_distl_method_2 = -1.0
noisiness_method_2 = -1.0
return StatsSingleImage(
n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=n_spot_4A,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
noisiness_method_1=noisiness_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_2=noisiness_method_2,
)
def log_sum_i_sigi_vs_resolution(reflections, imageset, plot_filename=None):
d_star_sq = flex.pow2(reflections['rlp'].norms())
hist = get_histogram(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = intensities.select(sel)
i_over_sigi = intensities / flex.sqrt(variances)
#log_i_over_sigi = flex.log(i_over_sigi)
slots = []
for slot in hist.slot_infos():
sel = (d_star_sq > slot.low_cutoff) & (d_star_sq < slot.high_cutoff)
if sel.count(True) > 0:
slots.append(math.log(flex.sum(i_over_sigi.select(sel))))
else:
slots.append(0)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
#ax.bar(hist.slot_centers()-0.5*hist.slot_width(), hist.slots(),
ax.scatter(hist.slot_centers() - 0.5 * hist.slot_width(),
slots,
s=20,
color='blue',
marker='o',
alpha=0.5)
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(sum(I/sigI))")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
def ice_rings_selection(reflections):
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
from dials.algorithms.integration import filtering
unit_cell = uctbx.unit_cell((4.498,4.498,7.338,90,90,120))
space_group = sgtbx.space_group_info(number=194).group()
width = 0.06
ice_filter = filtering.PowderRingFilter(
unit_cell, space_group, flex.min(d_spacings)-width, width)
ice_sel = ice_filter(d_spacings)
return ice_sel
def __init__(self, d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5):
n_slots = len(d_star_sq) // target_n_per_bin
if max_slots is not None:
n_slots = min(n_slots, max_slots)
if min_slots is not None:
n_slots = max(n_slots, min_slots)
self.bins = []
n_per_bin = len(d_star_sq) / n_slots
d_star_sq_sorted = flex.sorted(d_star_sq)
d_sorted = uctbx.d_star_sq_as_d(d_star_sq_sorted)
d_max = d_sorted[0]
for i in range(n_slots):
d_min = d_sorted[nint((i + 1) * n_per_bin) - 1]
self.bins.append(Slot(d_min, d_max))
d_max = d_min
def update_dvals(self):
refls = self.refls_pruned
if not self.centroid_px_mm_done:
refls.centroid_px_to_mm(self.experiments)
self.centroid_px_mm_done = True
refls.map_centroids_to_reciprocal_space(self.experiments)
d_star_sq = flex.pow2(refls['rlp'].norms())
refls['d'] = uctbx.d_star_sq_as_d(d_star_sq)
sel_lt = refls['d'] < self.params.center_scan.d_max
sel_gt = refls['d'] > self.params.center_scan.d_min
sel = sel_lt & sel_gt
count = sel.count(True)
if count > self.target_refl_count:
self.target_refl_count = count
self.dvals = refls.select(sel)['d']
def ice_rings_selection(reflections, width=0.004):
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
unit_cell = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
space_group = sgtbx.space_group_info(number=194).group()
if d_spacings:
ice_filter = filtering.PowderRingFilter(unit_cell, space_group,
flex.min(d_spacings), width)
ice_sel = ice_filter(d_spacings)
return ice_sel
else:
return None
def __init__(self, d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5):
from libtbx.math_utils import nearest_integer as nint
n_slots = len(d_star_sq)//target_n_per_bin
if max_slots is not None:
n_slots = min(n_slots, max_slots)
if min_slots is not None:
n_slots = max(n_slots, min_slots)
self.bins = []
n_per_bin = len(d_star_sq)/n_slots
d_star_sq_sorted = flex.sorted(d_star_sq)
d_sorted = uctbx.d_star_sq_as_d(d_star_sq_sorted)
d_max = d_sorted[0]
for i in range(n_slots):
d_min = d_sorted[nint((i+1)*n_per_bin)-1]
self.bins.append(slot(d_min, d_max))
d_max = d_min
def _filter_by_resolution(experiments, reflections, d_min=None, d_max=None):
reflections.centroid_px_to_mm(experiments)
reflections.map_centroids_to_reciprocal_space(experiments)
d_star_sq = flex.pow2(reflections["rlp"].norms())
reflections["d"] = uctbx.d_star_sq_as_d(d_star_sq)
# Filter based on resolution
if d_min is not None:
selection = reflections["d"] >= d_min
reflections = reflections.select(selection)
logger.debug(f"Selected {len(reflections)} reflections with d >= {d_min:f}")
# Filter based on resolution
if d_max is not None:
selection = reflections["d"] <= d_max
reflections = reflections.select(selection)
logger.debug(f"Selected {len(reflections)} reflections with d <= {d_max:f}")
return reflections
def log_sum_i_sigi_vs_resolution(reflections, imageset, plot_filename=None):
d_star_sq = flex.pow2(reflections['rlp'].norms())
hist = get_histogram(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = intensities.select(sel)
i_over_sigi = intensities/flex.sqrt(variances)
#log_i_over_sigi = flex.log(i_over_sigi)
slots = []
for slot in hist.slot_infos():
sel = (d_star_sq > slot.low_cutoff) & (d_star_sq < slot.high_cutoff)
if sel.count(True) > 0:
slots.append(math.log(flex.sum(i_over_sigi.select(sel))))
else:
slots.append(0)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
#ax.bar(hist.slot_centers()-0.5*hist.slot_width(), hist.slots(),
ax.scatter(hist.slot_centers()-0.5*hist.slot_width(), slots, s=20, color='blue', marker='o', alpha=0.5)
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(sum(I/sigI))")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
def plot_merging_stats(results, labels=None, plots=None, prefix=None,
size_inches=None, image_dir=None):
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
pyplot.style.use('ggplot')
if plots is None:
plots = ('r_merge', 'r_meas', 'r_pim', 'cc_one_half', 'i_over_sigma_mean')
if prefix is None:
prefix = ''
if labels is not None:
assert len(results) == len(labels)
if image_dir is None:
image_dir = '.'
for k in plots:
for i, result in enumerate(results):
if labels is not None:
label = labels[i]
else:
label = None
bins = result.bins
x = [bins[i].d_min for i in range(len(bins))]
x = [uctbx.d_as_d_star_sq(d) for d in x]
y = [getattr(bins[i], k) for i in range(len(bins))]
pyplot.plot(x, y, label=label)
pyplot.xlabel('d spacing')
pyplot.ylabel(k)
if k == 'cc_one_half':
pyplot.ylim(0, 1.05)
else:
pyplot.ylim(0, pyplot.ylim()[1])
ax = pyplot.gca()
xticks = ax.get_xticks()
xticks_d = [
'%.2f' %uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
ax.set_xticklabels(xticks_d)
if size_inches is not None:
fig = pyplot.gcf()
fig.set_size_inches(size_inches)
if labels is not None:
pyplot.legend(loc='best')
pyplot.tight_layout()
pyplot.savefig(os.path.join(image_dir, prefix+ k + '.png'))
pyplot.clf()
def ice_rings_selection(reflections):
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
from dials.algorithms.integration import filtering
unit_cell = uctbx.unit_cell((4.498,4.498,7.338,90,90,120))
space_group = sgtbx.space_group_info(number=194).group()
width = 0.06
if d_spacings:
ice_filter = filtering.PowderRingFilter(
unit_cell, space_group, flex.min(d_spacings)-width, width)
ice_sel = ice_filter(d_spacings)
return ice_sel
else:
return None
def __init__(self, experiments, observed, config):
self.experiments = experiments
self.observed = observed
self.cfg = config
# extract reflections and map to reciprocal space
self.refl = observed.select(observed["id"] == 0)
self.refl.centroid_px_to_mm([experiments[0]])
self.refl.map_centroids_to_reciprocal_space([experiments[0]])
# calculate d-spacings
self.ref_d_star_sq = flex.pow2(self.refl["rlp"].norms())
self.d_spacings = uctbx.d_star_sq_as_d(self.ref_d_star_sq)
self.d_min = flex.min(self.d_spacings)
# initialize desired parameters (so that can back out without having to
# re-run functions)
self.n_ice_rings = 0
self.mean_spot_shape_ratio = 1
self.n_overloads = self.count_overloads()
self.hres = 99.9
self.n_spots = 0
def estimate_resolution_limit(reflections, ice_sel=None, plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections["intensity.sum.value"]
variances = reflections["intensity.sum.variance"]
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities / flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(d_star_sq.select(~ice_sel),
log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(d_star_sq,
target_n_per_bin=20,
max_slots=20,
min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1 - low_percentile_limit
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
if sel.count(True) == 0:
continue
outliers = wilson_outliers(reflections.select(sel_all),
ice_sel=ice_sel.select(sel_all))
outliers_all.set_selected(sel_all, outliers)
isel = sel_all.iselection().select(~(outliers)
& ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
if m_upper == m_lower:
intersection = (-1, -1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
# with:
# a_ = m_upper
# b_ = m_lower
# c_ = c_upper
# d_ = c_lower
# intersection == ((d_ - c_) / (a_ - b_), (a_ * d_ - b_ * c_) / (a_ - b_))
intersection = (
(c_lower - c_upper) / (m_upper - m_lower),
(m_upper * c_lower - m_lower * c_upper) / (m_upper - m_lower),
)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
c_upper)
inside = inside & ~outliers_all & ~ice_sel
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(d_star_sq, log_i_over_sigi, marker="+")
ax.scatter(
d_star_sq.select(inside),
log_i_over_sigi.select(inside),
marker="+",
color="green",
)
ax.scatter(
d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker="+",
color="black",
)
ax.scatter(
d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker="+",
color="grey",
)
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker="+", color="red")
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker="+", color="red")
if intersection[0] <= ax.get_xlim(
)[1] and intersection[1] <= ax.get_ylim()[1]:
ax.scatter([intersection[0]], [intersection[1]],
marker="x",
s=50,
color="b")
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color="red")
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color="red")
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(
uctbx.d_as_d_star_sq(slot.d_max),
0,
ax.get_ylim()[1],
linestyle="dotted",
color="grey",
)
ax.vlines(
uctbx.d_as_d_star_sq(slot.d_min),
0,
ax.get_ylim()[1],
linestyle="dotted",
color="grey",
)
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def quasi_normalisation(reflection_table, experiment):
"""Calculate normalised intensity (Esq) values for reflections, for the purpose
of selecting subsets based on Esq for scaling. If more involved analyses of
normalised intensities are needed, then it may be necessary to split this
procedure to handle acentric and centric reflections separately."""
logger.info(
"Calculating normalised intensity values to select a reflection \n"
"subset for scaling. \n")
logger.debug(
"Negative intensities are set to zero for the purpose of \n"
"calculating mean intensity values for resolution bins. This is to avoid \n"
"spuriously high E^2 values due to a mean close to zero and should only \n"
"affect the E^2 values of the highest resolution bins. \n")
good_refl_sel = ~reflection_table.get_flags(
reflection_table.flags.bad_for_scaling, all=False)
rt_subset = reflection_table.select(good_refl_sel)
# Scaling subset is data that has not been flagged as bad or excluded
miller_set = miller.set(
crystal_symmetry=experiment.crystal.get_crystal_symmetry(),
indices=rt_subset["miller_index"],
)
# handle negative reflections to minimise effect on mean I values.
rt_subset["intensity_for_norm"] = copy.deepcopy(rt_subset["intensity"])
rt_subset["intensity_for_norm"].set_selected(rt_subset["intensity"] < 0.0,
0.0)
miller_array = miller.array(miller_set,
data=rt_subset["intensity_for_norm"])
n_refl = rt_subset.size()
# set up binning objects
if n_refl > 20000:
n_refl_shells = 20
elif n_refl > 15000:
n_refl_shells = 15
elif n_refl > 10000:
n_refl_shells = 10
else:
logger.info(
"No normalised intensity values were calculated, as an insufficient\n"
"number of reflections were detected. All normalised intensity \n"
"values will be set to 1 to allow use in scaling model determination. \n"
)
reflection_table["Esq"] = flex.double(reflection_table.size(), 1.0)
return reflection_table
d_star_sq = miller_array.d_star_sq().data()
d_star_sq_min = flex.min(d_star_sq)
d_star_sq_max = flex.max(d_star_sq)
span = d_star_sq_max - d_star_sq_min
relative_tolerance = 1e-6
d_star_sq_max += span * relative_tolerance
d_star_sq_min -= span * relative_tolerance
step = (d_star_sq_max - d_star_sq_min) / n_refl_shells
miller_array.setup_binner_d_star_sq_step(
auto_binning=False,
d_max=uctbx.d_star_sq_as_d(d_star_sq_max),
d_min=uctbx.d_star_sq_as_d(d_star_sq_min),
d_star_sq_step=step,
)
normalisations = miller_array.intensity_quasi_normalisations()
normalised_intensities = miller_array.customized_copy(
data=(miller_array.data() / normalisations.data()))
reflection_table["Esq"] = flex.double(reflection_table.size(), 0.0)
reflection_table["Esq"].set_selected(good_refl_sel,
normalised_intensities.data())
return reflection_table
def estimate_resolution_limit(reflections, imageset, ice_sel=None,
plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities/flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(
d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(
d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1-low_percentile_limit
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
#sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#print "%.2f" %(sel.count(True)/sel_all.count(True))
if sel.count(True) == 0:
#outliers_all.set_selected(sel_all & ice_sel, True)
continue
#if i_slot > i_slot_max:
#break
#else:
#continue
outliers = wilson_outliers(
reflections.select(sel_all), ice_sel=ice_sel.select(sel_all))
#print "rejecting %d wilson outliers" %outliers.count(True)
outliers_all.set_selected(sel_all, outliers)
#if sel.count(True)/sel_all.count(True) < 0.25:
#outliers_all.set_selected(sel_all & ice_sel, True)
#from scitbx.math import median_statistics
#intensities_sel = intensities.select(sel)
#stats = median_statistics(intensities_sel)
#z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
#outliers = z_score > 3.5
#perm = flex.sort_permutation(intensities_sel)
##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
##print ' '.join('%.2f' %v for v in z_score.select(perm))
##print
isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
#fit_upper.show_summary()
#fit_lower.show_summary()
if m_upper == m_lower:
intersection = (-1,-1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
intersection = (
(c_lower-c_upper)/(m_upper-m_lower),
(m_upper*c_lower-m_lower*c_upper)/(m_upper-m_lower))
a = m_upper
c_ = c_upper
b = m_lower
d = c_lower
assert intersection == ((d-c_)/(a-b), (a*d-b*c_)/(a-b))
#inside = points_inside_envelope(
#d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper)
inside = inside & ~outliers_all
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
#d_star_sq_estimate = intersection[0]
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
#resolution_estimate = max(resolution_estimate, flex.min(d_spacings))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
ax.scatter(d_star_sq.select(inside), log_i_over_sigi.select(inside),
marker='+', color='green')
ax.scatter(d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker='+', color='black')
ax.scatter(d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker='+', color='grey')
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')
if (intersection[0] <= ax.get_xlim()[1] and
intersection[1] <= ax.get_ylim()[1]):
ax.scatter([intersection[0]], [intersection[1]], marker='x', s=50, color='b')
#ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
ax.set_xlabel('d_star_sq')
ax.set_ylabel('ln(I/sigI)')
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax.vlines(uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def d_star_sq_to_d_ticks(d_star_sq, nticks):
min_d_star_sq = min(d_star_sq)
dstep = (max(d_star_sq) - min_d_star_sq) / nticks
tickvals = list(min_d_star_sq + (i * dstep) for i in range(nticks))
ticktext = ["%.2f" % (uctbx.d_star_sq_as_d(dsq)) for dsq in tickvals]
return tickvals, ticktext
def run(args):
import libtbx.load_env
usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments) > 1:
raise Sorry("More than one experiment present")
experiment = experiments[0]
assert(len(reflections) == 1)
reflections = reflections[0]
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
if 'intensity.prf.value' in reflections:
intensities = reflections['intensity.prf.value']
variances = reflections['intensity.prf.variance']
sel = (variances > 0)
intensities = intensities.select(sel)
variances = variances.select(sel)
sigmas = flex.sqrt(variances)
indices = reflections['miller_index'].select(sel)
from cctbx import crystal, miller
crystal_symmetry = crystal.symmetry(
space_group=experiment.crystal.get_space_group(),
unit_cell=experiment.crystal.get_unit_cell())
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=True,
indices=indices)
miller_array = miller.array(
miller_set=miller_set,
data=intensities,
sigmas=sigmas).set_observation_type_xray_intensity()
#miller_array.setup_binner(n_bins=50, reflections_per_bin=100)
miller_array.setup_binner(auto_binning=True, n_bins=20)
result = miller_array.i_over_sig_i(use_binning=True)
result.show()
from cctbx import uctbx
d_star_sq_centre = result.binner.bin_centers(2)
i_over_sig_i = flex.double(
[d if d is not None else 0 for d in result.data[1:-1]])
sel = (i_over_sig_i > 0)
d_star_sq_centre = d_star_sq_centre.select(sel)
i_over_sig_i = i_over_sig_i.select(sel)
log_i_over_sig_i = flex.log(i_over_sig_i)
weights = result.binner.counts()[1:-1].as_double().select(sel)
fit = flex.linear_regression(
d_star_sq_centre, log_i_over_sig_i, weights=weights)
m = fit.slope()
c = fit.y_intercept()
import math
y_cutoff = math.log(params.i_sigi_cutoff)
x_cutoff = (y_cutoff - c)/m
estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff)
print "estimated d_min: %.2f" %estimated_d_min
if params.plot:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.plot(
list(d_star_sq_centre),
list(log_i_over_sig_i),
label=r"ln(I/sigI)")
ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()], color='red')
ax.plot([x_cutoff, x_cutoff], pyplot.ylim(), color='grey', linestyle='dashed')
ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff], color='grey', linestyle='dashed')
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
pyplot.savefig("estimate_resolution_limit.png")
pyplot.clf()
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
import libtbx.load_env
usage = "%s [options] datablock.json" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=True,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0 or len(reflections) == 0:
parser.print_help()
exit(0)
imagesets = experiments.imagesets()
reflections = reflections[0]
shadowed = filter_shadowed_reflections(experiments, reflections)
print "# shadowed reflections: %i/%i (%.2f%%)" %(
shadowed.count(True), shadowed.size(),
shadowed.count(True)/shadowed.size() * 100)
expt = experiments[0]
x,y,z = reflections['xyzcal.px'].parts()
z_ = z * expt.scan.get_oscillation()[1]
zmin, zmax = expt.scan.get_oscillation_range()
hist_scan_angle = flex.histogram(z_.select(shadowed), n_slots=int(zmax-zmin))
#hist_scan_angle.show()
uc = experiments[0].crystal.get_unit_cell()
d_spacings = uc.d(reflections['miller_index'])
ds2 = uc.d_star_sq(reflections['miller_index'])
hist_res = flex.histogram(ds2.select(shadowed), flex.min(ds2), flex.max(ds2), n_slots=20, )
#hist_res.show()
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
plt.hist2d(z_.select(shadowed).as_numpy_array(),
ds2.select(shadowed).as_numpy_array(), bins=(40,40),
range=((flex.min(z_), flex.max(z_)),(flex.min(ds2), flex.max(ds2))))
yticks_dsq = flex.double(plt.yticks()[0])
from cctbx import uctbx
yticks_d = uctbx.d_star_sq_as_d(yticks_dsq)
plt.axes().set_yticklabels(['%.2f' %y for y in yticks_d])
plt.xlabel('Scan angle (degrees)')
plt.ylabel('Resolution (A^-1)')
cbar = plt.colorbar()
cbar.set_label('# shadowed reflections')
plt.savefig('n_shadowed_hist2d.png')
plt.clf()
plt.scatter(hist_scan_angle.slot_centers().as_numpy_array(), hist_scan_angle.slots().as_numpy_array())
plt.xlabel('Scan angle (degrees)')
plt.ylabel('# shadowed reflections')
plt.savefig("n_shadowed_vs_scan_angle.png")
plt.clf()
plt.scatter(hist_res.slot_centers().as_numpy_array(), hist_res.slots().as_numpy_array())
plt.xlabel('d_star_sq')
plt.savefig("n_shadowed_vs_resolution.png")
plt.clf()
def estimate_resolution_limit_distl_method2(
reflections, imageset, ice_sel=None, plot_filename=None):
# Implementation of Method 2 (section 2.4.4) of:
# Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
# J. Appl. Cryst. (2006). 39, 112-119
# http://dx.doi.org/10.1107/S0021889805040677
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = reflections['intensity.sum.value']
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
reflections = reflections.select(sel)
intensities = reflections['intensity.sum.value']
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
d_star_cubed = flex.pow(reflections['rlp'].norms(), 3)
binner = binner_d_star_cubed(d_spacings)
bin_counts = flex.size_t()
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#sel = ~(ice_sel) & sel_all
sel = sel_all
bin_counts.append(sel.count(True))
#print list(bin_counts)
t0 = (bin_counts[0] + bin_counts[1])/2
mu = 0.15
for i in range(len(bin_counts)-1):
tj = bin_counts[i]
tj1 = bin_counts[i+1]
if (tj < (mu * t0)) and (tj1 < (mu * t0)):
break
d_min = binner.bins[i].d_min
noisiness = 0
m = len(bin_counts)
for i in range(m):
for j in range(i+1, m):
if bin_counts[i] <= bin_counts[j]:
noisiness += 1
noisiness /= (0.5 * m * (m-1))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(range(len(bin_counts)), bin_counts)
#ax.set_xlabel('')
ax.set_ylabel('number of spots in shell')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
ax.vlines(i, ylim[0], ylim[1], colors='red')
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.savefig(plot_filename)
pyplot.close()
return d_min, noisiness
def estimate_resolution_limit_distl_method1(
reflections, imageset, ice_sel=None, plot_filename=None):
# Implementation of Method 1 (section 2.4.4) of:
# Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
# J. Appl. Cryst. (2006). 39, 112-119
# http://dx.doi.org/10.1107/S0021889805040677
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = reflections['intensity.sum.value']
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
reflections = reflections.select(sel)
intensities = reflections['intensity.sum.value']
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
d_star_cubed = flex.pow(reflections['rlp'].norms(), 3)
step = 2
while len(reflections)/step > 40:
step += 1
order = flex.sort_permutation(d_spacings, reverse=True)
ds3_subset = flex.double()
d_subset = flex.double()
for i in range(len(reflections)//step):
ds3_subset.append(d_star_cubed[order[i*step]])
d_subset.append(d_spacings[order[i*step]])
x = flex.double(range(len(ds3_subset)))
# (i)
# Usually, Pm is the last point, that is, m = n. But m could be smaller than
# n if an unusually high number of spots are detected around a certain
# intermediate resolution. In that case, our search for the image resolution
# does not go outside the spot 'bump;. This is particularly useful when
# ice-rings are present.
slopes = (ds3_subset[1:] - ds3_subset[0])/(x[1:]-x[0])
skip_first = 3
p_m = flex.max_index(slopes[skip_first:]) + 1 + skip_first
# (ii)
from scitbx import matrix
x1 = matrix.col((0, ds3_subset[0]))
x2 = matrix.col((p_m, ds3_subset[p_m]))
gaps = flex.double([0])
v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize()
for i in range(1, p_m):
x0 = matrix.col((i, ds3_subset[i]))
r = x1 - x0
g = abs(v.dot(r))
gaps.append(g)
mv = flex.mean_and_variance(gaps)
s = mv.unweighted_sample_standard_deviation()
# (iii)
p_k = flex.max_index(gaps)
g_k = gaps[p_k]
p_g = p_k
for i in range(p_k+1, len(gaps)):
g_i = gaps[i]
if g_i > (g_k - 0.5 * s):
p_g = i
ds3_g = ds3_subset[p_g]
d_g = d_subset[p_g]
noisiness = 0
n = len(ds3_subset)
for i in range(n-1):
for j in range(i+1, n-1):
if slopes[i] >= slopes[j]:
noisiness += 1
noisiness /= ((n-1)*(n-2)/2)
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(range(len(ds3_subset)), ds3_subset)
#ax.set_xlabel('')
ax.set_ylabel('D^-3')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
ax.vlines(p_g, ylim[0], ylim[1], colors='red')
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.savefig(plot_filename)
pyplot.close()
return d_g, noisiness
def exercise_functions():
d_star_sq_s = 1.2345
d_star_sq_a = flex.double((1.2345, 0.1234))
two_theta_s = 0.61725
two_theta_a = flex.double((0.61725,0.65432))
# forward conversions
assert approx_equal(
uctbx.d_star_sq_as_stol_sq(d_star_sq_s), d_star_sq_s / 4)
assert approx_equal(
uctbx.d_star_sq_as_stol_sq(d_star_sq_a),
[uctbx.d_star_sq_as_stol_sq(i) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_two_stol(d_star_sq_s)**2, d_star_sq_s)
assert approx_equal(
uctbx.d_star_sq_as_two_stol(d_star_sq_a),
[uctbx.d_star_sq_as_two_stol(i) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_stol(d_star_sq_s)**2, d_star_sq_s / 4)
assert approx_equal(
uctbx.d_star_sq_as_stol(d_star_sq_a),
[uctbx.d_star_sq_as_stol(i) for i in d_star_sq_a])
assert approx_equal(
1/(uctbx.d_star_sq_as_d(d_star_sq_s)**2), d_star_sq_s)
assert approx_equal(
uctbx.d_star_sq_as_d(d_star_sq_a),
[uctbx.d_star_sq_as_d(i) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
2 * asin(1.5/2*sqrt(d_star_sq_s)))
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, False))
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_a, 1.5),
[uctbx.d_star_sq_as_two_theta(i, 1.5) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5)*180/pi,
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, True))
# reverse conversions
for d_star_sq, two_theta in zip(
(d_star_sq_s, d_star_sq_a),(two_theta_s, two_theta_a)):
assert approx_equal(
uctbx.stol_sq_as_d_star_sq(
uctbx.d_star_sq_as_stol_sq(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.two_stol_as_d_star_sq(
uctbx.d_star_sq_as_two_stol(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.stol_as_d_star_sq(
uctbx.d_star_sq_as_stol(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.d_as_d_star_sq(
uctbx.d_star_sq_as_d(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.two_theta_as_d_star_sq(
uctbx.d_star_sq_as_two_theta(d_star_sq, 1.5), 1.5), d_star_sq)
assert approx_equal(
uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
uctbx.two_theta_as_d_star_sq(two_theta, 1.5, False))
assert approx_equal(
uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
uctbx.two_theta_as_d_star_sq(two_theta*180/pi, 1.5, True))
assert approx_equal(
uctbx.two_theta_as_d(two_theta, 1.5),
uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5)))
assert approx_equal(
uctbx.two_theta_as_d(two_theta, 1.5, True),
uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5, True)))
#
assert uctbx.fractional_unit_shifts(distance_frac=[0,0,0]) == (0,0,0)
assert uctbx.fractional_unit_shifts([0.6,7.4,-0.4]) == (1,7,0)
assert uctbx.fractional_unit_shifts([-6,3,-0.6]) == (-6,3,-1)
site_frac_1 = [0.3,-8.6,2.1]
for eps,expected_u2 in [(-1.e-5, 1), (1.e-5, 2)]:
site_frac_2 = [-3,5.6,0.6-eps]
assert uctbx.fractional_unit_shifts(
site_frac_1=site_frac_1,
site_frac_2=site_frac_2) == (3, -14, expected_u2)
def run(args=None,
phil=phil_scope): # type: (List[str], libtbx.phil.scope) -> None
usage = "dials.missing_reflections [options] scaled.expt scaled.refl"
parser = OptionParser(
usage=usage,
phil=phil,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
params, options = parser.parse_args(args=args, show_diff_phil=False)
# Configure the logging.
dials.util.log.config(options.verbose)
logger.info(dials_version())
# Log the difference between the PHIL scope definition and the active PHIL scope,
# which will include the parsed user inputs.
diff_phil = parser.diff_phil.as_str()
if diff_phil:
logger.info("The following parameters have been modified:\n%s",
diff_phil)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if not experiments or not reflections:
parser.print_help()
return
if len(reflections) != 1 and len(experiments) != len(reflections):
sys.exit(
"Number of experiments must equal the number of reflection tables")
from dials.util.multi_dataset_handling import (
assign_unique_identifiers,
parse_multiple_datasets,
)
reflections = parse_multiple_datasets(reflections)
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
if all("inverse_scale_factor" in refl for refl in reflections):
# Assume all arrays have been scaled
miller_array = scaled_data_as_miller_array(reflections,
experiments,
anomalous_flag=False)
else:
# Else get the integrated intensities
miller_arrays = filtered_arrays_from_experiments_reflections(
experiments,
reflections,
)
miller_array = miller_arrays[0]
for ma in miller_arrays[1:]:
miller_array = miller_array.concatenate(ma)
if params.d_min or params.d_max:
miller_array = miller_array.resolution_filter(d_min=params.d_min,
d_max=params.d_max)
# Print overall summary of input miller array
s = io.StringIO()
ma_unique = miller_array.unique_under_symmetry()
ma_unique.show_comprehensive_summary(f=s)
logger.info(f"\n{s.getvalue()}")
# Get the regions of missing reflections
complete_set, unique_ms = missing_reflections.connected_components(
miller_array)
unique_ms = [
ms for ms in unique_ms if ms.size() >= params.min_component_size
]
# Print some output for user
if len(unique_ms):
logger.info(
"The following connected regions of missing reflections have been identified:"
)
n_expected = complete_set.size()
rows = []
for ms in unique_ms:
d_max, d_min = (uctbx.d_star_sq_as_d(ds2)
for ds2 in ms.min_max_d_star_sq())
rows.append([
ms.size(),
f"{100 * ms.size() / n_expected:.1f}",
f"{d_max:.2f}-{d_min:.2f}",
])
logger.info(
tabulate(
rows,
headers=["# reflections", "% missing",
"Resolution range (Å)"]))
else:
logger.info("No connected regions of missing reflections identified")
def run(args):
import matplotlib
matplotlib.use("Agg")
import libtbx.load_env
usage = "%s [options]" % libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage, phil=phil_scope, check_format=False, epilog=help_message
)
params, options, args = parser.parse_args(
show_diff_phil=True, return_unhandled=True
)
for mtz in args:
print(mtz)
assert os.path.isfile(mtz), mtz
import iotbx.merging_statistics
i_obs = iotbx.merging_statistics.select_data(mtz, data_labels=params.labels)
if params.space_group is not None:
i_obs = i_obs.customized_copy(space_group_info=params.space_group)
from scitbx.array_family import flex
# set the sigmas to 1, and calculate the mean intensities and internal variances
intensities_copy = i_obs.customized_copy(sigmas=flex.double(i_obs.size(), 1))
merging_internal = intensities_copy.merge_equivalents(
use_internal_variance=True
)
merged = merging_internal.array()
merging_external = i_obs.merge_equivalents(use_internal_variance=False)
sigmas_internal = merging_internal.array().sigmas()
sigmas_external = merging_external.array().sigmas()
variances_internal = flex.pow2(sigmas_internal)
variances_external = flex.pow2(sigmas_external)
n_bins = 100
i_obs.setup_binner_counting_sorted(n_bins=n_bins)
sigmas_ratio = sigmas_external / sigmas_internal
variance_ratio = variances_external / variances_internal
array_sr = merging_external.array().customized_copy(
data=sigmas_ratio, sigmas=None
)
array_sr.use_binning_of(i_obs)
mean_sr = array_sr.mean(use_binning=True)
ds2 = mean_sr.binner.bin_centers(2)
sr = mean_sr.data[1:-1]
array_vr = merging_external.array().customized_copy(
data=variance_ratio, sigmas=None
)
array_vr.use_binning_of(i_obs)
mean_vr = array_vr.mean(use_binning=True)
d_star_sq = mean_vr.binner.bin_centers(2)
vr = mean_vr.data[1:-1]
prefix = params.prefix
if prefix is None:
prefix = ""
from matplotlib import pyplot
pyplot.style.use("ggplot")
pyplot.plot(d_star_sq, sr)
ax = pyplot.gca()
xticks = ax.get_xticks()
xticks_d = [
"%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
ax.set_xticklabels(xticks_d)
pyplot.xlabel("d spacing (A)")
pyplot.ylabel("<sigI_ext/sigI_int>")
pyplot.savefig("%ssigmas_ratio.png" % prefix)
pyplot.clf()
pyplot.plot(d_star_sq, vr)
ax = pyplot.gca()
xticks = ax.get_xticks()
xticks_d = [
"%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
ax.set_xticklabels(xticks_d)
pyplot.xlabel("d spacing (A)")
pyplot.ylabel("<varI_ext/varI_int>")
pyplot.savefig("%svariances_ratio.png" % prefix)
pyplot.clf()
def estimate_resolution_limit_distl_method1(reflections, plot_filename=None):
# Implementation of Method 1 (section 2.4.4) of:
# Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon
# J. Appl. Cryst. (2006). 39, 112-119
# https://doi.org/10.1107/S0021889805040677
variances = reflections["intensity.sum.variance"]
sel = variances > 0
reflections = reflections.select(sel)
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
d_star_cubed = flex.pow(reflections["rlp"].norms(), 3)
step = 2
while len(reflections) / step > 40:
step += 1
order = flex.sort_permutation(d_spacings, reverse=True)
ds3_subset = flex.double()
d_subset = flex.double()
for i in range(len(reflections) // step):
ds3_subset.append(d_star_cubed[order[i * step]])
d_subset.append(d_spacings[order[i * step]])
x = flex.double(range(len(ds3_subset)))
# (i)
# Usually, Pm is the last point, that is, m = n. But m could be smaller than
# n if an unusually high number of spots are detected around a certain
# intermediate resolution. In that case, our search for the image resolution
# does not go outside the spot 'bump;. This is particularly useful when
# ice-rings are present.
slopes = (ds3_subset[1:] - ds3_subset[0]) / (x[1:] - x[0])
skip_first = 3
p_m = flex.max_index(slopes[skip_first:]) + 1 + skip_first
# (ii)
x1 = matrix.col((0, ds3_subset[0]))
x2 = matrix.col((p_m, ds3_subset[p_m]))
gaps = flex.double([0])
v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize()
for i in range(1, p_m):
x0 = matrix.col((i, ds3_subset[i]))
r = x1 - x0
g = abs(v.dot(r))
gaps.append(g)
mv = flex.mean_and_variance(gaps)
s = mv.unweighted_sample_standard_deviation()
# (iii)
p_k = flex.max_index(gaps)
g_k = gaps[p_k]
p_g = p_k
for i in range(p_k + 1, len(gaps)):
g_i = gaps[i]
if g_i > (g_k - 0.5 * s):
p_g = i
d_g = d_subset[p_g]
noisiness = 0
n = len(ds3_subset)
for i in range(n - 1):
for j in range(i + 1, n - 1):
if slopes[i] >= slopes[j]:
noisiness += 1
noisiness /= (n - 1) * (n - 2) / 2
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(range(len(ds3_subset)), ds3_subset)
ax.set_ylabel("D^-3")
xlim = pyplot.xlim()
ylim = pyplot.ylim()
ax.vlines(p_g, ylim[0], ylim[1], colors="red")
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.savefig(plot_filename)
pyplot.close()
return d_g, noisiness
def run(args):
import libtbx.load_env
usage = "%s [options]" % libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage, phil=phil_scope, check_format=False, epilog=help_message
)
params, options, args = parser.parse_args(
show_diff_phil=True, return_unhandled=True
)
space_group = params.space_group
if space_group is None:
space_group = sgtbx.space_group()
else:
space_group = space_group.group()
unit_cell = params.unit_cell
if unit_cell is None:
unit_cell = space_group.info().any_compatible_unit_cell(volume=100000)
print(unit_cell)
assert len(args) == 2
from cctbx import crystal, miller
cs = crystal.symmetry(space_group=space_group, unit_cell=unit_cell)
intensities = []
for filename in args:
hkl, i, sigi = parse_best_hkl(filename)
ms = miller.set(cs, hkl)
ma = miller.array(ms, data=i, sigmas=sigi)
ma.set_observation_type_xray_intensity()
intensities.append(ma)
# ma.show_summary()
# Two subplots, the axes array is 1-d
from matplotlib import pyplot
ma1, ma2 = intensities
hist1 = flex.histogram(ma1.data(), n_slots=100)
hist2 = flex.histogram(ma2.data(), n_slots=100)
f, axarr = pyplot.subplots(2, sharex=True, figsize=(16, 12))
axarr[0].bar(
hist1.slot_centers() - 0.5 * hist1.slot_width(),
hist1.slots(),
align="center",
width=hist1.slot_width(),
color="black",
edgecolor=None,
)
axarr[1].bar(
hist2.slot_centers() - 0.5 * hist2.slot_width(),
hist2.slots(),
align="center",
width=hist2.slot_width(),
color="black",
edgecolor=None,
)
pyplot.savefig("hist_intensities.png")
pyplot.clf()
hist1 = flex.histogram(ma1.data() / ma1.sigmas(), n_slots=100)
hist2 = flex.histogram(ma2.data() / ma2.sigmas(), n_slots=100)
f, axarr = pyplot.subplots(2, sharex=True, figsize=(16, 12))
axarr[0].bar(
hist1.slot_centers() - 0.5 * hist1.slot_width(),
hist1.slots(),
align="center",
width=hist1.slot_width(),
color="black",
edgecolor=None,
)
axarr[1].bar(
hist2.slot_centers() - 0.5 * hist2.slot_width(),
hist2.slots(),
align="center",
width=hist2.slot_width(),
color="black",
edgecolor=None,
)
pyplot.savefig("hist_isigi.png")
pyplot.clf()
print(ma1.d_max_min())
print(ma2.d_max_min())
ma1.setup_binner(n_bins=20)
ma2.setup_binner(n_bins=20)
imean1 = ma1.mean(use_binning=True)
imean2 = ma2.mean(use_binning=True)
f, axarr = pyplot.subplots(2, sharex=True, figsize=(16, 12))
axarr[0].plot(imean1.binner.bin_centers(2), imean1.data[1:-1])
axarr[1].plot(imean2.binner.bin_centers(2), imean2.data[1:-1])
ax = pyplot.gca()
xticks = ax.get_xticks()
xticks_d = ["%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
ax.set_xticklabels(xticks_d)
pyplot.xlabel("d spacing (A)")
pyplot.savefig("imean_vs_resolution.png")
pyplot.clf()
isigi1 = ma1.i_over_sig_i(use_binning=True)
isigi2 = ma2.i_over_sig_i(use_binning=True)
f, axarr = pyplot.subplots(2, sharex=True, figsize=(16, 12))
axarr[0].plot(isigi1.binner.bin_centers(2), isigi1.data[1:-1])
axarr[1].plot(isigi2.binner.bin_centers(2), isigi2.data[1:-1])
ax = pyplot.gca()
xticks = ax.get_xticks()
xticks_d = ["%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
ax.set_xticklabels(xticks_d)
pyplot.xlabel("d spacing (A)")
pyplot.savefig("isigi_vs_resolution.png")
pyplot.clf()
best_cb_op = None
best_count = 0
for i_op, op in enumerate(
space_group.build_derived_reflection_intensity_group(False).all_ops()
):
if not op.t().is_zero():
continue
cb_op = sgtbx.change_of_basis_op(op) # .inverse())
ma1, ma2 = intensities
ma1, ma2 = ma1.common_sets(ma2.change_basis(cb_op))
# print cb_op
# print ma1.size(), ma2.size()
if ma1.size() > best_count:
best_cb_op = cb_op
best_count = ma1.size()
print("Best cb_op: %s (%i matches)" % (best_cb_op, best_count))
ma1, ma2 = intensities
ma1, ma2 = ma1.common_sets(ma2.change_basis(best_cb_op))
from matplotlib import pyplot
pyplot.scatter(ma1.data(), ma2.data(), marker="+", alpha=0.5)
m = max(pyplot.xlim()[1], pyplot.ylim()[1])
pyplot.plot((0, m), (0, m), c="black")
pyplot.xlabel(args[0])
pyplot.ylabel(args[1])
pyplot.savefig("scatter_intensities.png")
pyplot.clf()
pyplot.scatter(ma1.sigmas(), ma2.sigmas(), marker="+", alpha=0.5)
m = max(pyplot.xlim()[1], pyplot.ylim()[1])
pyplot.plot((0, m), (0, m), c="black")
pyplot.savefig("scatter_sigmas.png")
pyplot.clf()
pyplot.scatter(
flex.pow2(ma1.sigmas()), flex.pow2(ma2.sigmas()), marker="+", alpha=0.5
)
m = max(pyplot.xlim()[1], pyplot.ylim()[1])
pyplot.plot((0, m), (0, m), c="black")
pyplot.savefig("scatter_variances.png")
pyplot.clf()
isigi1 = ma1.data() / ma1.sigmas()
isigi2 = ma2.data() / ma2.sigmas()
pyplot.scatter(isigi1, isigi2, marker="+", alpha=0.5)
m = max(pyplot.xlim()[1], pyplot.ylim()[1])
pyplot.plot((0, m), (0, m), c="black")
pyplot.savefig("scatter_i_sig_i.png")
pyplot.clf()
return
