def filter_ice(reflections, steps):
from cctbx import miller, sgtbx, uctbx
from matplotlib import pyplot as plt
d_spacings = 1 / reflections["rlp"].norms()
d_star_sq = uctbx.d_as_d_star_sq(d_spacings)
from dials.algorithms.spot_finding.per_image_analysis import ice_rings_selection
from dials.algorithms.integration import filtering
ice_uc = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
ice_sg = sgtbx.space_group_info(number=194).group()
ice_generator = miller.index_generator(ice_uc, ice_sg.type(), False,
flex.min(d_spacings))
ice_indices = ice_generator.to_array()
ice_d_spacings = flex.sorted(ice_uc.d(ice_indices))
ice_d_star_sq = uctbx.d_as_d_star_sq(ice_d_spacings)
cubic_ice_uc = uctbx.unit_cell((6.358, 6.358, 6.358, 90, 90, 90))
cubic_ice_sg = sgtbx.space_group_info(number=227).group()
cubic_ice_generator = miller.index_generator(cubic_ice_uc,
cubic_ice_sg.type(), False,
flex.min(d_spacings))
cubic_ice_indices = cubic_ice_generator.to_array()
cubic_ice_d_spacings = flex.sorted(cubic_ice_uc.d(cubic_ice_indices))
cubic_ice_d_star_sq = uctbx.d_as_d_star_sq(cubic_ice_d_spacings)
import numpy
widths = flex.double(numpy.geomspace(start=0.0001, stop=0.01, num=steps))
n_spots = flex.double()
total_intensity = flex.double()
for width in widths:
d_min = flex.min(d_spacings)
ice_filter = filtering.PowderRingFilter(ice_uc, ice_sg, d_min, width)
ice_sel = ice_filter(d_spacings)
n_spots.append(ice_sel.count(False))
if "intensity.sum.value" in reflections:
total_intensity.append(
flex.sum(reflections["intensity.sum.value"].select(~ice_sel)))
fig, axes = plt.subplots(nrows=2, figsize=(12, 8), sharex=True)
axes[0].plot(widths, n_spots, label="#spots", marker="+")
if total_intensity.size():
axes[1].plot(widths,
total_intensity,
label="total intensity",
marker="+")
axes[0].set_ylabel("# spots remaining")
axes[1].set_xlabel("Ice ring width (1/d^2)")
axes[1].set_ylabel("Total intensity")
for ax in axes:
ax.set_xlim(0, flex.max(widths))
plt.savefig("ice_ring_filtering.png")
plt.clf()
return
def __init__(
self, dataset_statistics, anomalous_dataset_statistics, is_centric=False
):
self.dataset_statistics = dataset_statistics
self.anomalous_dataset_statistics = anomalous_dataset_statistics
self.d_star_sq_bins = [
0.5 * (uctbx.d_as_d_star_sq(b.d_max) + uctbx.d_as_d_star_sq(b.d_min))
for b in self.dataset_statistics.bins
]
self.d_star_sq_tickvals, self.d_star_sq_ticktext = d_star_sq_to_d_ticks(
self.d_star_sq_bins, nticks=5
)
self.is_centric = is_centric
def plot_resolution_limit(self, d):
from cctbx import uctbx
d_star_sq = uctbx.d_as_d_star_sq(d)
self.ax.plot([d_star_sq, d_star_sq],
self.ax.get_ylim(),
linestyle="--")
def resolution_fit_from_merging_stats(merging_stats,
metric,
model,
limit,
sel=None):
"""Estimate a resolution limit based on the input `metric`
The function defined by `model` will be fit to the selected `metric` which has been
pre-calculated by the `merging_stats` object. The estimated resolution limit is
chosen as the `d_star_sq` value at which the fitted function equals `limit`.
Args:
merging_stats (iotbx.merging_statistics.dataset_statistics): Pre-calculated
merging statistics object
metric (str): The metric to use for estimating a resolution limit. Must be a
metric calculated by `iotbx.merging_statistics.merging_stats` and
available as an attribute on the `bins` attribute of the input
`merging_stats` object.
model: The function to fit to the selected metric. Must be callable, taking as
input x (d_star_sq) and y (the metric to be fitted) values, returning the
fitted y(x) values.
limit (float): The resolution limit criterion.
sel (scitbx.array_family.flex.bool): An optional selection to apply to the
`merging_stats` bins.
Returns: The estimated resolution limit in units of Å^-1
"""
y_obs = flex.double(getattr(b, metric)
for b in merging_stats.bins).reversed()
d_star_sq = flex.double(
uctbx.d_as_d_star_sq(b.d_min) for b in merging_stats.bins).reversed()
return resolution_fit(d_star_sq, y_obs, model, limit, sel=sel)
def __init__(self, unit_cell, space_group, d_min, width):
"""
Initialise the filter.
:param unit_cell: The unit_cell of the powder rings
:param space_group: The space group of the powder rings
:param d_min: The maximum resolution to filter to
:param width: The resolution width to filter around
"""
assert d_min > 0
assert width > 0
# Correct unit cell
unit_cell = space_group.average_unit_cell(unit_cell)
self.half_width = width / 2.0
d_min = uctbx.d_star_sq_as_d(
uctbx.d_as_d_star_sq(d_min) + self.half_width)
# Generate a load of indices
generator = index_generator(unit_cell, space_group.type(), False,
d_min)
indices = generator.to_array()
# Compute d spacings and sort by resolution
self.d_star_sq = flex.sorted(unit_cell.d_star_sq(indices))
def test_resolution_fit(merging_stats):
d_star_sq = flex.double(
uctbx.d_as_d_star_sq(b.d_min) for b in merging_stats.bins)
y_obs = flex.double(b.r_merge for b in merging_stats.bins)
result = resolution_analysis.resolution_fit(
d_star_sq, y_obs, resolution_analysis.log_inv_fit, 0.6)
assert result.d_min == pytest.approx(1.278, abs=1e-3)
assert flex.max(flex.abs(result.y_obs - result.y_fit)) < 0.05
def plot_data(results, k, labels, linestyle=None):
for i, result in enumerate(results):
if labels is not None:
label = labels[i]
else:
label = None
bins = result.bins
x = [bins[i].d_min for i in range(len(bins))]
x = [uctbx.d_as_d_star_sq(d) for d in x]
y = [getattr(bins[i], k) for i in range(len(bins))]
pyplot.plot(x, y, label=label, linestyle=linestyle)
def __call__(self, d):
"""
True if within powder ring.
:param d: The resolution
:return: True/False in powder ring
"""
result = flex.bool(len(d), False)
d_star_sq = uctbx.d_as_d_star_sq(d)
for ds2 in self.d_star_sq:
result = result | (flex.abs(d_star_sq - ds2) < self.half_width)
return result
def count_ice_rings(self, width=0.002, verbose=False):
""" A function to find and count ice rings (modeled after
dials.algorithms.integration.filtering.PowderRingFilter, with some alterations:
1. Hard-coded with ice unit cell / space group
2. Returns spot counts vs. water diffraction resolution "bin"
Rather than finding ice rings themselves (which may be laborious and time
consuming), this method relies on counting the number of found spots that land
in regions of water diffraction. A weakness of this approach is that if any spot
filtering or spot-finding parameters are applied by prior methods, not all ice
rings may be found. This is acceptable, since the purpose of this method is to
determine if water and protein diffraction occupy the same resolutions.
"""
ice_start = time.time()
unit_cell = uctbx.unit_cell((4.498, 4.498, 7.338, 90, 90, 120))
space_group = sgtbx.space_group_info(number=194).group()
# Correct unit cell
unit_cell = space_group.average_unit_cell(unit_cell)
half_width = width / 2
d_min = uctbx.d_star_sq_as_d(uctbx.d_as_d_star_sq(self.d_min) + half_width)
# Generate a load of indices
generator = index_generator(unit_cell, space_group.type(), False, d_min)
indices = generator.to_array()
# Compute d spacings and sort by resolution
d_star_sq = flex.sorted(unit_cell.d_star_sq(indices))
d = uctbx.d_star_sq_as_d(d_star_sq)
dd = list(zip(d_star_sq, d))
# identify if spots fall within ice ring areas
results = []
for ds2, d_res in dd:
result = [i for i in (flex.abs(self.ref_d_star_sq - ds2) < half_width) if i]
results.append((d_res, len(result)))
possible_ice = [r for r in results if r[1] / len(self.observed) * 100 >= 5]
if verbose:
print(
"SCORER: ice ring time = {:.5f} seconds".format(time.time() - ice_start)
)
self.n_ice_rings = len(possible_ice) # output in info
return self.n_ice_rings
def __call__(self, d):
'''
True if within powder ring.
:param d: The resolution
:return: True/False in powder ring
'''
from dials.array_family import flex
from cctbx import uctbx
result = flex.bool(len(d), False)
d_star_sq = uctbx.d_as_d_star_sq(d)
for ds2 in self.d_star_sq:
result = result | (flex.abs(d_star_sq - ds2) < self.half_width)
return result
def plot_merging_stats(results, labels=None, plots=None, prefix=None,
size_inches=None, image_dir=None):
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
pyplot.style.use('ggplot')
if plots is None:
plots = ('r_merge', 'r_meas', 'r_pim', 'cc_one_half', 'i_over_sigma_mean')
if prefix is None:
prefix = ''
if labels is not None:
assert len(results) == len(labels)
if image_dir is None:
image_dir = '.'
for k in plots:
for i, result in enumerate(results):
if labels is not None:
label = labels[i]
else:
label = None
bins = result.bins
x = [bins[i].d_min for i in range(len(bins))]
x = [uctbx.d_as_d_star_sq(d) for d in x]
y = [getattr(bins[i], k) for i in range(len(bins))]
pyplot.plot(x, y, label=label)
pyplot.xlabel('d spacing')
pyplot.ylabel(k)
if k == 'cc_one_half':
pyplot.ylim(0, 1.05)
else:
pyplot.ylim(0, pyplot.ylim()[1])
ax = pyplot.gca()
xticks = ax.get_xticks()
xticks_d = [
'%.2f' %uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks]
ax.set_xticklabels(xticks_d)
if size_inches is not None:
fig = pyplot.gcf()
fig.set_size_inches(size_inches)
if labels is not None:
pyplot.legend(loc='best')
pyplot.tight_layout()
pyplot.savefig(os.path.join(image_dir, prefix+ k + '.png'))
pyplot.clf()
def exercise_functions():
d_star_sq_s = 1.2345
d_star_sq_a = flex.double((1.2345, 0.1234))
two_theta_s = 0.61725
two_theta_a = flex.double((0.61725, 0.65432))
# forward conversions
assert approx_equal(uctbx.d_star_sq_as_stol_sq(d_star_sq_s),
d_star_sq_s / 4)
assert approx_equal(uctbx.d_star_sq_as_stol_sq(d_star_sq_a),
[uctbx.d_star_sq_as_stol_sq(i) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_two_stol(d_star_sq_s)**2, d_star_sq_s)
assert approx_equal(uctbx.d_star_sq_as_two_stol(d_star_sq_a),
[uctbx.d_star_sq_as_two_stol(i) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_stol(d_star_sq_s)**2, d_star_sq_s / 4)
assert approx_equal(uctbx.d_star_sq_as_stol(d_star_sq_a),
[uctbx.d_star_sq_as_stol(i) for i in d_star_sq_a])
assert approx_equal(1 / (uctbx.d_star_sq_as_d(d_star_sq_s)**2),
d_star_sq_s)
assert approx_equal(uctbx.d_star_sq_as_d(d_star_sq_a),
[uctbx.d_star_sq_as_d(i) for i in d_star_sq_a])
assert approx_equal(uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
2 * asin(1.5 / 2 * sqrt(d_star_sq_s)))
assert approx_equal(uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, False))
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_a, 1.5),
[uctbx.d_star_sq_as_two_theta(i, 1.5) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5) * 180 / pi,
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, True))
# reverse conversions
for d_star_sq, two_theta in zip((d_star_sq_s, d_star_sq_a),
(two_theta_s, two_theta_a)):
assert approx_equal(
uctbx.stol_sq_as_d_star_sq(uctbx.d_star_sq_as_stol_sq(d_star_sq)),
d_star_sq)
assert approx_equal(
uctbx.two_stol_as_d_star_sq(
uctbx.d_star_sq_as_two_stol(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.stol_as_d_star_sq(uctbx.d_star_sq_as_stol(d_star_sq)),
d_star_sq)
assert approx_equal(
uctbx.d_as_d_star_sq(uctbx.d_star_sq_as_d(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.two_theta_as_d_star_sq(
uctbx.d_star_sq_as_two_theta(d_star_sq, 1.5), 1.5), d_star_sq)
assert approx_equal(
uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
uctbx.two_theta_as_d_star_sq(two_theta, 1.5, False))
assert approx_equal(
uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
uctbx.two_theta_as_d_star_sq(two_theta * 180 / pi, 1.5, True))
assert approx_equal(
uctbx.two_theta_as_d(two_theta, 1.5),
uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5)))
assert approx_equal(
uctbx.two_theta_as_d(two_theta, 1.5, True),
uctbx.d_star_sq_as_d(
uctbx.two_theta_as_d_star_sq(two_theta, 1.5, True)))
#
assert uctbx.fractional_unit_shifts(distance_frac=[0, 0, 0]) == (0, 0, 0)
assert uctbx.fractional_unit_shifts([0.6, 7.4, -0.4]) == (1, 7, 0)
assert uctbx.fractional_unit_shifts([-6, 3, -0.6]) == (-6, 3, -1)
site_frac_1 = [0.3, -8.6, 2.1]
for eps, expected_u2 in [(-1.e-5, 1), (1.e-5, 2)]:
site_frac_2 = [-3, 5.6, 0.6 - eps]
assert uctbx.fractional_unit_shifts(
site_frac_1=site_frac_1,
site_frac_2=site_frac_2) == (3, -14, expected_u2)
def run(args):
from dials.util.options import OptionParser
import libtbx.load_env
usage = "%s [options] find_spots.json" % (libtbx.env.dispatcher_name)
parser = OptionParser(usage=usage, phil=phil_scope, epilog=help_message)
params, options, args = parser.parse_args(show_diff_phil=True,
return_unhandled=True)
positions = None
if params.positions is not None:
with open(params.positions, "rb") as f:
positions = flex.vec2_double()
for line in f.readlines():
line = (line.replace("(", " ").replace(")", "").replace(
",", " ").strip().split())
assert len(line) == 3
i, x, y = [float(l) for l in line]
positions.append((x, y))
assert len(args) == 1
json_file = args[0]
import json
with open(json_file, "rb") as f:
results = json.load(f)
n_indexed = flex.double()
fraction_indexed = flex.double()
n_spots = flex.double()
n_lattices = flex.double()
crystals = []
image_names = flex.std_string()
for r in results:
n_spots.append(r["n_spots_total"])
image_names.append(str(r["image"]))
if "n_indexed" in r:
n_indexed.append(r["n_indexed"])
fraction_indexed.append(r["fraction_indexed"])
n_lattices.append(len(r["lattices"]))
for d in r["lattices"]:
from dxtbx.serialize.crystal import from_dict
crystals.append(from_dict(d["crystal"]))
else:
n_indexed.append(0)
fraction_indexed.append(0)
n_lattices.append(0)
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot
blue = "#3498db"
red = "#e74c3c"
marker = "o"
alpha = 0.5
lw = 0
plot = True
table = True
grid = params.grid
from libtbx import group_args
from dials.algorithms.peak_finding.per_image_analysis import plot_stats, print_table
estimated_d_min = flex.double()
d_min_distl_method_1 = flex.double()
d_min_distl_method_2 = flex.double()
n_spots_total = flex.int()
n_spots_no_ice = flex.int()
total_intensity = flex.double()
for d in results:
estimated_d_min.append(d["estimated_d_min"])
d_min_distl_method_1.append(d["d_min_distl_method_1"])
d_min_distl_method_2.append(d["d_min_distl_method_2"])
n_spots_total.append(d["n_spots_total"])
n_spots_no_ice.append(d["n_spots_no_ice"])
total_intensity.append(d["total_intensity"])
stats = group_args(
image=image_names,
n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=None,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_1=None,
noisiness_method_2=None,
)
if plot:
plot_stats(stats)
pyplot.clf()
if table:
print_table(stats)
print("Number of indexed lattices: ", (n_indexed > 0).count(True))
print(
"Number with valid d_min but failed indexing: ",
((d_min_distl_method_1 > 0)
& (d_min_distl_method_2 > 0)
& (estimated_d_min > 0)
& (n_indexed == 0)).count(True),
)
n_rows = 10
n_rows = min(n_rows, len(n_spots_total))
perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True)
print("Top %i images sorted by number of spots:" % n_rows)
print_table(stats, perm=perm_n_spots_total, n_rows=n_rows)
n_bins = 20
spot_count_histogram(n_spots_total,
n_bins=n_bins,
filename="hist_n_spots_total.png",
log=True)
spot_count_histogram(n_spots_no_ice,
n_bins=n_bins,
filename="hist_n_spots_no_ice.png",
log=True)
spot_count_histogram(
n_indexed.select(n_indexed > 0),
n_bins=n_bins,
filename="hist_n_indexed.png",
log=False,
)
if len(crystals):
plot_unit_cell_histograms(crystals)
if params.stereographic_projections and len(crystals):
from dxtbx.datablock import DataBlockFactory
datablocks = DataBlockFactory.from_filenames([image_names[0]],
verbose=False)
assert len(datablocks) == 1
imageset = datablocks[0].extract_imagesets()[0]
s0 = imageset.get_beam().get_s0()
# XXX what if no goniometer?
rotation_axis = imageset.get_goniometer().get_rotation_axis()
indices = ((1, 0, 0), (0, 1, 0), (0, 0, 1))
for i, index in enumerate(indices):
from cctbx import crystal, miller
from scitbx import matrix
miller_indices = flex.miller_index([index])
symmetry = crystal.symmetry(
unit_cell=crystals[0].get_unit_cell(),
space_group=crystals[0].get_space_group(),
)
miller_set = miller.set(symmetry, miller_indices)
d_spacings = miller_set.d_spacings()
d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
d_spacings = d_spacings.generate_bijvoet_mates()
miller_indices = d_spacings.indices()
# plane normal
d0 = matrix.col(s0).normalize()
d1 = d0.cross(matrix.col(rotation_axis)).normalize()
d2 = d1.cross(d0).normalize()
reference_poles = (d0, d1, d2)
from dials.command_line.stereographic_projection import (
stereographic_projection, )
projections = []
for cryst in crystals:
reciprocal_space_points = (
list(cryst.get_U() * cryst.get_B()) *
miller_indices.as_vec3_double())
projections.append(
stereographic_projection(reciprocal_space_points,
reference_poles))
# from dials.algorithms.indexing.compare_orientation_matrices import \
# difference_rotation_matrix_and_euler_angles
# R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles(
# crystals[0], cryst)
# print max(euler_angles)
from dials.command_line.stereographic_projection import plot_projections
plot_projections(projections,
filename="projections_%s.png" % ("hkl"[i]))
pyplot.clf()
def plot_grid(
values,
grid,
file_name,
cmap=pyplot.cm.Reds,
vmin=None,
vmax=None,
invalid="white",
):
values = values.as_double()
# At DLS, fast direction appears to be largest direction
if grid[0] > grid[1]:
values.reshape(flex.grid(reversed(grid)))
values = values.matrix_transpose()
else:
values.reshape(flex.grid(grid))
Z = values.as_numpy_array()
# f, (ax1, ax2) = pyplot.subplots(2)
f, ax1 = pyplot.subplots(1)
mesh1 = ax1.pcolormesh(values.as_numpy_array(),
cmap=cmap,
vmin=vmin,
vmax=vmax)
mesh1.cmap.set_under(color=invalid, alpha=None)
mesh1.cmap.set_over(color=invalid, alpha=None)
# mesh2 = ax2.contour(Z, cmap=cmap, vmin=vmin, vmax=vmax)
# mesh2 = ax2.contourf(Z, cmap=cmap, vmin=vmin, vmax=vmax)
ax1.set_aspect("equal")
ax1.invert_yaxis()
# ax2.set_aspect('equal')
# ax2.invert_yaxis()
pyplot.colorbar(mesh1, ax=ax1)
# pyplot.colorbar(mesh2, ax=ax2)
pyplot.savefig(file_name, dpi=600)
pyplot.clf()
def plot_positions(
values,
positions,
file_name,
cmap=pyplot.cm.Reds,
vmin=None,
vmax=None,
invalid="white",
):
values = values.as_double()
assert positions.size() >= values.size()
positions = positions[:values.size()]
if vmin is None:
vmin = flex.min(values)
if vmax is None:
vmax = flex.max(values)
x, y = positions.parts()
dx = flex.abs(x[1:] - x[:-1])
dy = flex.abs(y[1:] - y[:-1])
dx = dx.select(dx > 0)
dy = dy.select(dy > 0)
scale = 1 / flex.min(dx)
# print scale
x = (x * scale).iround()
y = (y * scale).iround()
from libtbx.math_utils import iceil
z = flex.double(
flex.grid(iceil(flex.max(y)) + 1,
iceil(flex.max(x)) + 1), -2)
# print z.all()
for x_, y_, z_ in zip(x, y, values):
z[y_, x_] = z_
plot_grid(
z.as_1d(),
z.all(),
file_name,
cmap=cmap,
vmin=vmin,
vmax=vmax,
invalid=invalid,
)
return
if grid is not None or positions is not None:
if grid is not None:
positions = tuple(reversed(grid))
plotter = plot_grid
else:
plotter = plot_positions
cmap = pyplot.get_cmap(params.cmap)
plotter(
n_spots_total,
positions,
"grid_spot_count_total.png",
cmap=cmap,
invalid=params.invalid,
)
plotter(
n_spots_no_ice,
positions,
"grid_spot_count_no_ice.png",
cmap=cmap,
invalid=params.invalid,
)
plotter(
total_intensity,
positions,
"grid_total_intensity.png",
cmap=cmap,
invalid=params.invalid,
)
if flex.max(n_indexed) > 0:
plotter(
n_indexed,
positions,
"grid_n_indexed.png",
cmap=cmap,
invalid=params.invalid,
)
plotter(
fraction_indexed,
positions,
"grid_fraction_indexed.png",
cmap=cmap,
vmin=0,
vmax=1,
invalid=params.invalid,
)
for i, d_min in enumerate(
(estimated_d_min, d_min_distl_method_1, d_min_distl_method_2)):
from cctbx import uctbx
d_star_sq = uctbx.d_as_d_star_sq(d_min)
d_star_sq.set_selected(d_star_sq == 1, 0)
vmin = flex.min(d_star_sq.select(d_star_sq > 0))
vmax = flex.max(d_star_sq)
vmin = flex.min(d_min.select(d_min > 0))
vmax = flex.max(d_min)
cmap = pyplot.get_cmap("%s_r" % params.cmap)
d_min.set_selected(d_min <= 0, vmax)
if i == 0:
plotter(
d_min,
positions,
"grid_d_min.png",
cmap=cmap,
vmin=vmin,
vmax=vmax,
invalid=params.invalid,
)
else:
plotter(
d_min,
positions,
"grid_d_min_method_%i.png" % i,
cmap=cmap,
vmin=vmin,
vmax=vmax,
invalid=params.invalid,
)
if flex.max(n_indexed) > 0:
pyplot.hexbin(n_spots,
n_indexed,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
# pyplot.scatter(n_spots, n_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
xlim = pyplot.xlim()
ylim = pyplot.ylim()
pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel("# spots")
pyplot.ylabel("# indexed")
pyplot.savefig("n_spots_vs_n_indexed.png")
pyplot.clf()
pyplot.hexbin(n_spots,
fraction_indexed,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
# pyplot.scatter(
# n_spots, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel("# spots")
pyplot.ylabel("Fraction indexed")
pyplot.savefig("n_spots_vs_fraction_indexed.png")
pyplot.clf()
pyplot.hexbin(n_indexed,
fraction_indexed,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
# pyplot.scatter(
# n_indexed, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel("# indexed")
pyplot.ylabel("Fraction indexed")
pyplot.savefig("n_indexed_vs_fraction_indexed.png")
pyplot.clf()
pyplot.hexbin(n_spots,
n_lattices,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
# pyplot.scatter(
# n_spots, n_lattices, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel("# spots")
pyplot.ylabel("# lattices")
pyplot.savefig("n_spots_vs_n_lattices.png")
pyplot.clf()
# pyplot.scatter(
# estimated_d_min, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(
estimated_d_min,
d_min_distl_method_1,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50,
)
pyplot.colorbar()
# pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(estimated_d_min), max(d_min_distl_method_1))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel("estimated_d_min")
pyplot.ylabel("d_min_distl_method_1")
pyplot.savefig("d_min_vs_distl_method_1.png")
pyplot.clf()
# pyplot.scatter(
# estimated_d_min, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(
estimated_d_min,
d_min_distl_method_2,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50,
)
pyplot.colorbar()
# pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(estimated_d_min), max(d_min_distl_method_2))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel("estimated_d_min")
pyplot.ylabel("d_min_distl_method_2")
pyplot.savefig("d_min_vs_distl_method_2.png")
pyplot.clf()
# pyplot.scatter(
# d_min_distl_method_1, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(
d_min_distl_method_1,
d_min_distl_method_2,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50,
)
pyplot.colorbar()
# pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(d_min_distl_method_1), max(d_min_distl_method_2))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel("d_min_distl_method_1")
pyplot.ylabel("d_min_distl_method_2")
pyplot.savefig("distl_method_1_vs_distl_method_2.png")
pyplot.clf()
pyplot.hexbin(n_spots,
estimated_d_min,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
# pyplot.scatter(
# n_spots, estimated_d_min, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel("# spots")
pyplot.ylabel("estimated_d_min")
pyplot.savefig("n_spots_vs_d_min.png")
pyplot.clf()
pyplot.hexbin(n_spots,
d_min_distl_method_1,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
# pyplot.scatter(
# n_spots, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel("# spots")
pyplot.ylabel("d_min_distl_method_1")
pyplot.savefig("n_spots_vs_distl_method_1.png")
pyplot.clf()
pyplot.hexbin(n_spots,
d_min_distl_method_2,
bins="log",
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
# pyplot.scatter(
# n_spots, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel("# spots")
pyplot.ylabel("d_min_distl_method_2")
pyplot.savefig("n_spots_vs_distl_method_2.png")
pyplot.clf()
def test_ResolutionPlotsAndStats(iobs):
i_obs_anom = iobs.as_anomalous_array()
iobs_anom = i_obs_anom.map_to_asu().customized_copy(info=iobs.info())
n_bins = 2
result = dataset_statistics(
iobs, assert_is_not_unique_set_under_symmetry=False, n_bins=n_bins
)
anom_result = dataset_statistics(
iobs_anom,
assert_is_not_unique_set_under_symmetry=False,
anomalous=True,
n_bins=n_bins,
)
plotter = ResolutionPlotsAndStats(result, anom_result)
assert plotter.d_star_sq_ticktext == ["1.74", "1.53", "1.38", "1.27", "1.18"]
assert plotter.d_star_sq_tickvals == pytest.approx(
[
0.32984033277048164,
0.42706274943714834,
0.524285166103815,
0.6215075827704818,
0.7187299994371485,
],
1e-4,
)
tables = plotter.statistics_tables()
assert len(tables) == 2 # overall and per resolution
# test plots individually
d = plotter.cc_one_half_plot()
assert len(d["cc_one_half"]["data"]) == 6
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:4])
d["cc_one_half"]["data"][0]["x"] == [
0.5 * (uctbx.d_as_d_star_sq(b.d_max) + uctbx.d_as_d_star_sq(b.d_min))
for b in result.bins
]
d = plotter.i_over_sig_i_plot()
assert len(d["i_over_sig_i"]["data"]) == 1
assert len(d["i_over_sig_i"]["data"][0]["y"]) == n_bins
d = plotter.r_pim_plot()
assert len(d["r_pim"]["data"]) == 1
assert len(d["r_pim"]["data"][0]["y"]) == n_bins
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
# now try centric options and sigma tau for cc_one_half
plotter = ResolutionPlotsAndStats(result, anom_result, is_centric=True)
d = plotter.cc_one_half_plot(method="sigma_tau")
assert len(d["cc_one_half"]["data"]) == 6
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:2])
assert d["cc_one_half"]["data"][2] == {} # no anomalous plots
assert d["cc_one_half"]["data"][3] == {} # no anomalous plots
assert d["cc_one_half"]["data"][4] == {} # no cc_fit
assert d["cc_one_half"]["data"][5] == {} # no d_min
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
assert d["completeness"]["data"][1] == {}
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
assert d["multiplicity_vs_resolution"]["data"][1] == {}
plots = plotter.make_all_plots()
assert list(plots.keys()) == [
"cc_one_half",
"i_over_sig_i",
"completeness",
"multiplicity_vs_resolution",
"r_pim",
]
for plot in plots.values():
assert plot["layout"]["xaxis"]["ticktext"] == plotter.d_star_sq_ticktext
assert plot["layout"]["xaxis"]["tickvals"] == plotter.d_star_sq_tickvals
def plot_data(
results,
k,
ylabel,
labels,
linestyle=None,
n_rows=None,
n_cols=None,
global_labels=None,
alpha=0.3,
):
from matplotlib import pyplot as plt
colors = plt.rcParams["axes.prop_cycle"].by_key()["color"]
ref_ax = None
for i, result in enumerate(results):
if not isinstance(result, (list, tuple)):
result = (result, )
if labels is not None:
label = labels[i].replace("\\$", "$")
else:
label = None
if n_cols > 1:
ax = plt.subplot(n_rows,
n_cols,
i + 1,
sharex=ref_ax,
sharey=ref_ax)
if label:
ax.set_title(label, loc="left")
if ref_ax is None:
ref_ax = ax
for other in results:
if isinstance(other,
iotbx.merging_statistics.dataset_statistics):
other = (other, )
for res in other:
if res is not None:
x = [
0.5 * (uctbx.d_as_d_star_sq(b.d_max) +
uctbx.d_as_d_star_sq(b.d_min))
for b in res.bins
]
y = [getattr(b, k) for b in res.bins]
ax.plot(
x,
y,
linestyle="-",
color="grey",
linewidth=1,
alpha=alpha,
)
else:
ax = plt.gca()
for i_res, res in enumerate(result):
if res is not None:
if global_labels is not None:
l = global_labels[i_res]
else:
l = label
x = [
0.5 * (uctbx.d_as_d_star_sq(b.d_max) +
uctbx.d_as_d_star_sq(b.d_min)) for b in res.bins
]
y = [getattr(b, k) for b in res.bins]
color = colors[i_res] if n_cols > 1 else colors[i]
ax.plot(x, y, label=l, linestyle=linestyle, color=color)
if n_cols > 1 or i == len(results) - 1:
ax.set_xlabel(r"Resolution ($\AA$)")
ax.set_ylabel(ylabel)
ax.label_outer()
if k in ("cc_one_half", "cc_one_half_sigma_tau", "completeness"):
ax.set_ylim(0, 1.05)
elif k in ("cc_anom", ):
ax.set_ylim(min(0, ax.get_ylim()[0]), 1.05)
elif k in ("r_merge", ):
ax.set_ylim(0, min(4, ax.get_ylim()[1]))
elif k in ("r_meas", "r_pim"):
ax.set_ylim(0, min(2, ax.get_ylim()[1]))
else:
ax.set_ylim(0)
xticks = ax.get_xticks()
xticks_d = [
"%.2f" % uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0
for ds2 in xticks
]
ax.set_xticklabels(xticks_d)
def estimate_resolution_limit(reflections, ice_sel=None, plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections["rlp"].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections["intensity.sum.value"]
variances = reflections["intensity.sum.variance"]
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities / flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(d_star_sq.select(~ice_sel),
log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(d_star_sq,
target_n_per_bin=20,
max_slots=20,
min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1 - low_percentile_limit
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
if sel.count(True) == 0:
continue
outliers = wilson_outliers(reflections.select(sel_all),
ice_sel=ice_sel.select(sel_all))
outliers_all.set_selected(sel_all, outliers)
isel = sel_all.iselection().select(~(outliers)
& ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
if m_upper == m_lower:
intersection = (-1, -1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
# with:
# a_ = m_upper
# b_ = m_lower
# c_ = c_upper
# d_ = c_lower
# intersection == ((d_ - c_) / (a_ - b_), (a_ * d_ - b_ * c_) / (a_ - b_))
intersection = (
(c_lower - c_upper) / (m_upper - m_lower),
(m_upper * c_lower - m_lower * c_upper) / (m_upper - m_lower),
)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
c_upper)
inside = inside & ~outliers_all & ~ice_sel
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
if plot_filename is not None:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(d_star_sq, log_i_over_sigi, marker="+")
ax.scatter(
d_star_sq.select(inside),
log_i_over_sigi.select(inside),
marker="+",
color="green",
)
ax.scatter(
d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker="+",
color="black",
)
ax.scatter(
d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker="+",
color="grey",
)
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker="+", color="red")
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker="+", color="red")
if intersection[0] <= ax.get_xlim(
)[1] and intersection[1] <= ax.get_ylim()[1]:
ax.scatter([intersection[0]], [intersection[1]],
marker="x",
s=50,
color="b")
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color="red")
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color="red")
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(
uctbx.d_as_d_star_sq(slot.d_max),
0,
ax.get_ylim()[1],
linestyle="dotted",
color="grey",
)
ax.vlines(
uctbx.d_as_d_star_sq(slot.d_min),
0,
ax.get_ylim()[1],
linestyle="dotted",
color="grey",
)
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def estimate_resolution_limit(reflections,
imageset,
ice_sel=None,
plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities / flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(d_star_sq.select(~ice_sel),
log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(d_star_sq,
target_n_per_bin=20,
max_slots=20,
min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1 - low_percentile_limit
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
#sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#print "%.2f" %(sel.count(True)/sel_all.count(True))
if sel.count(True) == 0:
#outliers_all.set_selected(sel_all & ice_sel, True)
continue
#if i_slot > i_slot_max:
#break
#else:
#continue
outliers = wilson_outliers(reflections.select(sel_all),
ice_sel=ice_sel.select(sel_all))
#print "rejecting %d wilson outliers" %outliers.count(True)
outliers_all.set_selected(sel_all, outliers)
#if sel.count(True)/sel_all.count(True) < 0.25:
#outliers_all.set_selected(sel_all & ice_sel, True)
#from scitbx.math import median_statistics
#intensities_sel = intensities.select(sel)
#stats = median_statistics(intensities_sel)
#z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
#outliers = z_score > 3.5
#perm = flex.sort_permutation(intensities_sel)
##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
##print ' '.join('%.2f' %v for v in z_score.select(perm))
##print
isel = sel_all.iselection().select(~(outliers)
& ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
#fit_upper.show_summary()
#fit_lower.show_summary()
if m_upper == m_lower:
intersection = (-1, -1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
intersection = ((c_lower - c_upper) / (m_upper - m_lower),
(m_upper * c_lower - m_lower * c_upper) /
(m_upper - m_lower))
a = m_upper
c_ = c_upper
b = m_lower
d = c_lower
assert intersection == ((d - c_) / (a - b), (a * d - b * c_) / (a - b))
#inside = points_inside_envelope(
#d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper,
c_upper)
inside = inside & ~outliers_all
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
#d_star_sq_estimate = intersection[0]
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
#resolution_estimate = max(resolution_estimate, flex.min(d_spacings))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
ax.scatter(d_star_sq.select(inside),
log_i_over_sigi.select(inside),
marker='+',
color='green')
ax.scatter(d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker='+',
color='black')
ax.scatter(d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker='+',
color='grey')
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')
if (intersection[0] <= ax.get_xlim()[1]
and intersection[1] <= ax.get_ylim()[1]):
ax.scatter([intersection[0]], [intersection[1]],
marker='x',
s=50,
color='b')
#ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
ax.set_xlabel('d_star_sq')
ax.set_ylabel('ln(I/sigI)')
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(uctbx.d_as_d_star_sq(slot.d_max),
0,
ax.get_ylim()[1],
linestyle='dotted',
color='grey')
ax.vlines(uctbx.d_as_d_star_sq(slot.d_min),
0,
ax.get_ylim()[1],
linestyle='dotted',
color='grey')
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def plot_result(metric, result):
if metric == metrics.CC_HALF:
return plots.cc_half_plot(
result.d_star_sq,
result.y_obs,
cc_half_critical_values=result.critical_values,
cc_half_fit=result.y_fit,
d_min=result.d_min,
)
else:
d = {
metrics.MISIGMA: "Merged <I/σ(I)>",
metrics.ISIGMA: "Unmerged <I/σ(I)>",
metrics.I_MEAN_OVER_SIGMA_MEAN: "<I>/<σ(I)>",
metrics.RMERGE: "R<sub>merge</sub> ",
metrics.COMPLETENESS: "Completeness",
}
d_star_sq_tickvals, d_star_sq_ticktext = plots.d_star_sq_to_d_ticks(
result.d_star_sq, 5)
return {
"data": [
{
"x": list(result.d_star_sq), # d_star_sq
"y": list(result.y_obs),
"type": "scatter",
"name": "y_obs",
},
({
"x": list(result.d_star_sq),
"y": list(result.y_fit),
"type": "scatter",
"name": "y_fit",
"line": {
"color": "rgb(47, 79, 79)"
},
} if result.y_fit else {}),
({
"x": [uctbx.d_as_d_star_sq(result.d_min)] * 2,
"y": [
0,
max(
1,
flex.max(result.y_obs),
flex.max(result.y_fit) if result.y_fit else 0,
),
],
"type":
"scatter",
"name":
"d_min = %.2f Å" % result.d_min,
"mode":
"lines",
"line": {
"color": "rgb(169, 169, 169)",
"dash": "dot"
},
} if result.d_min else {}),
],
"layout": {
"title": f"{d.get(metric)} vs. resolution",
"xaxis": {
"title": "Resolution (Å)",
"tickvals": d_star_sq_tickvals,
"ticktext": d_star_sq_ticktext,
},
"yaxis": {
"title": d.get(metric),
"rangemode": "tozero"
},
},
}
def _record_individual_report(self, data_manager, report, cluster_name):
d = self._report_as_dict(report)
self._individual_report_dicts[
cluster_name] = self._individual_report_dict(d, cluster_name)
self._comparison_graphs.setdefault(
"radar",
{
"data": [],
"layout": {
"polar": {
"radialaxis": {
"visible": True,
"showticklabels": False,
"range": [0, 1],
}
}
},
},
)
self._comparison_graphs["radar"]["data"].append({
"type": "scatterpolar",
"r": [],
"theta": [],
"fill": "toself",
"name": cluster_name,
})
for k, text in (
("cc_one_half", "CC½"),
("mean_redundancy", "Multiplicity"),
("completeness", "Completeness"),
("i_over_sigma_mean", "I/σ(I)"),
):
self._comparison_graphs["radar"]["data"][-1]["r"].append(
getattr(report.merging_stats.overall, k))
self._comparison_graphs["radar"]["data"][-1]["theta"].append(text)
self._comparison_graphs["radar"]["data"][-1]["r"].append(
uctbx.d_as_d_star_sq(report.merging_stats.overall.d_min))
self._comparison_graphs["radar"]["data"][-1]["theta"].append(
"Resolution")
for graph in (
"cc_one_half",
"i_over_sig_i",
"completeness",
"multiplicity_vs_resolution",
"r_pim",
):
self._comparison_graphs.setdefault(graph, {
"layout": d[graph]["layout"],
"data": []
})
data = copy.deepcopy(d[graph]["data"][0])
data["name"] = cluster_name
data.pop("line", None) # remove default color override
self._comparison_graphs[graph]["data"].append(data)
def remove_html_tags(table):
return [[
s.replace("<strong>", "").replace("</strong>", "").replace(
"<sub>", "").replace("</sub>", "")
if isinstance(s, str) else s for s in row
] for row in table]
logger.info(
"\nOverall merging statistics:\n%s",
tabulate(remove_html_tags(d["overall_statistics_table"]),
headers="firstrow"),
)
logger.info(
"\nResolution shells:\n%s",
tabulate(remove_html_tags(d["merging_statistics_table"]),
headers="firstrow"),
)
def plot_resolution_limit(self, d): from cctbx import uctbx d_star_sq = uctbx.d_as_d_star_sq(d) self.ax.plot([d_star_sq, d_star_sq], self.ax.get_ylim(), linestyle='--')
def gpeak_from_d_spacing(d, wavl):
"""take a d-spacing and return a peak in GSASII format"""
twoth = d_star_sq_as_two_theta(d_as_d_star_sq(d), wavl, deg=True)
return [twoth, 1000, True, False, 0, 0, 0, d, 0]
def make_dano_plots(anomalous_data):
"""
Make dicts of data for plotting e.g. for plotly.
Args:
anomalous_data (dict) : A dict of (wavelength, anomalous array) data.
Returns:
dict: A dictionary containing the plotting data.
"""
data = {
"dF": {
"dano": {
"data": [],
"help": """\
This plot shows the size of the anomalous differences of F relative to the uncertainties,
(<|F(+)-F(-)|/σ(F(+)-F(-))>). A value of 0.8 is indicative of pure noise, and
a suggested cutoff is when the value falls below 1.2, although these guides require
reliable sigma estimates. For further information see
https://strucbio.biologie.uni-konstanz.de/ccp4wiki/index.php?title=SHELX_C/D/E
""",
},
},
}
for i, (wave, anom) in enumerate(anomalous_data.items()):
dFsdF, resolution_bin_edges = dano_over_sigdano_stats(anom)
d_star_sq_bins = [
0.5
* (
uctbx.d_as_d_star_sq(resolution_bin_edges[i])
+ uctbx.d_as_d_star_sq(resolution_bin_edges[i + 1])
)
for i in range(0, len(resolution_bin_edges[:-1]))
]
d_star_sq_tickvals, d_star_sq_ticktext = d_star_sq_to_d_ticks(
d_star_sq_bins, nticks=5
)
data["dF"]["dano"]["data"].append(
{
"x": d_star_sq_bins,
"y": list(dFsdF),
"type": "scatter",
"name": "\u03BB" + f"={wave:.4f}",
}
)
data["dF"]["dano"]["data"].append(
{
"x": d_star_sq_bins,
"y": [0.8] * len(d_star_sq_bins),
"type": "scatter",
"mode": "lines",
"name": "random noise level",
}
)
data["dF"]["dano"]["data"].append(
{
"x": d_star_sq_bins,
"y": [1.2] * len(d_star_sq_bins),
"type": "scatter",
"mode": "lines",
"name": "an approximate <br>threshold for a<br>resolution cutoff",
}
)
data["dF"]["dano"]["layout"] = {
"title": "<|ΔF|/σ(ΔF)> vs resolution",
"xaxis": {
"title": "Resolution (Å)",
"tickvals": d_star_sq_tickvals,
"ticktext": d_star_sq_ticktext,
},
"yaxis": {"title": "<|ΔF|/σ(ΔF)>", "rangemode": "tozero"},
}
return data
def estimate_resolution_limit(reflections, imageset, ice_sel=None,
plot_filename=None):
if ice_sel is None:
ice_sel = flex.bool(len(reflections), False)
d_star_sq = flex.pow2(reflections['rlp'].norms())
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
intensities = reflections['intensity.sum.value']
variances = reflections['intensity.sum.variance']
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
ice_sel = ice_sel.select(sel)
i_over_sigi = intensities/flex.sqrt(variances)
log_i_over_sigi = flex.log(i_over_sigi)
fit = flex.linear_regression(
d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel))
m = fit.slope()
c = fit.y_intercept()
log_i_sigi_lower = flex.double()
d_star_sq_lower = flex.double()
log_i_sigi_upper = flex.double()
d_star_sq_upper = flex.double()
binner = binner_equal_population(
d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5)
outliers_all = flex.bool(len(reflections), False)
low_percentile_limit = 0.1
upper_percentile_limit = 1-low_percentile_limit
d_spacings = uctbx.d_star_sq_as_d(d_star_sq)
for i_slot, slot in enumerate(binner.bins):
sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
sel = ~(ice_sel) & sel_all
#sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min)
#print "%.2f" %(sel.count(True)/sel_all.count(True))
if sel.count(True) == 0:
#outliers_all.set_selected(sel_all & ice_sel, True)
continue
#if i_slot > i_slot_max:
#break
#else:
#continue
outliers = wilson_outliers(
reflections.select(sel_all), ice_sel=ice_sel.select(sel_all))
#print "rejecting %d wilson outliers" %outliers.count(True)
outliers_all.set_selected(sel_all, outliers)
#if sel.count(True)/sel_all.count(True) < 0.25:
#outliers_all.set_selected(sel_all & ice_sel, True)
#from scitbx.math import median_statistics
#intensities_sel = intensities.select(sel)
#stats = median_statistics(intensities_sel)
#z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation
#outliers = z_score > 3.5
#perm = flex.sort_permutation(intensities_sel)
##print ' '.join('%.2f' %v for v in intensities_sel.select(perm))
##print ' '.join('%.2f' %v for v in z_score.select(perm))
##print
isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all))
log_i_over_sigi_sel = log_i_over_sigi.select(isel)
d_star_sq_sel = d_star_sq.select(isel)
perm = flex.sort_permutation(log_i_over_sigi_sel)
i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))]
i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))]
log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower])
log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper])
d_star_sq_upper.append(d_star_sq_sel[i_lower])
d_star_sq_lower.append(d_star_sq_sel[i_upper])
fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper)
m_upper = fit_upper.slope()
c_upper = fit_upper.y_intercept()
fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower)
m_lower = fit_lower.slope()
c_lower = fit_lower.y_intercept()
#fit_upper.show_summary()
#fit_lower.show_summary()
if m_upper == m_lower:
intersection = (-1,-1)
resolution_estimate = -1
inside = flex.bool(len(d_star_sq), False)
else:
# http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines
intersection = (
(c_lower-c_upper)/(m_upper-m_lower),
(m_upper*c_lower-m_lower*c_upper)/(m_upper-m_lower))
a = m_upper
c_ = c_upper
b = m_lower
d = c_lower
assert intersection == ((d-c_)/(a-b), (a*d-b*c_)/(a-b))
#inside = points_inside_envelope(
#d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower)
inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper)
inside = inside & ~outliers_all
if inside.count(True) > 0:
d_star_sq_estimate = flex.max(d_star_sq.select(inside))
#d_star_sq_estimate = intersection[0]
resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate)
else:
resolution_estimate = -1
#resolution_estimate = max(resolution_estimate, flex.min(d_spacings))
if plot_filename is not None:
if pyplot is None:
raise Sorry("matplotlib must be installed to generate a plot.")
fig = pyplot.figure()
ax = fig.add_subplot(1,1,1)
ax.scatter(d_star_sq, log_i_over_sigi, marker='+')
ax.scatter(d_star_sq.select(inside), log_i_over_sigi.select(inside),
marker='+', color='green')
ax.scatter(d_star_sq.select(ice_sel),
log_i_over_sigi.select(ice_sel),
marker='+', color='black')
ax.scatter(d_star_sq.select(outliers_all),
log_i_over_sigi.select(outliers_all),
marker='+', color='grey')
ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red')
ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red')
if (intersection[0] <= ax.get_xlim()[1] and
intersection[1] <= ax.get_ylim()[1]):
ax.scatter([intersection[0]], [intersection[1]], marker='x', s=50, color='b')
#ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30)
xlim = pyplot.xlim()
ax.plot(xlim, [(m * x + c) for x in xlim])
ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red')
ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red')
ax.set_xlabel('d_star_sq')
ax.set_ylabel('ln(I/sigI)')
ax.set_xlim((max(-xlim[1], -0.05), xlim[1]))
ax.set_ylim((0, ax.get_ylim()[1]))
for i_slot, slot in enumerate(binner.bins):
if i_slot == 0:
ax.vlines(uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax.vlines(uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1],
linestyle='dotted', color='grey')
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ]
xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" %d for d in xticks_d])
#pyplot.show()
pyplot.savefig(plot_filename)
pyplot.close()
return resolution_estimate
def cc_half_plot(
d_star_sq,
cc_half,
cc_anom=None,
cc_half_critical_values=None,
cc_anom_critical_values=None,
cc_half_fit=None,
d_min=None,
):
d_star_sq_tickvals, d_star_sq_ticktext = d_star_sq_to_d_ticks(d_star_sq, nticks=5)
return {
"data": [
{
"x": list(d_star_sq),
"y": list(cc_half),
"type": "scatter",
"name": "CC<sub>½</sub>",
"mode": "lines",
"line": {"color": "rgb(31, 119, 180)"},
},
(
{
"x": list(d_star_sq),
"y": list(cc_half_critical_values),
"type": "scatter",
"name": "CC<sub>½</sub> critical value (p=0.01)",
"line": {"color": "rgb(31, 119, 180)", "dash": "dot"},
}
if cc_half_critical_values
else {}
),
(
{
"x": list(d_star_sq),
"y": list(cc_anom),
"type": "scatter",
"name": "CC-anom",
"mode": "lines",
"line": {"color": "rgb(255, 127, 14)"},
}
if cc_anom
else {}
),
(
{
"x": list(d_star_sq),
"y": list(cc_anom_critical_values),
"type": "scatter",
"name": "CC-anom critical value (p=0.01)",
"mode": "lines",
"line": {"color": "rgb(255, 127, 14)", "dash": "dot"},
}
if cc_anom_critical_values
else {}
),
(
{
"x": list(d_star_sq),
"y": list(cc_half_fit),
"type": "scatter",
"name": "CC<sub>½</sub> fit",
"line": {"color": "rgb(47, 79, 79)"},
}
if cc_half_fit
else {}
),
(
{
"x": [uctbx.d_as_d_star_sq(d_min)] * 2,
"y": [0, 1],
"type": "scatter",
"name": f"d_min = {d_min:.2f} Å",
"mode": "lines",
"line": {"color": "rgb(169, 169, 169)", "dash": "dot"},
}
if d_min
else {}
),
],
"layout": {
"title": "CC<sub>½</sub> vs resolution",
"xaxis": {
"title": "Resolution (Å)",
"tickvals": d_star_sq_tickvals,
"ticktext": d_star_sq_ticktext,
},
"yaxis": {
"title": "CC<sub>½</sub>",
"range": [min(cc_half + cc_anom if cc_anom else [] + [0]), 1],
},
},
"help": """\
The correlation coefficients, CC<sub>½</sub>, between random half-datasets. A correlation
coefficient of +1 indicates good correlation, and 0 indicates no correlation.
CC<sub>½</sub> is typically close to 1 at low resolution, falling off to close to zero at
higher resolution. A typical resolution cutoff based on CC<sub>½</sub> is around 0.3-0.5.
[1] Karplus, P. A., & Diederichs, K. (2012). Science, 336(6084), 1030-1033.
https://doi.org/10.1126/science.1218231
[2] Diederichs, K., & Karplus, P. A. (2013). Acta Cryst D, 69(7), 1215-1222.
https://doi.org/10.1107/S0907444913001121
[3] Evans, P. R., & Murshudov, G. N. (2013). Acta Cryst D, 69(7), 1204-1214.
https://doi.org/10.1107/S0907444913000061
""",
}
def gpeak_from_d_spacing(d, wavl):
from cctbx.uctbx import d_as_d_star_sq, d_star_sq_as_two_theta
twoth = d_star_sq_as_two_theta(d_as_d_star_sq(d), wavl, deg=True)
return [twoth, 1000, True, False, 0, 0, 0, d, 0]
def exercise_functions():
d_star_sq_s = 1.2345
d_star_sq_a = flex.double((1.2345, 0.1234))
two_theta_s = 0.61725
two_theta_a = flex.double((0.61725,0.65432))
# forward conversions
assert approx_equal(
uctbx.d_star_sq_as_stol_sq(d_star_sq_s), d_star_sq_s / 4)
assert approx_equal(
uctbx.d_star_sq_as_stol_sq(d_star_sq_a),
[uctbx.d_star_sq_as_stol_sq(i) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_two_stol(d_star_sq_s)**2, d_star_sq_s)
assert approx_equal(
uctbx.d_star_sq_as_two_stol(d_star_sq_a),
[uctbx.d_star_sq_as_two_stol(i) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_stol(d_star_sq_s)**2, d_star_sq_s / 4)
assert approx_equal(
uctbx.d_star_sq_as_stol(d_star_sq_a),
[uctbx.d_star_sq_as_stol(i) for i in d_star_sq_a])
assert approx_equal(
1/(uctbx.d_star_sq_as_d(d_star_sq_s)**2), d_star_sq_s)
assert approx_equal(
uctbx.d_star_sq_as_d(d_star_sq_a),
[uctbx.d_star_sq_as_d(i) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
2 * asin(1.5/2*sqrt(d_star_sq_s)))
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5),
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, False))
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_a, 1.5),
[uctbx.d_star_sq_as_two_theta(i, 1.5) for i in d_star_sq_a])
assert approx_equal(
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5)*180/pi,
uctbx.d_star_sq_as_two_theta(d_star_sq_s, 1.5, True))
# reverse conversions
for d_star_sq, two_theta in zip(
(d_star_sq_s, d_star_sq_a),(two_theta_s, two_theta_a)):
assert approx_equal(
uctbx.stol_sq_as_d_star_sq(
uctbx.d_star_sq_as_stol_sq(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.two_stol_as_d_star_sq(
uctbx.d_star_sq_as_two_stol(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.stol_as_d_star_sq(
uctbx.d_star_sq_as_stol(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.d_as_d_star_sq(
uctbx.d_star_sq_as_d(d_star_sq)), d_star_sq)
assert approx_equal(
uctbx.two_theta_as_d_star_sq(
uctbx.d_star_sq_as_two_theta(d_star_sq, 1.5), 1.5), d_star_sq)
assert approx_equal(
uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
uctbx.two_theta_as_d_star_sq(two_theta, 1.5, False))
assert approx_equal(
uctbx.two_theta_as_d_star_sq(two_theta, 1.5),
uctbx.two_theta_as_d_star_sq(two_theta*180/pi, 1.5, True))
assert approx_equal(
uctbx.two_theta_as_d(two_theta, 1.5),
uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5)))
assert approx_equal(
uctbx.two_theta_as_d(two_theta, 1.5, True),
uctbx.d_star_sq_as_d(uctbx.two_theta_as_d_star_sq(two_theta, 1.5, True)))
#
assert uctbx.fractional_unit_shifts(distance_frac=[0,0,0]) == (0,0,0)
assert uctbx.fractional_unit_shifts([0.6,7.4,-0.4]) == (1,7,0)
assert uctbx.fractional_unit_shifts([-6,3,-0.6]) == (-6,3,-1)
site_frac_1 = [0.3,-8.6,2.1]
for eps,expected_u2 in [(-1.e-5, 1), (1.e-5, 2)]:
site_frac_2 = [-3,5.6,0.6-eps]
assert uctbx.fractional_unit_shifts(
site_frac_1=site_frac_1,
site_frac_2=site_frac_2) == (3, -14, expected_u2)
def run(args):
from dials.util.options import OptionParser
import libtbx.load_env
usage = "%s [options] find_spots.json" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
epilog=help_message)
params, options, args = parser.parse_args(
show_diff_phil=True, return_unhandled=True)
positions = None
if params.positions is not None:
with open(params.positions, 'rb') as f:
positions = flex.vec2_double()
for line in f.readlines():
line = line.replace('(', ' ').replace(')', '').replace(',', ' ').strip().split()
assert len(line) == 3
i, x, y = [float(l) for l in line]
positions.append((x, y))
assert len(args) == 1
json_file = args[0]
import json
with open(json_file, 'rb') as f:
results = json.load(f)
n_indexed = flex.double()
fraction_indexed = flex.double()
n_spots = flex.double()
n_lattices = flex.double()
crystals = []
image_names = flex.std_string()
for r in results:
n_spots.append(r['n_spots_total'])
image_names.append(str(r['image']))
if 'n_indexed' in r:
n_indexed.append(r['n_indexed'])
fraction_indexed.append(r['fraction_indexed'])
n_lattices.append(len(r['lattices']))
for d in r['lattices']:
from dxtbx.serialize.crystal import from_dict
crystals.append(from_dict(d['crystal']))
else:
n_indexed.append(0)
fraction_indexed.append(0)
n_lattices.append(0)
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
blue = '#3498db'
red = '#e74c3c'
marker = 'o'
alpha = 0.5
lw = 0
plot = True
table = True
grid = params.grid
from libtbx import group_args
from dials.algorithms.peak_finding.per_image_analysis \
import plot_stats, print_table
estimated_d_min = flex.double()
d_min_distl_method_1 = flex.double()
d_min_distl_method_2 = flex.double()
n_spots_total = flex.int()
n_spots_no_ice = flex.int()
total_intensity = flex.double()
for d in results:
estimated_d_min.append(d['estimated_d_min'])
d_min_distl_method_1.append(d['d_min_distl_method_1'])
d_min_distl_method_2.append(d['d_min_distl_method_2'])
n_spots_total.append(d['n_spots_total'])
n_spots_no_ice.append(d['n_spots_no_ice'])
total_intensity.append(d['total_intensity'])
stats = group_args(image=image_names,
n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=None,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_1=None,
noisiness_method_2=None)
if plot:
plot_stats(stats)
pyplot.clf()
if table:
print_table(stats)
print "Number of indexed lattices: ", (n_indexed > 0).count(True)
print "Number with valid d_min but failed indexing: ", (
(d_min_distl_method_1 > 0) &
(d_min_distl_method_2 > 0) &
(estimated_d_min > 0) &
(n_indexed == 0)).count(True)
n_rows = 10
n_rows = min(n_rows, len(n_spots_total))
perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True)
print 'Top %i images sorted by number of spots:' %n_rows
print_table(stats, perm=perm_n_spots_total, n_rows=n_rows)
n_bins = 20
spot_count_histogram(
n_spots_total, n_bins=n_bins, filename='hist_n_spots_total.png', log=True)
spot_count_histogram(
n_spots_no_ice, n_bins=n_bins, filename='hist_n_spots_no_ice.png', log=True)
spot_count_histogram(
n_indexed.select(n_indexed > 0), n_bins=n_bins, filename='hist_n_indexed.png', log=False)
if len(crystals):
plot_unit_cell_histograms(crystals)
if params.stereographic_projections and len(crystals):
from dxtbx.datablock import DataBlockFactory
datablocks = DataBlockFactory.from_filenames(
[image_names[0]], verbose=False)
assert len(datablocks) == 1
imageset = datablocks[0].extract_imagesets()[0]
s0 = imageset.get_beam().get_s0()
# XXX what if no goniometer?
rotation_axis = imageset.get_goniometer().get_rotation_axis()
indices = ((1,0,0), (0,1,0), (0,0,1))
for i, index in enumerate(indices):
from cctbx import crystal, miller
from scitbx import matrix
miller_indices = flex.miller_index([index])
symmetry = crystal.symmetry(
unit_cell=crystals[0].get_unit_cell(),
space_group=crystals[0].get_space_group())
miller_set = miller.set(symmetry, miller_indices)
d_spacings = miller_set.d_spacings()
d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
d_spacings = d_spacings.generate_bijvoet_mates()
miller_indices = d_spacings.indices()
# plane normal
d0 = matrix.col(s0).normalize()
d1 = d0.cross(matrix.col(rotation_axis)).normalize()
d2 = d1.cross(d0).normalize()
reference_poles = (d0, d1, d2)
from dials.command_line.stereographic_projection import stereographic_projection
projections = []
for cryst in crystals:
reciprocal_space_points = list(cryst.get_U() * cryst.get_B()) * miller_indices.as_vec3_double()
projections.append(stereographic_projection(
reciprocal_space_points, reference_poles))
#from dials.algorithms.indexing.compare_orientation_matrices import \
# difference_rotation_matrix_and_euler_angles
#R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles(
# crystals[0], cryst)
#print max(euler_angles)
from dials.command_line.stereographic_projection import plot_projections
plot_projections(projections, filename='projections_%s.png' %('hkl'[i]))
pyplot.clf()
def plot_grid(values, grid, file_name, cmap=pyplot.cm.Reds,
vmin=None, vmax=None, invalid='white'):
values = values.as_double()
# At DLS, fast direction appears to be largest direction
if grid[0] > grid[1]:
values.reshape(flex.grid(reversed(grid)))
values = values.matrix_transpose()
else:
values.reshape(flex.grid(grid))
Z = values.as_numpy_array()
#f, (ax1, ax2) = pyplot.subplots(2)
f, ax1 = pyplot.subplots(1)
mesh1 = ax1.pcolormesh(
values.as_numpy_array(), cmap=cmap, vmin=vmin, vmax=vmax)
mesh1.cmap.set_under(color=invalid, alpha=None)
mesh1.cmap.set_over(color=invalid, alpha=None)
#mesh2 = ax2.contour(Z, cmap=cmap, vmin=vmin, vmax=vmax)
#mesh2 = ax2.contourf(Z, cmap=cmap, vmin=vmin, vmax=vmax)
ax1.set_aspect('equal')
ax1.invert_yaxis()
#ax2.set_aspect('equal')
#ax2.invert_yaxis()
pyplot.colorbar(mesh1, ax=ax1)
#pyplot.colorbar(mesh2, ax=ax2)
pyplot.savefig(file_name, dpi=600)
pyplot.clf()
def plot_positions(values, positions, file_name, cmap=pyplot.cm.Reds,
vmin=None, vmax=None, invalid='white'):
values = values.as_double()
assert positions.size() >= values.size()
positions = positions[:values.size()]
if vmin is None:
vmin = flex.min(values)
if vmax is None:
vmax = flex.max(values)
x, y = positions.parts()
dx = flex.abs(x[1:] - x[:-1])
dy = flex.abs(y[1:] - y[:-1])
dx = dx.select(dx > 0)
dy = dy.select(dy > 0)
scale = 1/flex.min(dx)
#print scale
x = (x * scale).iround()
y = (y * scale).iround()
from libtbx.math_utils import iceil
z = flex.double(flex.grid(iceil(flex.max(y))+1, iceil(flex.max(x))+1), -2)
#print z.all()
for x_, y_, z_ in zip(x, y, values):
z[y_, x_] = z_
plot_grid(z.as_1d(), z.all(), file_name, cmap=cmap, vmin=vmin, vmax=vmax,
invalid=invalid)
return
if grid is not None or positions is not None:
if grid is not None:
positions = tuple(reversed(grid))
plotter = plot_grid
else:
plotter = plot_positions
cmap = pyplot.get_cmap(params.cmap)
plotter(n_spots_total, positions, 'grid_spot_count_total.png', cmap=cmap,
invalid=params.invalid)
plotter(n_spots_no_ice, positions, 'grid_spot_count_no_ice.png', cmap=cmap,
invalid=params.invalid)
plotter(total_intensity, positions, 'grid_total_intensity.png', cmap=cmap,
invalid=params.invalid)
if flex.max(n_indexed) > 0:
plotter(n_indexed, positions, 'grid_n_indexed.png', cmap=cmap,
invalid=params.invalid)
plotter(fraction_indexed, positions, 'grid_fraction_indexed.png',
cmap=cmap, vmin=0, vmax=1, invalid=params.invalid)
for i, d_min in enumerate((estimated_d_min, d_min_distl_method_1, d_min_distl_method_2)):
from cctbx import uctbx
d_star_sq = uctbx.d_as_d_star_sq(d_min)
d_star_sq.set_selected(d_star_sq == 1, 0)
vmin = flex.min(d_star_sq.select(d_star_sq > 0))
vmax = flex.max(d_star_sq)
vmin = flex.min(d_min.select(d_min > 0))
vmax = flex.max(d_min)
cmap = pyplot.get_cmap('%s_r' %params.cmap)
d_min.set_selected(d_min <= 0, vmax)
if i == 0:
plotter(d_min, positions, 'grid_d_min.png', cmap=cmap, vmin=vmin,
vmax=vmax, invalid=params.invalid)
else:
plotter(
d_min, positions, 'grid_d_min_method_%i.png' %i, cmap=cmap,
vmin=vmin, vmax=vmax, invalid=params.invalid)
if flex.max(n_indexed) > 0:
pyplot.hexbin(
n_spots, n_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.scatter(n_spots, n_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
xlim = pyplot.xlim()
ylim = pyplot.ylim()
pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('# spots')
pyplot.ylabel('# indexed')
pyplot.savefig('n_spots_vs_n_indexed.png')
pyplot.clf()
pyplot.hexbin(
n_spots, fraction_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('Fraction indexed')
pyplot.savefig('n_spots_vs_fraction_indexed.png')
pyplot.clf()
pyplot.hexbin(
n_indexed, fraction_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_indexed, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# indexed')
pyplot.ylabel('Fraction indexed')
pyplot.savefig('n_indexed_vs_fraction_indexed.png')
pyplot.clf()
pyplot.hexbin(
n_spots, n_lattices, bins='log', cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, n_lattices, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('# lattices')
pyplot.savefig('n_spots_vs_n_lattices.png')
pyplot.clf()
#pyplot.scatter(
# estimated_d_min, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(estimated_d_min, d_min_distl_method_1, bins='log',
cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(estimated_d_min), max(d_min_distl_method_1))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('estimated_d_min')
pyplot.ylabel('d_min_distl_method_1')
pyplot.savefig('d_min_vs_distl_method_1.png')
pyplot.clf()
#pyplot.scatter(
# estimated_d_min, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(estimated_d_min, d_min_distl_method_2, bins='log',
cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(estimated_d_min), max(d_min_distl_method_2))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('estimated_d_min')
pyplot.ylabel('d_min_distl_method_2')
pyplot.savefig('d_min_vs_distl_method_2.png')
pyplot.clf()
#pyplot.scatter(
# d_min_distl_method_1, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(d_min_distl_method_1, d_min_distl_method_2, bins='log',
cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(d_min_distl_method_1), max(d_min_distl_method_2))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('d_min_distl_method_1')
pyplot.ylabel('d_min_distl_method_2')
pyplot.savefig('distl_method_1_vs_distl_method_2.png')
pyplot.clf()
pyplot.hexbin(
n_spots, estimated_d_min, bins='log', cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, estimated_d_min, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('estimated_d_min')
pyplot.savefig('n_spots_vs_d_min.png')
pyplot.clf()
pyplot.hexbin(
n_spots, d_min_distl_method_1, bins='log', cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('d_min_distl_method_1')
pyplot.savefig('n_spots_vs_distl_method_1.png')
pyplot.clf()
pyplot.hexbin(
n_spots, d_min_distl_method_2, bins='log', cmap=pyplot.cm.jet, gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('d_min_distl_method_2')
pyplot.savefig('n_spots_vs_distl_method_2.png')
pyplot.clf()