def _compute_merging_stats(self):
from iotbx import merging_statistics
self.merging_stats = merging_statistics.dataset_statistics(
self.intensities,
n_bins=self.params.resolution_bins,
cc_one_half_significance_level=self.params.
cc_half_significance_level,
eliminate_sys_absent=self.params.eliminate_sys_absent,
use_internal_variance=self.params.use_internal_variance,
assert_is_not_unique_set_under_symmetry=False)
intensities_anom = self.intensities.as_anomalous_array()
intensities_anom = intensities_anom.map_to_asu().customized_copy(
info=self.intensities.info())
self.merging_stats_anom = merging_statistics.dataset_statistics(
intensities_anom,
n_bins=self.params.resolution_bins,
anomalous=True,
cc_one_half_significance_level=self.params.
cc_half_significance_level,
eliminate_sys_absent=self.params.eliminate_sys_absent,
use_internal_variance=self.params.use_internal_variance,
assert_is_not_unique_set_under_symmetry=False)
self.d_star_sq_bins = [(1 / bin_stats.d_min**2)
for bin_stats in self.merging_stats.bins]
self.d_star_sq_tickvals, self.d_star_sq_ticktext = d_star_sq_to_d_ticks(
self.d_star_sq_bins, nticks=5)
def resolution_plots_and_stats(self):
self.merging_stats = merging_statistics.dataset_statistics(
self.intensities,
n_bins=self.params.resolution_bins,
cc_one_half_significance_level=self.params.cc_half_significance_level,
eliminate_sys_absent=self.params.eliminate_sys_absent,
use_internal_variance=self.params.use_internal_variance,
assert_is_not_unique_set_under_symmetry=False,
)
intensities_anom = self.intensities.as_anomalous_array()
intensities_anom = intensities_anom.map_to_asu().customized_copy(
info=self.intensities.info()
)
self.merging_stats_anom = merging_statistics.dataset_statistics(
intensities_anom,
n_bins=self.params.resolution_bins,
anomalous=True,
cc_one_half_significance_level=self.params.cc_half_significance_level,
eliminate_sys_absent=self.params.eliminate_sys_absent,
use_internal_variance=self.params.use_internal_variance,
assert_is_not_unique_set_under_symmetry=False,
)
is_centric = self.intensities.space_group().is_centric()
plotter = ResolutionPlotsAndStats(
self.merging_stats, self.merging_stats_anom, is_centric
)
d = OrderedDict()
d.update(plotter.make_all_plots(cc_one_half_method=self.params.cc_half_method))
overall_stats = plotter.overall_statistics_table(self.params.cc_half_method)
merging_stats = plotter.merging_statistics_table(self.params.cc_half_method)
return overall_stats, merging_stats, d
def run(files):
assert len(files) == 2
hkl1 = xds_ascii.XDS_ASCII(files[0], sys.stdout)
hkl2 = xds_ascii.XDS_ASCII(files[1], sys.stdout)
hkl1_points = numpy.column_stack((hkl1.xd, hkl1.yd, hkl1.zd))
tree1 = spatial.cKDTree(hkl1_points)
n_ovl, n_nonovl = 0, 0
novl_indices, novl_i, novl_sigma = flex.miller_index(), flex.double(), flex.double()
for i in xrange(len(hkl2.indices)):
x, y, z = hkl2.xd[i], hkl2.yd[i], hkl2.zd[i]
#if z > 180:
# continue
dists, idxs = tree1.query((x,y,z), k=3, p=1)
overlaps = []
for dist, idx in zip(dists, idxs):
idx = int(idx)
xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx]
if abs(z-zo) < 2.5 and (xo-x)**2+(yo-y)**2 < 15**2: # FIXME MAGIC NUMBER!
overlaps.append((dist,idx))
if len(overlaps) == 0:
novl_indices.append(hkl2.indices[i])
novl_i.append(hkl2.iobs[i])
novl_sigma.append(hkl2.sigma_iobs[i])
n_nonovl += 1
else:
print hkl2.indices[i], x, y, z
for dist, idx in overlaps:
xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx]
print hkl1.indices[idx], xo, yo, zo
print dist, idx
print
print
n_ref = len(hkl2.indices)
print "%.2f%% overlap!" % (100.*(n_ref-n_nonovl)/n_ref)
novl_array = miller.array(miller_set=miller.set(crystal_symmetry=hkl2.symm, indices=novl_indices),
data=novl_i, sigmas=novl_sigma)
stats = dataset_statistics(novl_array, anomalous=False, sigma_filtering="xds")
stats.show(out=sys.stdout)
novl_array = novl_array.customized_copy(anomalous_flag=False).map_to_asu()
novl_array = novl_array.eliminate_sys_absent()
novl_array = novl_array.select(novl_array.sigmas() >= 0)
filtr = filter_intensities_by_sigma(novl_array, "xds")
hklout = os.path.splitext(os.path.basename(files[1]))[0] + "_novl.mtz"
filtr.array_merged.set_observation_type_xray_intensity().as_mtz_dataset(column_root_label="IMEAN").mtz_object().write(hklout)
def __init__(
self,
intensities,
batches,
n_bins=20,
d_min=None,
cc_one_half_method="sigma_tau",
group_size=None,
):
self.intensities = intensities
self.batches = batches
self._cc_one_half_method = cc_one_half_method
self._n_bins = n_bins
unmerged_intensities = None
for ma in intensities:
if unmerged_intensities is None:
unmerged_intensities = ma
else:
unmerged_intensities = unmerged_intensities.concatenate(
ma, assert_is_similar_symmetry=False).set_observation_type(
unmerged_intensities.observation_type())
self.binner = unmerged_intensities.eliminate_sys_absent(
).setup_binner_counting_sorted(n_bins=self._n_bins)
self.merging_statistics = dataset_statistics(
unmerged_intensities,
n_bins=n_bins,
cc_one_half_significance_level=0.01,
binning_method="counting_sorted",
anomalous=True,
use_internal_variance=False,
eliminate_sys_absent=False,
cc_one_half_method=self._cc_one_half_method,
assert_is_not_unique_set_under_symmetry=False,
)
if self._cc_one_half_method == "sigma_tau":
self.cc_overall = self.merging_statistics.cc_one_half_sigma_tau_overall
else:
self.cc_overall = self.merging_statistics.cc_one_half_overall
self._group_size = group_size
self._setup_processing_groups()
self.delta_cc = self._compute_delta_ccs()
def test_ResolutionPlotsAndStats(iobs):
i_obs_anom = iobs.as_anomalous_array()
iobs_anom = i_obs_anom.map_to_asu().customized_copy(info=iobs.info())
n_bins = 2
result = dataset_statistics(iobs,
assert_is_not_unique_set_under_symmetry=False,
n_bins=n_bins)
anom_result = dataset_statistics(
iobs_anom,
assert_is_not_unique_set_under_symmetry=False,
anomalous=True,
n_bins=n_bins,
)
plotter = ResolutionPlotsAndStats(result, anom_result)
assert plotter.d_star_sq_ticktext == [
"1.26", "1.19", "1.13", "1.08", "1.04"
]
assert plotter.d_star_sq_tickvals == pytest.approx(
[0.6319, 0.7055, 0.7792, 0.8528, 0.9264], 1e-4)
tables = plotter.statistics_tables()
assert len(tables) == 2 # overall and per resolution
# test plots individually
d = plotter.cc_one_half_plot()
assert len(d["cc_one_half"]["data"]) == 4
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"])
d = plotter.i_over_sig_i_plot()
assert len(d["i_over_sig_i"]["data"]) == 1
assert len(d["i_over_sig_i"]["data"][0]["y"]) == n_bins
d = plotter.r_pim_plot()
assert len(d["r_pim"]["data"]) == 1
assert len(d["r_pim"]["data"][0]["y"]) == n_bins
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
# now try centric options and sigma tau for cc_one_half
plotter = ResolutionPlotsAndStats(result, anom_result, is_centric=True)
d = plotter.cc_one_half_plot(method="sigma_tau")
assert len(d["cc_one_half"]["data"]) == 4
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:2])
assert d["cc_one_half"]["data"][2] == {} # no anomalous plots
assert d["cc_one_half"]["data"][3] == {} # no anomalous plots
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
assert d["completeness"]["data"][1] == {}
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
assert d["multiplicity_vs_resolution"]["data"][1] == {}
plots = plotter.make_all_plots()
assert list(plots.keys()) == [
"cc_one_half",
"i_over_sig_i",
"completeness",
"multiplicity_vs_resolution",
"r_pim",
]
for plot in plots.values():
assert plot["layout"]["xaxis"][
"ticktext"] == plotter.d_star_sq_ticktext
assert plot["layout"]["xaxis"][
"tickvals"] == plotter.d_star_sq_tickvals
def __init__(self, unmerged_intensities, batches_all, n_bins=20, d_min=None,
cc_one_half_method='sigma_tau', id_to_batches=None):
sel = unmerged_intensities.sigmas() > 0
unmerged_intensities = unmerged_intensities.select(sel).set_info(
unmerged_intensities.info())
batches_all = batches_all.select(sel)
unmerged_intensities.setup_binner(n_bins=n_bins)
self.unmerged_intensities = unmerged_intensities
self.merged_intensities = unmerged_intensities.merge_equivalents().array()
separate = separate_unmerged(unmerged_intensities, batches_all,
id_to_batches=id_to_batches)
self.intensities = separate.intensities
self.batches = separate.batches
self.run_id_to_batch_id = separate.run_id_to_batch_id
from iotbx.merging_statistics import dataset_statistics
self.merging_statistics = dataset_statistics(
unmerged_intensities, n_bins=n_bins,
cc_one_half_significance_level=0.01,
binning_method='counting_sorted',
anomalous=True,
use_internal_variance=False,
eliminate_sys_absent=False,
cc_one_half_method=cc_one_half_method)
if cc_one_half_method == 'sigma_tau':
cc_overall = self.merging_statistics.cc_one_half_sigma_tau_overall
else:
cc_overall = self.merging_statistics.cc_one_half_overall
self.merging_statistics.show()
self.delta_cc = flex.double()
for test_k in self.intensities.keys():
#print test_k
indices_i = flex.miller_index()
data_i = flex.double()
sigmas_i = flex.double()
for k, unmerged in self.intensities.iteritems():
if k == test_k: continue
indices_i.extend(unmerged.indices())
data_i.extend(unmerged.data())
sigmas_i.extend(unmerged.sigmas())
unmerged_i = unmerged_intensities.customized_copy(
indices=indices_i, data=data_i, sigmas=sigmas_i).set_info(
unmerged_intensities.info())
unmerged_i.setup_binner_counting_sorted(n_bins=n_bins)
if cc_one_half_method == 'sigma_tau':
cc_bins = unmerged_i.cc_one_half_sigma_tau(
use_binning=True, return_n_refl=True)
else:
cc_bins = unmerged_i.cc_one_half(use_binning=True, return_n_refl=True)
cc_i = flex.mean_weighted(
flex.double(b[0] for b in cc_bins.data[1:-1]),
flex.double(b[1] for b in cc_bins.data[1:-1]))
delta_cc_i = cc_i - cc_overall
self.delta_cc.append(delta_cc_i)
def merging_and_model_statistics(f_obs,
f_model,
r_free_flags,
unmerged_i_obs,
n_bins=20,
sigma_filtering=Auto,
anomalous=False,
use_internal_variance=True):
"""
Compute merging statistics - CC* in particular - together with measures of
model quality using reciprocal-space data (R-factors and CCs). See Karplus
& Diederichs 2012 for rationale.
"""
from iotbx import merging_statistics
free_sel = r_free_flags
# very important: must use original intensities for i_obs, not squared f_obs,
# because French-Wilson treatment is one-way
assert (unmerged_i_obs.sigmas() is not None)
info = unmerged_i_obs.info()
assert (info is not None)
unmerged_i_obs = unmerged_i_obs.customized_copy(crystal_symmetry=f_obs)
unmerged_i_obs = unmerged_i_obs.select(
unmerged_i_obs.sigmas() >= 0).set_info(info)
filter = merging_statistics.filter_intensities_by_sigma(
array=unmerged_i_obs, sigma_filtering=sigma_filtering)
i_obs = filter.array_merged
unmerged_i_obs = filter.array
# average Bijvoet pairs if not anomalous
if (not anomalous):
if (i_obs.anomalous_flag()):
i_obs = i_obs.average_bijvoet_mates()
if (f_obs.anomalous_flag()):
f_obs = f_obs.average_bijvoet_mates()
if (f_model.anomalous_flag()):
f_model = f_model.average_bijvoet_mates()
if (free_sel.anomalous_flag()):
free_sel = free_sel.average_bijvoet_mates()
# create Bijvoet pairs if an array is not anomalous
else:
if (not i_obs.anomalous_flag()):
i_obs = i_obs.generate_bijvoet_mates()
if (not f_obs.anomalous_flag()):
f_obs = f_obs.generate_bijvoet_mates()
if (not f_model.anomalous_flag()):
f_model = f_model.generate_bijvoet_mates()
if (not free_sel.anomalous_flag()):
free_sel = free_sel.generate_bijvoet_mates()
if (free_sel.data().count(True) == 0):
raise Sorry(
"R-free array does not select any reflections. To calculate " +
"CC* and related statistics, a valid set of R-free flags must be used."
)
work_sel = free_sel.customized_copy(data=~free_sel.data())
i_obs, f_obs = i_obs.common_sets(other=f_obs)
i_obs, f_model = i_obs.common_sets(other=f_model)
i_obs, work_sel = i_obs.common_sets(other=work_sel)
i_obs, free_sel = i_obs.common_sets(other=free_sel)
i_calc = abs(f_model).f_as_f_sq()
d_max, d_min = i_calc.d_max_min()
model_arrays = merging_statistics.model_based_arrays(f_obs=f_obs,
i_obs=i_obs,
i_calc=i_calc,
work_sel=work_sel,
free_sel=free_sel)
return merging_statistics.dataset_statistics(
i_obs=unmerged_i_obs,
crystal_symmetry=i_calc,
d_min=d_min,
d_max=d_max,
n_bins=n_bins,
model_arrays=model_arrays,
anomalous=anomalous,
use_internal_variance=use_internal_variance,
sigma_filtering=None) # no need, since it was done here
def write(self, experiments, reflections):
"""
Write the experiments and reflections to file
"""
# if mmmcif filename is auto, then choose scaled.cif or integrated.cif
if self.params.mmcif.hklout in (None, Auto, "auto"):
if ("intensity.scale.value"
in reflections) and ("intensity.scale.variance"
in reflections):
filename = "scaled.cif"
logger.info(
"Data appears to be scaled, setting mmcif.hklout = 'scaled_unmerged.cif'"
)
else:
filename = "integrated.cif"
logger.info(
"Data appears to be unscaled, setting mmcif.hklout = 'integrated.cif'"
)
# Select reflections
selection = reflections.get_flags(reflections.flags.integrated,
all=True)
reflections = reflections.select(selection)
# Filter out bad variances and other issues, but don't filter on ice rings
# or alter partialities.
### Assumes you want to apply the lp and dqe corrections to sum and prf
### Do we want to combine partials?
reflections = filter_reflection_table(
reflections,
self.params.intensity,
combine_partials=False,
partiality_threshold=0.0,
d_min=self.params.mtz.d_min,
)
# Get the cif block
cif_block = iotbx.cif.model.block()
# Audit trail
dials_version = dials.util.version.dials_version()
cif_block["_audit.creation_method"] = dials_version
cif_block["_audit.creation_date"] = datetime.date.today().isoformat()
cif_block["_computing.data_reduction"] = (
"%s (Winter, G. et al., 2018)" % dials_version)
cif_block[
"_publ.section_references"] = "Winter, G. et al. (2018) Acta Cryst. D74, 85-97."
# Hard coding X-ray
cif_block["_pdbx_diffrn_data_section.id"] = "dials"
cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray"
cif_block["_pdbx_diffrn_data_section.type_merged"] = "false"
cif_block["_pdbx_diffrn_data_section.type_scaled"] = str(
"scale" in self.params.intensity).lower()
# FIXME Haven't put in any of these bits yet
#
# Facility/beamline proposal tracking details
#
# cif_block["_pdbx_diffrn_data_section_experiment.ordinal"] = 1
# cif_block["_pdbx_diffrn_data_section_experiment.data_section_id"] = "dials"
# cif_block["_pdbx_diffrn_data_section_experiment.proposal_id"] = "<PROPOSAL ID>
# Facility/beamline details for this data collection
#
# cif_block["_pdbx_diffrn_data_section_site.data_section_id"] = 'dials'
# cif_block["_pdbx_diffrn_data_section_site.facility"] = "DIAMOND"
# cif_block["_pdbx_diffrn_data_section_site.beamline"] = "VMX-M"
# cif_block["_pdbx_diffrn_data_section_site.collection_date"] = scan.epochs()[0]
# cif_block["_pdbx_diffrn_data_section_site.detector"] = detector[0].name()
# cif_block["_pdbx_diffrn_data_section_site.detector_type"] = detector[0].type()
# Write the crystal information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_unmerged_cell.ordinal",
"_pdbx_diffrn_unmerged_cell.crystal_id",
"_pdbx_diffrn_unmerged_cell.wavelength",
"_pdbx_diffrn_unmerged_cell.cell_length_a",
"_pdbx_diffrn_unmerged_cell.cell_length_b",
"_pdbx_diffrn_unmerged_cell.cell_length_c",
"_pdbx_diffrn_unmerged_cell.cell_angle_alpha",
"_pdbx_diffrn_unmerged_cell.cell_angle_beta",
"_pdbx_diffrn_unmerged_cell.cell_angle_gamma",
"_pdbx_diffrn_unmerged_cell.Bravais_lattice",
))
crystals = experiments.crystals()
crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)}
for i, exp in enumerate(experiments):
crystal = exp.crystal
crystal_id = crystal_to_id[crystal]
wavelength = exp.beam.get_wavelength()
a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters()
latt_type = str(
bravais_types.bravais_lattice(group=crystal.get_space_group()))
cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha,
beta, gamma, latt_type))
cif_block.add_loop(cif_loop)
# Write the scan information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_scan.scan_id",
"_pdbx_diffrn_scan.crystal_id",
"_pdbx_diffrn_scan.image_id_begin",
"_pdbx_diffrn_scan.image_id_end",
"_pdbx_diffrn_scan.scan_angle_begin",
"_pdbx_diffrn_scan.scan_angle_end",
))
for i, exp in enumerate(experiments):
scan = exp.scan
crystal_id = crystal_to_id[exp.crystal]
image_range = scan.get_image_range()
osc_range = scan.get_oscillation_range(deg=True)
cif_loop.add_row((
i + 1,
crystal_id,
image_range[0],
image_range[1],
osc_range[0],
osc_range[1],
))
cif_block.add_loop(cif_loop)
# Make a dict of unit_cell parameters
unit_cell_parameters = {}
if crystal.num_scan_points > 1:
for i in range(crystal.num_scan_points):
a, b, c, alpha, beta, gamma = crystal.get_unit_cell_at_scan_point(
i).parameters()
unit_cell_parameters[i] = (a, b, c, alpha, beta, gamma)
else:
unit_cell_parameters[0] = (a, b, c, alpha, beta, gamma)
### _pdbx_diffrn_image_proc has been removed from the dictionary extension.
### Keeping this section commented out as it may be added back in some
### form in future
#
# Write the image data
# scan = experiments[0].scan
# z0 = scan.get_image_range()[0]
#
# cif_loop = iotbx.cif.model.loop(
# header=("_pdbx_diffrn_image_proc.image_id",
# "_pdbx_diffrn_image_proc.crystal_id",
# "_pdbx_diffrn_image_proc.image_number",
# "_pdbx_diffrn_image_proc.phi_value",
# "_pdbx_diffrn_image_proc.wavelength",
# "_pdbx_diffrn_image_proc.cell_length_a",
# "_pdbx_diffrn_image_proc.cell_length_b",
# "_pdbx_diffrn_image_proc.cell_length_c",
# "_pdbx_diffrn_image_proc.cell_angle_alpha",
# "_pdbx_diffrn_image_proc.cell_angle_beta",
# "_pdbx_diffrn_image_proc.cell_angle_gamma"))
# for i in range(len(scan)):
# z = z0 + i
# if crystal.num_scan_points > 1:
# a, b, c, alpha, beta, gamma = unit_cell_parameters[i]
# else:
# a, b, c, alpha, beta, gamma = unit_cell_parameters[0]
# # phi is the angle at the image centre
# phi = scan.get_angle_from_image_index(z + 0.5, deg=True)
# cif_loop.add_row((i+1, 1, z, phi, wavelength,
# a, b, c, alpha, beta, gamma))
# cif_block.add_loop(cif_loop)
# Write reflection data
# Required columns
header = (
"_pdbx_diffrn_unmerged_refln.reflection_id",
"_pdbx_diffrn_unmerged_refln.scan_id",
"_pdbx_diffrn_unmerged_refln.image_id_begin",
"_pdbx_diffrn_unmerged_refln.image_id_end",
"_pdbx_diffrn_unmerged_refln.index_h",
"_pdbx_diffrn_unmerged_refln.index_k",
"_pdbx_diffrn_unmerged_refln.index_l",
)
headernames = {
"scales": "_pdbx_diffrn_unmerged_refln.scale_value",
"intensity.scale.value":
"_pdbx_diffrn_unmerged_refln.intensity_meas",
"intensity.scale.sigma":
"_pdbx_diffrn_unmerged_refln.intensity_sigma",
"intensity.sum.value": "_pdbx_diffrn_unmerged_refln.intensity_sum",
"intensity.sum.sigma":
"_pdbx_diffrn_unmerged_refln.intensity_sum_sigma",
"intensity.prf.value": "_pdbx_diffrn_unmerged_refln.intensity_prf",
"intensity.prf.sigma":
"_pdbx_diffrn_unmerged_refln.intensity_prf_sigma",
"angle": "_pdbx_diffrn_unmerged_refln.scan_angle_reflection",
"partiality": "_pdbx_diffrn_unmerged_refln.partiality",
}
variables_present = []
if "scale" in self.params.intensity:
reflections["scales"] = 1.0 / reflections["inverse_scale_factor"]
reflections["intensity.scale.sigma"] = (
reflections["intensity.scale.variance"]**0.5)
variables_present.extend(
["scales", "intensity.scale.value", "intensity.scale.sigma"])
if "sum" in self.params.intensity:
reflections["intensity.sum.sigma"] = (
reflections["intensity.sum.variance"]**0.5)
variables_present.extend(
["intensity.sum.value", "intensity.sum.sigma"])
if "profile" in self.params.intensity:
reflections["intensity.prf.sigma"] = (
reflections["intensity.prf.variance"]**0.5)
variables_present.extend(
["intensity.prf.value", "intensity.prf.sigma"])
# Should always exist
reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG
variables_present.extend(["angle"])
if "partiality" in reflections:
variables_present.extend(["partiality"])
for name in variables_present:
if name in reflections:
header += (headernames[name], )
if "scale" in self.params.intensity:
# Write dataset_statistics - first make a miller array
crystal_symmetry = cctbxcrystal.symmetry(
space_group=experiments[0].crystal.get_space_group(),
unit_cell=experiments[0].crystal.get_unit_cell(),
)
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=reflections["miller_index"],
anomalous_flag=False,
)
i_obs = miller.array(miller_set,
data=reflections["intensity.scale.value"])
i_obs.set_observation_type_xray_intensity()
i_obs.set_sigmas(reflections["intensity.scale.sigma"])
i_obs.set_info(
miller.array_info(source="DIALS",
source_type="reflection_tables"))
result = dataset_statistics(
i_obs=i_obs,
crystal_symmetry=crystal_symmetry,
use_internal_variance=False,
eliminate_sys_absent=False,
)
cif_block.update(result.as_cif_block())
cif_loop = iotbx.cif.model.loop(header=header)
for i, r in enumerate(reflections.rows()):
refl_id = i + 1
scan_id = r["id"] + 1
_, _, _, _, z0, z1 = r["bbox"]
h, k, l = r["miller_index"]
variable_values = tuple((r[name]) for name in variables_present)
cif_loop.add_row((refl_id, scan_id, z0, z1, h, k, l) +
variable_values)
cif_block.add_loop(cif_loop)
# Add the block
self._cif["dials"] = cif_block
# Print to file
with open(filename, "w") as fh:
self._cif.show(out=fh)
# Log
logger.info("Wrote reflections to %s" % filename)
def run(files):
assert len(files) == 2
hkl1 = xds_ascii.XDS_ASCII(files[0], sys.stdout)
hkl2 = xds_ascii.XDS_ASCII(files[1], sys.stdout)
hkl1_points = numpy.column_stack((hkl1.xd, hkl1.yd, hkl1.zd))
tree1 = spatial.cKDTree(hkl1_points)
n_ovl, n_nonovl = 0, 0
novl_indices, novl_i, novl_sigma = flex.miller_index(), flex.double(
), flex.double()
for i in xrange(len(hkl2.indices)):
x, y, z = hkl2.xd[i], hkl2.yd[i], hkl2.zd[i]
#if z > 180:
# continue
dists, idxs = tree1.query((x, y, z), k=3, p=1)
overlaps = []
for dist, idx in zip(dists, idxs):
idx = int(idx)
xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx]
if abs(z - zo) < 2.5 and (xo - x)**2 + (
yo - y)**2 < 15**2: # FIXME MAGIC NUMBER!
overlaps.append((dist, idx))
if len(overlaps) == 0:
novl_indices.append(hkl2.indices[i])
novl_i.append(hkl2.iobs[i])
novl_sigma.append(hkl2.sigma_iobs[i])
n_nonovl += 1
else:
print hkl2.indices[i], x, y, z
for dist, idx in overlaps:
xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx]
print hkl1.indices[idx], xo, yo, zo
print dist, idx
print
print
n_ref = len(hkl2.indices)
print "%.2f%% overlap!" % (100. * (n_ref - n_nonovl) / n_ref)
novl_array = miller.array(miller_set=miller.set(crystal_symmetry=hkl2.symm,
indices=novl_indices),
data=novl_i,
sigmas=novl_sigma)
stats = dataset_statistics(novl_array,
anomalous=False,
sigma_filtering="xds")
stats.show(out=sys.stdout)
novl_array = novl_array.customized_copy(anomalous_flag=False).map_to_asu()
novl_array = novl_array.eliminate_sys_absent()
novl_array = novl_array.select(novl_array.sigmas() >= 0)
filtr = filter_intensities_by_sigma(novl_array, "xds")
hklout = os.path.splitext(os.path.basename(files[1]))[0] + "_novl.mtz"
filtr.array_merged.set_observation_type_xray_intensity().as_mtz_dataset(
column_root_label="IMEAN").mtz_object().write(hklout)
def make_cif_block(self, experiments, reflections):
"""Write the data to a cif block"""
# Select reflections
selection = reflections.get_flags(reflections.flags.integrated, all=True)
reflections = reflections.select(selection)
# Filter out bad variances and other issues, but don't filter on ice rings
# or alter partialities.
# Assumes you want to apply the lp and dqe corrections to sum and prf
# Do we want to combine partials?
reflections = filter_reflection_table(
reflections,
self.params.intensity,
combine_partials=False,
partiality_threshold=0.0,
d_min=self.params.mtz.d_min,
)
# Get the cif block
cif_block = iotbx.cif.model.block()
# Audit trail
dials_version = dials.util.version.dials_version()
cif_block["_audit.revision_id"] = 1
cif_block["_audit.creation_method"] = dials_version
cif_block["_audit.creation_date"] = datetime.date.today().isoformat()
cif_block["_entry.id"] = "DIALS"
# add software loop
mmcif_software_header = (
"_software.pdbx_ordinal",
"_software.citation_id",
"_software.name", # as defined at [1]
"_software.version",
"_software.type",
"_software.classification",
"_software.description",
)
mmcif_citations_header = (
"_citation.id",
"_citation.journal_abbrev",
"_citation.journal_volume",
"_citation.journal_issue",
"_citation.page_first",
"_citation.page_last",
"_citation.year",
"_citation.title",
)
software_loop = iotbx.cif.model.loop(header=mmcif_software_header)
citations_loop = iotbx.cif.model.loop(header=mmcif_citations_header)
software_loop.add_row(
(
1,
1,
"DIALS",
dials_version,
"package",
"data processing",
"Data processing and integration within the DIALS software package",
)
)
citations_loop.add_row(
(
1,
"Acta Cryst. D",
74,
2,
85,
97,
2018,
"DIALS: implementation and evaluation of a new integration package",
)
)
if "scale" in self.params.intensity:
software_loop.add_row(
(
2,
2,
"DIALS",
dials_version,
"program",
"data scaling",
"Data scaling and merging within the DIALS software package",
)
)
citations_loop.add_row(
(
2,
"Acta Cryst. D",
76,
4,
385,
399,
2020,
"Scaling diffraction data in the DIALS software package: algorithms and new approaches for multi-crystal scaling",
)
)
cif_block.add_loop(software_loop)
cif_block.add_loop(citations_loop)
# Hard coding X-ray
if self.params.mmcif.pdb_version == "v5_next":
cif_block["_pdbx_diffrn_data_section.id"] = "dials"
cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray"
cif_block["_pdbx_diffrn_data_section.type_merged"] = "false"
cif_block["_pdbx_diffrn_data_section.type_scaled"] = str(
"scale" in self.params.intensity
).lower()
# FIXME finish metadata addition - detector and source details needed
# http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html
# Add source information;
# _diffrn_source.pdbx_wavelength_list = (list of wavelengths)
# _diffrn_source.source = (general class of source e.g. synchrotron)
# _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04)
wls = []
epochs = []
for exp in experiments:
wls.append(round(exp.beam.get_wavelength(), 5))
epochs.append(exp.scan.get_epochs()[0])
unique_wls = set(wls)
cif_block["_exptl_crystal.id"] = 1 # links to crystal_id
cif_block["_diffrn.id"] = 1 # links to diffrn_id
cif_block["_diffrn.crystal_id"] = 1
cif_block["_diffrn_source.diffrn_id"] = 1
cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join(
str(w) for w in unique_wls
)
# Add detector information;
# _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc)
# _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd)
# _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M)
# One date is required, so if multiple just use the first date.
min_epoch = min(epochs)
date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch))
cif_block["_diffrn_detector.diffrn_id"] = 1
cif_block["_diffrn_detector.pdbx_collection_date"] = date_str
# Write reflection data
# Required columns
header = (
"_pdbx_diffrn_unmerged_refln.reflection_id",
"_pdbx_diffrn_unmerged_refln.scan_id",
"_pdbx_diffrn_unmerged_refln.image_id_begin",
"_pdbx_diffrn_unmerged_refln.image_id_end",
"_pdbx_diffrn_unmerged_refln.index_h",
"_pdbx_diffrn_unmerged_refln.index_k",
"_pdbx_diffrn_unmerged_refln.index_l",
)
extra_items = {
"scales": ("_pdbx_diffrn_unmerged_refln.scale_value", "%5.3f"),
"intensity.scale.value": (
"_pdbx_diffrn_unmerged_refln.intensity_meas",
"%8.3f",
),
"intensity.scale.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_sigma",
"%8.3f",
),
"intensity.sum.value": (
"_pdbx_diffrn_unmerged_refln.intensity_sum",
"%8.3f",
),
"intensity.sum.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_sum_sigma",
"%8.3f",
),
"intensity.prf.value": (
"_pdbx_diffrn_unmerged_refln.intensity_prf",
"%8.3f",
),
"intensity.prf.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_prf_sigma",
"%8.3f",
),
"angle": ("_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "%7.4f"),
"partiality": ("_pdbx_diffrn_unmerged_refln.partiality", "%7.4f"),
}
variables_present = []
if "scale" in self.params.intensity:
reflections["scales"] = 1.0 / reflections["inverse_scale_factor"]
reflections["intensity.scale.sigma"] = flex.sqrt(
reflections["intensity.scale.variance"]
)
variables_present.extend(
["scales", "intensity.scale.value", "intensity.scale.sigma"]
)
if "sum" in self.params.intensity:
reflections["intensity.sum.sigma"] = flex.sqrt(
reflections["intensity.sum.variance"]
)
variables_present.extend(["intensity.sum.value", "intensity.sum.sigma"])
if "profile" in self.params.intensity:
reflections["intensity.prf.sigma"] = flex.sqrt(
reflections["intensity.prf.variance"]
)
variables_present.extend(["intensity.prf.value", "intensity.prf.sigma"])
# Should always exist
reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG
variables_present.extend(["angle"])
if "partiality" in reflections:
variables_present.extend(["partiality"])
for name in variables_present:
if name in reflections:
header += (extra_items[name][0],)
self._fmt += " " + extra_items[name][1]
if "scale" in self.params.intensity:
# Write dataset_statistics - first make a miller array
crystal_symmetry = cctbxcrystal.symmetry(
space_group=experiments[0].crystal.get_space_group(),
unit_cell=experiments[0].crystal.get_unit_cell(),
)
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=reflections["miller_index"],
anomalous_flag=False,
)
i_obs = miller.array(miller_set, data=reflections["intensity.scale.value"])
i_obs.set_observation_type_xray_intensity()
i_obs.set_sigmas(reflections["intensity.scale.sigma"])
i_obs.set_info(
miller.array_info(source="DIALS", source_type="reflection_tables")
)
result = dataset_statistics(
i_obs=i_obs,
crystal_symmetry=crystal_symmetry,
use_internal_variance=False,
eliminate_sys_absent=False,
)
merged_block = iotbx.cif.model.block()
merged_block["_reflns.pdbx_ordinal"] = 1
merged_block["_reflns.pdbx_diffrn_id"] = 1
merged_block["_reflns.entry_id"] = "DIALS"
merged_data = result.as_cif_block()
merged_block.update(merged_data)
cif_block.update(merged_block)
# Write the crystal information
# if v5, thats all so return
if self.params.mmcif.pdb_version == "v5":
return cif_block
# continue if v5_next
cif_loop = iotbx.cif.model.loop(
header=(
"_pdbx_diffrn_unmerged_cell.ordinal",
"_pdbx_diffrn_unmerged_cell.crystal_id",
"_pdbx_diffrn_unmerged_cell.wavelength",
"_pdbx_diffrn_unmerged_cell.cell_length_a",
"_pdbx_diffrn_unmerged_cell.cell_length_b",
"_pdbx_diffrn_unmerged_cell.cell_length_c",
"_pdbx_diffrn_unmerged_cell.cell_angle_alpha",
"_pdbx_diffrn_unmerged_cell.cell_angle_beta",
"_pdbx_diffrn_unmerged_cell.cell_angle_gamma",
"_pdbx_diffrn_unmerged_cell.Bravais_lattice",
)
)
crystals = experiments.crystals()
crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)}
for i, exp in enumerate(experiments):
crystal = exp.crystal
crystal_id = crystal_to_id[crystal]
wavelength = exp.beam.get_wavelength()
a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters()
latt_type = str(
bravais_types.bravais_lattice(group=crystal.get_space_group())
)
cif_loop.add_row(
(i + 1, crystal_id, wavelength, a, b, c, alpha, beta, gamma, latt_type)
)
cif_block.add_loop(cif_loop)
# Write the scan information
cif_loop = iotbx.cif.model.loop(
header=(
"_pdbx_diffrn_scan.scan_id",
"_pdbx_diffrn_scan.crystal_id",
"_pdbx_diffrn_scan.image_id_begin",
"_pdbx_diffrn_scan.image_id_end",
"_pdbx_diffrn_scan.scan_angle_begin",
"_pdbx_diffrn_scan.scan_angle_end",
)
)
expid_to_scan_id = {exp.identifier: i + 1 for i, exp in enumerate(experiments)}
for i, exp in enumerate(experiments):
scan = exp.scan
crystal_id = crystal_to_id[exp.crystal]
image_range = scan.get_image_range()
osc_range = scan.get_oscillation_range(deg=True)
cif_loop.add_row(
(
i + 1,
crystal_id,
image_range[0],
image_range[1],
osc_range[0],
osc_range[1],
)
)
cif_block.add_loop(cif_loop)
_, _, _, _, z0, z1 = reflections["bbox"].parts()
h, k, l = [
hkl.iround() for hkl in reflections["miller_index"].as_vec3_double().parts()
]
# make scan id consistent with header as defined above
scan_id = flex.int(reflections.size(), 0)
for id_ in reflections.experiment_identifiers().keys():
expid = reflections.experiment_identifiers()[id_]
sel = reflections["id"] == id_
scan_id.set_selected(sel, expid_to_scan_id[expid])
loop_values = [
flex.size_t_range(1, len(reflections) + 1),
scan_id,
z0,
z1,
h,
k,
l,
] + [reflections[name] for name in variables_present]
cif_loop = iotbx.cif.model.loop(data=dict(zip(header, loop_values)))
cif_block.add_loop(cif_loop)
return cif_block
def merging_and_model_statistics (
f_obs,
f_model,
r_free_flags,
unmerged_i_obs,
n_bins=20,
sigma_filtering=Auto,
anomalous=False,
use_internal_variance=True) :
"""
Compute merging statistics - CC* in particular - together with measures of
model quality using reciprocal-space data (R-factors and CCs). See Karplus
& Diederichs 2012 for rationale.
"""
from iotbx import merging_statistics
free_sel = r_free_flags
# very important: must use original intensities for i_obs, not squared f_obs,
# because French-Wilson treatment is one-way
assert (unmerged_i_obs.sigmas() is not None)
info = unmerged_i_obs.info()
assert (info is not None)
unmerged_i_obs = unmerged_i_obs.customized_copy(crystal_symmetry=f_obs)
unmerged_i_obs = unmerged_i_obs.select(
unmerged_i_obs.sigmas() >= 0).set_info(info)
filter = merging_statistics.filter_intensities_by_sigma(
array=unmerged_i_obs,
sigma_filtering=sigma_filtering)
i_obs = filter.array_merged
unmerged_i_obs = filter.array
# average Bijvoet pairs if not anomalous
if (not anomalous):
if (i_obs.anomalous_flag()):
i_obs = i_obs.average_bijvoet_mates()
if (f_obs.anomalous_flag()):
f_obs = f_obs.average_bijvoet_mates()
if (f_model.anomalous_flag()):
f_model = f_model.average_bijvoet_mates()
if (free_sel.anomalous_flag()):
free_sel = free_sel.average_bijvoet_mates()
# create Bijvoet pairs if an array is not anomalous
else:
if (not i_obs.anomalous_flag()):
i_obs = i_obs.generate_bijvoet_mates()
if (not f_obs.anomalous_flag()):
f_obs = f_obs.generate_bijvoet_mates()
if (not f_model.anomalous_flag()):
f_model = f_model.generate_bijvoet_mates()
if (not free_sel.anomalous_flag()):
free_sel = free_sel.generate_bijvoet_mates()
if (free_sel.data().count(True) == 0) :
raise Sorry("R-free array does not select any reflections. To calculate "+
"CC* and related statistics, a valid set of R-free flags must be used.")
work_sel = free_sel.customized_copy(data=~free_sel.data())
i_obs, f_model = i_obs.common_sets(other=f_model)
i_obs, f_obs = i_obs.common_sets(other=f_obs)
i_obs, work_sel = i_obs.common_sets(other=work_sel)
i_obs, free_sel = i_obs.common_sets(other=free_sel)
i_calc = abs(f_model).f_as_f_sq()
d_max, d_min = i_calc.d_max_min()
model_arrays = merging_statistics.model_based_arrays(
f_obs=f_obs,
i_obs=i_obs,
i_calc=i_calc,
work_sel=work_sel,
free_sel=free_sel)
return merging_statistics.dataset_statistics(
i_obs=unmerged_i_obs,
crystal_symmetry=i_calc,
d_min=d_min,
d_max=d_max,
n_bins=n_bins,
model_arrays=model_arrays,
anomalous=anomalous,
use_internal_variance=use_internal_variance,
sigma_filtering=None) # no need, since it was done here
def write(self, experiments, reflections):
"""
Write the experiments and reflections to file
"""
# if mmcif filename is auto, then choose scaled.cif or integrated.cif
if self.params.mmcif.hklout in (None, Auto, "auto"):
if ("intensity.scale.value"
in reflections) and ("intensity.scale.variance"
in reflections):
filename = "scaled.cif"
logger.info(
"Data appears to be scaled, setting mmcif.hklout = 'scaled.cif'"
)
else:
filename = "integrated.cif"
logger.info(
"Data appears to be unscaled, setting mmcif.hklout = 'integrated.cif'"
)
else:
filename = self.params.mmcif.hklout
# Select reflections
selection = reflections.get_flags(reflections.flags.integrated,
all=True)
reflections = reflections.select(selection)
# Filter out bad variances and other issues, but don't filter on ice rings
# or alter partialities.
# Assumes you want to apply the lp and dqe corrections to sum and prf
# Do we want to combine partials?
reflections = filter_reflection_table(
reflections,
self.params.intensity,
combine_partials=False,
partiality_threshold=0.0,
d_min=self.params.mtz.d_min,
)
# Get the cif block
cif_block = iotbx.cif.model.block()
# Audit trail
dials_version = dials.util.version.dials_version()
cif_block["_audit.creation_method"] = dials_version
cif_block["_audit.creation_date"] = datetime.date.today().isoformat()
cif_block["_computing.data_reduction"] = (
"%s (Winter, G. et al., 2018)" % dials_version)
cif_block[
"_publ.section_references"] = "Winter, G. et al. (2018) Acta Cryst. D74, 85-97."
if "scale" in self.params.intensity:
cif_block[
"_publ.section_references"] += "\nBeilsten-Edmands, J. et al. (2020) Acta Cryst. D76, 385-399."
# Hard coding X-ray
cif_block["_pdbx_diffrn_data_section.id"] = "dials"
cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray"
cif_block["_pdbx_diffrn_data_section.type_merged"] = "false"
cif_block["_pdbx_diffrn_data_section.type_scaled"] = str(
"scale" in self.params.intensity).lower()
# FIXME finish metadata addition - detector and source details needed
# http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html
# Add source information;
# _diffrn_source.pdbx_wavelength_list = (list of wavelengths)
# _diffrn_source.source = (general class of source e.g. synchrotron)
# _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04)
wls = []
epochs = []
for exp in experiments:
wls.append(round(exp.beam.get_wavelength(), 5))
epochs.append(exp.scan.get_epochs()[0])
unique_wls = set(wls)
cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join(
str(w) for w in unique_wls)
# Add detector information;
# _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc)
# _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd)
# _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M)
# One date is required, so if multiple just use the first date.
min_epoch = min(epochs)
date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch))
cif_block["_diffrn_detector.pdbx_collection_date"] = date_str
# Write the crystal information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_unmerged_cell.ordinal",
"_pdbx_diffrn_unmerged_cell.crystal_id",
"_pdbx_diffrn_unmerged_cell.wavelength",
"_pdbx_diffrn_unmerged_cell.cell_length_a",
"_pdbx_diffrn_unmerged_cell.cell_length_b",
"_pdbx_diffrn_unmerged_cell.cell_length_c",
"_pdbx_diffrn_unmerged_cell.cell_angle_alpha",
"_pdbx_diffrn_unmerged_cell.cell_angle_beta",
"_pdbx_diffrn_unmerged_cell.cell_angle_gamma",
"_pdbx_diffrn_unmerged_cell.Bravais_lattice",
))
crystals = experiments.crystals()
crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)}
for i, exp in enumerate(experiments):
crystal = exp.crystal
crystal_id = crystal_to_id[crystal]
wavelength = exp.beam.get_wavelength()
a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters()
latt_type = str(
bravais_types.bravais_lattice(group=crystal.get_space_group()))
cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha,
beta, gamma, latt_type))
cif_block.add_loop(cif_loop)
# Write the scan information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_scan.scan_id",
"_pdbx_diffrn_scan.crystal_id",
"_pdbx_diffrn_scan.image_id_begin",
"_pdbx_diffrn_scan.image_id_end",
"_pdbx_diffrn_scan.scan_angle_begin",
"_pdbx_diffrn_scan.scan_angle_end",
))
for i, exp in enumerate(experiments):
scan = exp.scan
crystal_id = crystal_to_id[exp.crystal]
image_range = scan.get_image_range()
osc_range = scan.get_oscillation_range(deg=True)
cif_loop.add_row((
i + 1,
crystal_id,
image_range[0],
image_range[1],
osc_range[0],
osc_range[1],
))
cif_block.add_loop(cif_loop)
# Make a dict of unit_cell parameters
unit_cell_parameters = {}
if crystal.num_scan_points > 1:
for i in range(crystal.num_scan_points):
a, b, c, alpha, beta, gamma = crystal.get_unit_cell_at_scan_point(
i).parameters()
unit_cell_parameters[i] = (a, b, c, alpha, beta, gamma)
else:
unit_cell_parameters[0] = (a, b, c, alpha, beta, gamma)
### _pdbx_diffrn_image_proc has been removed from the dictionary extension.
### Keeping this section commented out as it may be added back in some
### form in future
#
# Write the image data
# scan = experiments[0].scan
# z0 = scan.get_image_range()[0]
#
# cif_loop = iotbx.cif.model.loop(
# header=("_pdbx_diffrn_image_proc.image_id",
# "_pdbx_diffrn_image_proc.crystal_id",
# "_pdbx_diffrn_image_proc.image_number",
# "_pdbx_diffrn_image_proc.phi_value",
# "_pdbx_diffrn_image_proc.wavelength",
# "_pdbx_diffrn_image_proc.cell_length_a",
# "_pdbx_diffrn_image_proc.cell_length_b",
# "_pdbx_diffrn_image_proc.cell_length_c",
# "_pdbx_diffrn_image_proc.cell_angle_alpha",
# "_pdbx_diffrn_image_proc.cell_angle_beta",
# "_pdbx_diffrn_image_proc.cell_angle_gamma"))
# for i in range(len(scan)):
# z = z0 + i
# if crystal.num_scan_points > 1:
# a, b, c, alpha, beta, gamma = unit_cell_parameters[i]
# else:
# a, b, c, alpha, beta, gamma = unit_cell_parameters[0]
# # phi is the angle at the image centre
# phi = scan.get_angle_from_image_index(z + 0.5, deg=True)
# cif_loop.add_row((i+1, 1, z, phi, wavelength,
# a, b, c, alpha, beta, gamma))
# cif_block.add_loop(cif_loop)
# Write reflection data
# Required columns
header = (
"_pdbx_diffrn_unmerged_refln.reflection_id",
"_pdbx_diffrn_unmerged_refln.scan_id",
"_pdbx_diffrn_unmerged_refln.image_id_begin",
"_pdbx_diffrn_unmerged_refln.image_id_end",
"_pdbx_diffrn_unmerged_refln.index_h",
"_pdbx_diffrn_unmerged_refln.index_k",
"_pdbx_diffrn_unmerged_refln.index_l",
)
headernames = {
"scales": "_pdbx_diffrn_unmerged_refln.scale_value",
"intensity.scale.value":
"_pdbx_diffrn_unmerged_refln.intensity_meas",
"intensity.scale.sigma":
"_pdbx_diffrn_unmerged_refln.intensity_sigma",
"intensity.sum.value": "_pdbx_diffrn_unmerged_refln.intensity_sum",
"intensity.sum.sigma":
"_pdbx_diffrn_unmerged_refln.intensity_sum_sigma",
"intensity.prf.value": "_pdbx_diffrn_unmerged_refln.intensity_prf",
"intensity.prf.sigma":
"_pdbx_diffrn_unmerged_refln.intensity_prf_sigma",
"angle": "_pdbx_diffrn_unmerged_refln.scan_angle_reflection",
"partiality": "_pdbx_diffrn_unmerged_refln.partiality",
}
variables_present = []
if "scale" in self.params.intensity:
reflections["scales"] = 1.0 / reflections["inverse_scale_factor"]
reflections["intensity.scale.sigma"] = flex.sqrt(
reflections["intensity.scale.variance"])
variables_present.extend(
["scales", "intensity.scale.value", "intensity.scale.sigma"])
if "sum" in self.params.intensity:
reflections["intensity.sum.sigma"] = flex.sqrt(
reflections["intensity.sum.variance"])
variables_present.extend(
["intensity.sum.value", "intensity.sum.sigma"])
if "profile" in self.params.intensity:
reflections["intensity.prf.sigma"] = flex.sqrt(
reflections["intensity.prf.variance"])
variables_present.extend(
["intensity.prf.value", "intensity.prf.sigma"])
# Should always exist
reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG
variables_present.extend(["angle"])
if "partiality" in reflections:
variables_present.extend(["partiality"])
for name in variables_present:
if name in reflections:
header += (headernames[name], )
if "scale" in self.params.intensity:
# Write dataset_statistics - first make a miller array
crystal_symmetry = cctbxcrystal.symmetry(
space_group=experiments[0].crystal.get_space_group(),
unit_cell=experiments[0].crystal.get_unit_cell(),
)
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=reflections["miller_index"],
anomalous_flag=False,
)
i_obs = miller.array(miller_set,
data=reflections["intensity.scale.value"])
i_obs.set_observation_type_xray_intensity()
i_obs.set_sigmas(reflections["intensity.scale.sigma"])
i_obs.set_info(
miller.array_info(source="DIALS",
source_type="reflection_tables"))
result = dataset_statistics(
i_obs=i_obs,
crystal_symmetry=crystal_symmetry,
use_internal_variance=False,
eliminate_sys_absent=False,
)
cif_block.update(result.as_cif_block())
cif_loop = iotbx.cif.model.loop(header=header)
for i, r in enumerate(reflections.rows()):
refl_id = i + 1
scan_id = r["id"] + 1
_, _, _, _, z0, z1 = r["bbox"]
h, k, l = r["miller_index"]
variable_values = tuple((r[name]) for name in variables_present)
cif_loop.add_row((refl_id, scan_id, z0, z1, h, k, l) +
variable_values)
cif_block.add_loop(cif_loop)
# Add the block
self._cif["dials"] = cif_block
# Print to file
with open(filename, "w") as fh:
self._cif.show(out=fh)
# Log
logger.info("Wrote reflections to %s" % filename)
def run(args):
from iotbx.reflection_file_reader import any_reflection_file
from xia2.Modules.Analysis import phil_scope
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
args, custom_processor='collect_remaining')
params = params.extract()
n_bins = params.resolution_bins
args = unhandled
intensities = None
batches = None
scales = None
dose = None
reader = any_reflection_file(args[0])
assert reader.file_type() == 'ccp4_mtz'
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
if ma.info().labels == ['BATCH']:
batches = ma
elif ma.info().labels == ['DOSE']:
dose = ma
elif ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
elif ma.info().labels == ['SCALEUSED']:
scales = ma
assert intensities is not None
assert batches is not None
mtz_object = reader.file_content()
indices = mtz_object.extract_original_index_miller_indices()
intensities = intensities.customized_copy(
indices=indices, info=intensities.info())
batches = batches.customized_copy(indices=indices, info=batches.info())
from iotbx import merging_statistics
merging_stats = merging_statistics.dataset_statistics(
intensities, n_bins=n_bins)
merging_acentric = intensities.select_acentric().merge_equivalents()
merging_centric = intensities.select_centric().merge_equivalents()
multiplicities_acentric = {}
multiplicities_centric = {}
for x in sorted(set(merging_acentric.redundancies().data())):
multiplicities_acentric[x] = merging_acentric.redundancies().data().count(x)
for x in sorted(set(merging_centric.redundancies().data())):
multiplicities_centric[x] = merging_centric.redundancies().data().count(x)
headers = [u'Resolution (Å)', 'N(obs)', 'N(unique)', 'Multiplicity', 'Completeness',
'Mean(I)', 'Mean(I/sigma)', 'Rmerge', 'Rmeas', 'Rpim', 'CC1/2', 'CCano']
rows = []
for bin_stats in merging_stats.bins:
row = ['%.2f - %.2f' %(bin_stats.d_max, bin_stats.d_min),
bin_stats.n_obs, bin_stats.n_uniq, '%.2f' %bin_stats.mean_redundancy,
'%.2f' %(100*bin_stats.completeness), '%.1f' %bin_stats.i_mean,
'%.1f' %bin_stats.i_over_sigma_mean, '%.3f' %bin_stats.r_merge,
'%.3f' %bin_stats.r_meas, '%.3f' %bin_stats.r_pim,
'%.3f' %bin_stats.cc_one_half, '%.3f' %bin_stats.cc_anom]
rows.append(row)
from xia2.lib.tabulate import tabulate
merging_stats_table_html = tabulate(rows, headers, tablefmt='html')
merging_stats_table_html = merging_stats_table_html.replace(
'<table>', '<table class="table table-hover table-condensed">')
unit_cell_params = intensities.unit_cell().parameters()
headers = ['', 'Overall', 'Low resolution', 'High resolution']
stats = (merging_stats.overall, merging_stats.bins[0], merging_stats.bins[-1])
rows = [
[u'Resolution (Å)'] + [
'%.2f - %.2f' %(s.d_max, s.d_min) for s in stats],
['Observations'] + ['%i' %s.n_obs for s in stats],
['Unique reflections'] + ['%i' %s.n_uniq for s in stats],
['Multiplicity'] + ['%.1f' %s.mean_redundancy for s in stats],
['Completeness'] + ['%.2f%%' %(s.completeness * 100) for s in stats],
#['Mean intensity'] + ['%.1f' %s.i_mean for s in stats],
['Mean I/sigma(I)'] + ['%.1f' %s.i_over_sigma_mean for s in stats],
['Rmerge'] + ['%.3f' %s.r_merge for s in stats],
['Rmeas'] + ['%.3f' %s.r_meas for s in stats],
['Rpim'] + ['%.3f' %s.r_pim for s in stats],
['CC1/2'] + ['%.3f' %s.cc_one_half for s in stats],
]
rows = [[u'<strong>%s</strong>' %r[0]] + r[1:] for r in rows]
overall_stats_table_html = tabulate(rows, headers, tablefmt='html')
overall_stats_table_html = overall_stats_table_html.replace(
'<table>', '<table class="table table-hover table-condensed">')
#headers = ['Crystal symmetry', '']
#rows = [
#[u'Unit cell: a (Å)', '%.3f' %unit_cell_params[0]],
#[u'b (Å)', '%.3f' %unit_cell_params[1]],
#[u'c (Å)', '%.3f' %unit_cell_params[2]],
#[u'α (°)', '%.3f' %unit_cell_params[3]],
#[u'β (°)', '%.3f' %unit_cell_params[4]],
#[u'γ (°)', '%.3f' %unit_cell_params[5]],
#['Space group', intensities.space_group_info().symbol_and_number()],
#]
#symmetry_table_html = tabulate(rows, headers, tablefmt='html')
symmetry_table_html = """
<p>
<b>Filename:</b> %s
<br>
<b>Unit cell:</b> %s
<br>
<b>Space group:</b> %s
</p>
""" %(os.path.abspath(reader.file_name()),
intensities.space_group_info().symbol_and_number(),
str(intensities.unit_cell()))
if params.anomalous:
intensities = intensities.as_anomalous_array()
batches = batches.as_anomalous_array()
from xia2.Modules.PyChef2.PyChef import remove_batch_gaps
new_batch_data = remove_batch_gaps(batches.data())
new_batches = batches.customized_copy(data=new_batch_data)
sc_vs_b = scales_vs_batch(scales, new_batches)
rmerge_vs_b = rmerge_vs_batch(intensities, new_batches)
intensities.setup_binner(n_bins=n_bins)
merged_intensities = intensities.merge_equivalents().array()
from mmtbx.scaling import twin_analyses
normalised_intensities = twin_analyses.wilson_normalised_intensities(
miller_array=merged_intensities)
nz_test = twin_analyses.n_z_test(
normalised_acentric=normalised_intensities.acentric,
normalised_centric=normalised_intensities.centric)
from mmtbx.scaling import data_statistics
if not intensities.space_group().is_centric():
wilson_scaling = data_statistics.wilson_scaling(
miller_array=merged_intensities, n_residues=200) # XXX default n_residues?
acentric = intensities.select_acentric()
centric = intensities.select_centric()
if acentric.size():
acentric.setup_binner(n_bins=n_bins)
second_moments_acentric = acentric.second_moment_of_intensities(use_binning=True)
if centric.size():
centric.setup_binner(n_bins=n_bins)
second_moments_centric = centric.second_moment_of_intensities(use_binning=True)
d_star_sq_bins = [
(1/bin_stats.d_min**2) for bin_stats in merging_stats.bins]
i_over_sig_i_bins = [
bin_stats.i_over_sigma_mean for bin_stats in merging_stats.bins]
cc_one_half_bins = [
bin_stats.cc_one_half for bin_stats in merging_stats.bins]
cc_anom_bins = [
bin_stats.cc_anom for bin_stats in merging_stats.bins]
from xia2.Modules.PyChef2 import PyChef
if params.chef_min_completeness:
d_min = PyChef.resolution_limit(
mtz_file=args[0], min_completeness=params.chef_min_completeness, n_bins=8)
print 'Estimated d_min for CHEF analysis: %.2f' %d_min
sel = flex.bool(intensities.size(), True)
d_spacings = intensities.d_spacings().data()
sel &= d_spacings >= d_min
intensities = intensities.select(sel)
batches = batches.select(sel)
if dose is not None:
dose = dose.select(sel)
if dose is None:
dose = PyChef.batches_to_dose(batches.data(), params.dose)
else:
dose = dose.data()
pychef_stats = PyChef.Statistics(intensities, dose)
pychef_dict = pychef_stats.to_dict()
def d_star_sq_to_d_ticks(d_star_sq, nticks):
from cctbx import uctbx
d_spacings = uctbx.d_star_sq_as_d(flex.double(d_star_sq))
min_d_star_sq = min(d_star_sq)
dstep = (max(d_star_sq) - min_d_star_sq)/nticks
tickvals = list(min_d_star_sq + (i*dstep) for i in range(nticks))
ticktext = ['%.2f' %(uctbx.d_star_sq_as_d(dsq)) for dsq in tickvals]
return tickvals, ticktext
tickvals, ticktext = d_star_sq_to_d_ticks(d_star_sq_bins, nticks=5)
tickvals_wilson, ticktext_wilson = d_star_sq_to_d_ticks(
wilson_scaling.d_star_sq, nticks=5)
second_moment_d_star_sq = []
if acentric.size():
second_moment_d_star_sq.extend(second_moments_acentric.binner.bin_centers(2))
if centric.size():
second_moment_d_star_sq.extend(second_moments_centric.binner.bin_centers(2))
tickvals_2nd_moment, ticktext_2nd_moment = d_star_sq_to_d_ticks(
second_moment_d_star_sq, nticks=5)
json_data = {
'multiplicities': {
'data': [
{
'x': multiplicities_acentric.keys(),
'y': multiplicities_acentric.values(),
'type': 'bar',
'name': 'Acentric',
'opacity': 0.75,
},
{
'x': multiplicities_centric.keys(),
'y': multiplicities_centric.values(),
'type': 'bar',
'name': 'Centric',
'opacity': 0.75,
},
],
'layout': {
'title': 'Distribution of multiplicities',
'xaxis': {'title': 'Multiplicity'},
'yaxis': {
'title': 'Frequency',
#'rangemode': 'tozero'
},
'bargap': 0,
'barmode': 'overlay',
},
},
'scale_rmerge_vs_batch': {
'data': [
{
'x': sc_vs_b.batches,
'y': sc_vs_b.data,
'type': 'scatter',
'name': 'Scale',
'opacity': 0.75,
},
{
'x': rmerge_vs_b.batches,
'y': rmerge_vs_b.data,
'yaxis': 'y2',
'type': 'scatter',
'name': 'Rmerge',
'opacity': 0.75,
},
],
'layout': {
'title': 'Scale and Rmerge vs batch',
'xaxis': {'title': 'N'},
'yaxis': {
'title': 'Scale',
'rangemode': 'tozero'
},
'yaxis2': {
'title': 'Rmerge',
'overlaying': 'y',
'side': 'right',
'rangemode': 'tozero'
}
},
},
'cc_one_half': {
'data': [
{
'x': d_star_sq_bins, # d_star_sq
'y': cc_one_half_bins,
'type': 'scatter',
'name': 'CC-half',
},
({
'x': d_star_sq_bins, # d_star_sq
'y': cc_anom_bins,
'type': 'scatter',
'name': 'CC-anom',
} if not intensities.space_group().is_centric() else {}),
],
'layout':{
'title': 'CC-half vs resolution',
'xaxis': {
'title': u'Resolution (Å)',
'tickvals': tickvals,
'ticktext': ticktext,
},
'yaxis': {
'title': 'CC-half',
'range': [min(cc_one_half_bins + cc_anom_bins + [0]), 1]
},
},
},
'i_over_sig_i': {
'data': [{
'x': d_star_sq_bins, # d_star_sq
'y': i_over_sig_i_bins,
'type': 'scatter',
'name': 'Scales vs batch',
}],
'layout': {
'title': '<I/sig(I)> vs resolution',
'xaxis': {
'title': u'Resolution (Å)',
'tickvals': tickvals,
'ticktext': ticktext,
},
'yaxis': {
'title': '<I/sig(I)>',
'rangemode': 'tozero'
},
}
},
'second_moments': {
'data': [
({
'x': list(second_moments_acentric.binner.bin_centers(2)), # d_star_sq
'y': second_moments_acentric.data[1:-1],
'type': 'scatter',
'name': '<I^2> acentric',
} if acentric.size() else {}),
({
'x': list(second_moments_centric.binner.bin_centers(2)), # d_star_sq
'y': second_moments_centric.data[1:-1],
'type': 'scatter',
'name': '<I^2> centric',
} if centric.size() else {})
],
'layout': {
'title': 'Second moment of I',
'xaxis': {
'title': u'Resolution (Å)',
'tickvals': tickvals_2nd_moment,
'ticktext': ticktext_2nd_moment,
},
'yaxis': {
'title': '<I^2>',
'rangemode': 'tozero'
},
}
},
'cumulative_intensity_distribution': {
'data': [
{
'x': list(nz_test.z),
'y': list(nz_test.ac_obs),
'type': 'scatter',
'name': 'Acentric observed',
'mode': 'lines',
'line': {
'color': 'rgb(31, 119, 180)',
},
},
{
'x': list(nz_test.z),
'y': list(nz_test.c_obs),
'type': 'scatter',
'name': 'Centric observed',
'mode': 'lines',
'line': {
'color': 'rgb(255, 127, 14)',
},
},
{
'x': list(nz_test.z),
'y': list(nz_test.ac_untwinned),
'type': 'scatter',
'name': 'Acentric theory',
'mode': 'lines',
'line': {
'color': 'rgb(31, 119, 180)',
'dash': 'dot',
},
'opacity': 0.8,
},
{
'x': list(nz_test.z),
'y': list(nz_test.c_untwinned),
'type': 'scatter',
'name': 'Centric theory',
'mode': 'lines',
'line': {
'color': 'rgb(255, 127, 14)',
'dash': 'dot',
},
'opacity': 0.8,
},
],
'layout': {
'title': 'Cumulative intensity distribution',
'xaxis': {'title': 'z'},
'yaxis': {
'title': 'P(Z <= Z)',
'rangemode': 'tozero'
},
}
},
'wilson_intensity_plot': {
'data': ([
{
'x': list(wilson_scaling.d_star_sq),
'y': list(wilson_scaling.mean_I_obs_data),
'type': 'scatter',
'name': 'Observed',
},
{
'x': list(wilson_scaling.d_star_sq),
'y': list(wilson_scaling.mean_I_obs_theory),
'type': 'scatter',
'name': 'Expected',
},
{
'x': list(wilson_scaling.d_star_sq),
'y': list(wilson_scaling.mean_I_normalisation),
'type': 'scatter',
'name': 'Smoothed',
}] if not intensities.space_group().is_centric() else []),
'layout': {
'title': 'Wilson intensity plot',
'xaxis': {
'title': u'Resolution (Å)',
'tickvals': tickvals_wilson,
'ticktext': ticktext_wilson,
},
'yaxis': {
'type': 'log',
'title': 'Mean(I)',
'rangemode': 'tozero',
},
},
},
}
json_data.update(pychef_dict)
from dials.report import html_report
report = html_report.html_report()
page_header = html_report.page_header('xia2 report')
report.add_content(page_header)
overall_panel = html_report.panel('Overall', 'overall', show=True)
overall_table = html_report.table_responsive(
overall_stats_table_html, width=800)
overall_panel.add_content(overall_table)
merging_stats_panel = html_report.panel('Resolution shells', 'merging_stats')
merging_stats_table = html_report.table_responsive(merging_stats_table_html)
merging_stats_panel.add_content(merging_stats_table)
merging_stats_panel_group = html_report.panel_group(
[overall_panel, merging_stats_panel])
div = html_report.div()
div.add_content(html_report.raw_html('<h2>Merging statistics</h2>'))
div.add_content(html_report.raw_html(symmetry_table_html))
div.add_content(merging_stats_panel_group)
report.add_content(div)
resolution_plots_panel = html_report.panel('Analysis by resolution', 'resolution')
for graph in ('cc_one_half', 'i_over_sig_i', 'second_moments',
'wilson_intensity_plot'):
resolution_plots_panel.add_content(html_report.plotly_graph(
json_data[graph], graph))
batch_plots_panel = html_report.panel('Analysis by batch', 'batch')
for graph in ('scale_rmerge_vs_batch', 'completeness_vs_dose',
'rcp_vs_dose', 'scp_vs_dose', 'rd_vs_batch_difference'):
batch_plots_panel.add_content(html_report.plotly_graph(
json_data[graph], graph))
misc_plots_panel = html_report.panel('Miscellaneous', 'misc')
for graph in ('multiplicities', 'cumulative_intensity_distribution'):
misc_plots_panel.add_content(html_report.plotly_graph(
json_data[graph], graph))
analysis_plots_panel_group = html_report.panel_group(
[resolution_plots_panel, batch_plots_panel, misc_plots_panel])
div = html_report.div()
div.add_content(html_report.raw_html('<h2>Analysis plots</h2>'))
div.add_content(analysis_plots_panel_group)
report.add_content(div)
html = report.html()
import json
json_str = json.dumps(json_data)
with open('xia2-report.json', 'wb') as f:
print >> f, json_str
with open('xia2-report.html', 'wb') as f:
print >> f, html.encode('ascii', 'xmlcharrefreplace')
return
def test_ResolutionPlotsAndStats(iobs):
i_obs_anom = iobs.as_anomalous_array()
iobs_anom = i_obs_anom.map_to_asu().customized_copy(info=iobs.info())
n_bins = 2
result = dataset_statistics(
iobs, assert_is_not_unique_set_under_symmetry=False, n_bins=n_bins
)
anom_result = dataset_statistics(
iobs_anom,
assert_is_not_unique_set_under_symmetry=False,
anomalous=True,
n_bins=n_bins,
)
plotter = ResolutionPlotsAndStats(result, anom_result)
assert plotter.d_star_sq_ticktext == ["1.74", "1.53", "1.38", "1.27", "1.18"]
assert plotter.d_star_sq_tickvals == pytest.approx(
[
0.32984033277048164,
0.42706274943714834,
0.524285166103815,
0.6215075827704818,
0.7187299994371485,
],
1e-4,
)
tables = plotter.statistics_tables()
assert len(tables) == 2 # overall and per resolution
# test plots individually
d = plotter.cc_one_half_plot()
assert len(d["cc_one_half"]["data"]) == 6
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:4])
d["cc_one_half"]["data"][0]["x"] == [
0.5 * (uctbx.d_as_d_star_sq(b.d_max) + uctbx.d_as_d_star_sq(b.d_min))
for b in result.bins
]
d = plotter.i_over_sig_i_plot()
assert len(d["i_over_sig_i"]["data"]) == 1
assert len(d["i_over_sig_i"]["data"][0]["y"]) == n_bins
d = plotter.r_pim_plot()
assert len(d["r_pim"]["data"]) == 1
assert len(d["r_pim"]["data"][0]["y"]) == n_bins
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
# now try centric options and sigma tau for cc_one_half
plotter = ResolutionPlotsAndStats(result, anom_result, is_centric=True)
d = plotter.cc_one_half_plot(method="sigma_tau")
assert len(d["cc_one_half"]["data"]) == 6
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:2])
assert d["cc_one_half"]["data"][2] == {} # no anomalous plots
assert d["cc_one_half"]["data"][3] == {} # no anomalous plots
assert d["cc_one_half"]["data"][4] == {} # no cc_fit
assert d["cc_one_half"]["data"][5] == {} # no d_min
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
assert d["completeness"]["data"][1] == {}
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
assert d["multiplicity_vs_resolution"]["data"][1] == {}
plots = plotter.make_all_plots()
assert list(plots.keys()) == [
"cc_one_half",
"i_over_sig_i",
"completeness",
"multiplicity_vs_resolution",
"r_pim",
]
for plot in plots.values():
assert plot["layout"]["xaxis"]["ticktext"] == plotter.d_star_sq_ticktext
assert plot["layout"]["xaxis"]["tickvals"] == plotter.d_star_sq_tickvals
def return_scaler(self):
"""return scaler method"""
from dials.algorithms.scaling.scaler import MultiScalerBase
print_step_table(self)
if self._scaler.id_ == "single":
if self._parameters.apm_list[0].var_cov_matrix:
self._scaler.update_var_cov(self._parameters.apm_list[0])
self._scaler.experiment.scaling_model.set_scaling_model_as_scaled()
elif self._scaler.id_ == "multi" or self._scaler.id_ == "target":
if self._parameters.apm_list[0].var_cov_matrix: # test if has been set
for i, scaler in enumerate(self._scaler.active_scalers):
scaler.update_var_cov(self._parameters.apm_list[i])
scaler.experiment.scaling_model.set_scaling_model_as_scaled()
if not isinstance(self._scaler, MultiScalerBase):
self._scaler.experiment.scaling_model.normalise_components()
logger.debug("\n" + str(self._scaler.experiment.scaling_model))
if self._scaler.Ih_table.free_Ih_table:
i_obs = self._scaler.Ih_table.as_miller_array(
self._scaler.experiment.crystal.get_unit_cell(),
return_free_set_data=True,
)
res = merging_statistics.dataset_statistics(
i_obs=i_obs,
n_bins=20,
anomalous=False,
sigma_filtering=None,
use_internal_variance=False,
eliminate_sys_absent=False,
cc_one_half_method="sigma_tau",
)
free_rpim = res.overall.r_pim
free_cc12 = res.overall.cc_one_half
ccs = flex.double([b.cc_one_half for b in res.bins])
n_refl = flex.double([b.n_obs for b in res.bins])
n_tot = sum(n_refl)
free_wcc12 = sum(ccs * n_refl / n_tot)
i_obs = self._scaler.Ih_table.as_miller_array(
self._scaler.experiment.crystal.get_unit_cell()
)
res = merging_statistics.dataset_statistics(
i_obs=i_obs,
n_bins=20,
anomalous=False,
sigma_filtering=None,
use_internal_variance=False,
eliminate_sys_absent=False,
cc_one_half_method="sigma_tau",
)
work_rpim = res.overall.r_pim
work_cc12 = res.overall.cc_one_half
ccs = flex.double([b.cc_one_half for b in res.bins])
n_refl = flex.double([b.n_obs for b in res.bins])
n_tot = sum(n_refl)
work_wcc12 = sum(ccs * n_refl / n_tot)
rpim_gap = free_rpim - work_rpim
cc12_gap = work_cc12 - free_cc12
wcc12_gap = work_wcc12 - free_wcc12
self._scaler.final_rmsds = [
work_rpim,
free_rpim,
rpim_gap,
work_cc12,
free_cc12,
cc12_gap,
work_wcc12,
free_wcc12,
wcc12_gap,
]
header = ["", "Work", "Free", "Gap"]
rows = [
[
"Rpim",
str(round(work_rpim, 5)),
str(round(free_rpim, 5)),
str(round(rpim_gap, 5)),
],
[
"CC1/2",
str(round(work_cc12, 5)),
str(round(free_cc12, 5)),
str(round(cc12_gap, 5)),
],
[
"CC1/2 (weighted-avg)",
str(round(work_wcc12, 5)),
str(round(free_wcc12, 5)),
str(round(wcc12_gap, 5)),
],
]
logger.info(
"""\nWork/Free set quality indicators:
(CC1/2 calculated using the sigma-tau method, weighted-avg is the
average CC1/2 over resolution bins, weighted by n_obs per bin.)
Gaps are defined as Rfree-Rwork and CCWork-CCfree."""
)
st = simple_table(rows, header)
logger.info(st.format())
