def resolution_plots_and_stats(self):
self.merging_stats = merging_statistics.dataset_statistics(
self.intensities,
n_bins=self.params.resolution_bins,
cc_one_half_significance_level=self.params.cc_half_significance_level,
eliminate_sys_absent=self.params.eliminate_sys_absent,
use_internal_variance=self.params.use_internal_variance,
assert_is_not_unique_set_under_symmetry=False,
)
intensities_anom = self.intensities.as_anomalous_array()
intensities_anom = intensities_anom.map_to_asu().customized_copy(
info=self.intensities.info()
)
self.merging_stats_anom = merging_statistics.dataset_statistics(
intensities_anom,
n_bins=self.params.resolution_bins,
anomalous=True,
cc_one_half_significance_level=self.params.cc_half_significance_level,
eliminate_sys_absent=self.params.eliminate_sys_absent,
use_internal_variance=self.params.use_internal_variance,
assert_is_not_unique_set_under_symmetry=False,
)
is_centric = self.intensities.space_group().is_centric()
plotter = ResolutionPlotsAndStats(
self.merging_stats, self.merging_stats_anom, is_centric
)
d = OrderedDict()
d.update(plotter.make_all_plots(cc_one_half_method=self.params.cc_half_method))
overall_stats = plotter.overall_statistics_table(self.params.cc_half_method)
merging_stats = plotter.merging_statistics_table(self.params.cc_half_method)
return overall_stats, merging_stats, d
def make_merging_stats_plots(script):
"""Make merging stats plots for HTML report"""
d = {
"scaling_tables": ([], []),
"resolution_plots": {},
"batch_plots": {},
"misc_plots": {},
"anom_plots": {},
"image_range_tables": [],
}
(
batches,
rvb,
isigivb,
svb,
batch_data,
) = reflection_tables_to_batch_dependent_properties( # pylint: disable=unbalanced-tuple-unpacking
script.reflections,
script.experiments,
script.scaled_miller_array,
)
bm = batch_manager(batches, batch_data)
image_range_tables = make_image_range_table(script.experiments, bm)
if script.merging_statistics_result:
stats = script.merging_statistics_result
anom_stats = script.anom_merging_statistics_result
is_centric = script.scaled_miller_array.space_group().is_centric()
# Now calculate batch data
plotter = ResolutionPlotsAndStats(stats, anom_stats, is_centric)
d["resolution_plots"].update(plotter.make_all_plots())
d["scaling_tables"] = plotter.statistics_tables()
d["batch_plots"].update(scale_rmerge_vs_batch_plot(bm, rvb, svb))
d["batch_plots"].update(i_over_sig_i_vs_batch_plot(bm, isigivb))
plotter = IntensityStatisticsPlots(
script.scaled_miller_array, run_xtriage_analysis=False
)
d["resolution_plots"].update(plotter.generate_resolution_dependent_plots())
if d["resolution_plots"]["cc_one_half"]["data"][2]:
cc_anom = d["resolution_plots"]["cc_one_half"]["data"][2]["y"]
significance = d["resolution_plots"]["cc_one_half"]["data"][3]["y"]
sig = flex.double(cc_anom) > flex.double(significance)
max_anom = 0
for i, v in enumerate(sig):
if v:
max_anom = i
else:
break
d_min = uctbx.d_star_sq_as_d(plotter.binner.limits())[max_anom + 1]
else:
d_min = 0.0
d["misc_plots"].update(plotter.generate_miscellanous_plots())
intensities_anom = script.scaled_miller_array.as_anomalous_array()
intensities_anom = intensities_anom.map_to_asu().customized_copy(
info=script.scaled_miller_array.info()
)
anom_plotter = AnomalousPlotter(intensities_anom, strong_cutoff=d_min)
d["anom_plots"].update(anom_plotter.make_plots())
d["image_range_tables"] = [image_range_tables]
return d
def make_plots(self):
"""Generate tables of overall and resolution-binned merging statistics."""
d = {
"scaling_tables": ([], []),
"resolution_plots": OrderedDict(),
"batch_plots": OrderedDict(),
"misc_plots": OrderedDict(),
"anom_plots": OrderedDict(),
"image_range_tables": [],
}
if "statistics" in self.data:
plotter = ResolutionPlotsAndStats(
self.data["statistics"],
self.data["anomalous_statistics"],
is_centric=self.data["is_centric"],
)
d["resolution_plots"].update(plotter.make_all_plots())
d["scaling_tables"] = plotter.statistics_tables()
d["batch_plots"].update(
scale_rmerge_vs_batch_plot(
self.data["bm"],
self.data["r_merge_vs_batch"],
self.data["scale_vs_batch"],
))
d["batch_plots"].update(
i_over_sig_i_vs_batch_plot(self.data["bm"],
self.data["isigivsbatch"]))
plotter = IntensityStatisticsPlots(
self.data["scaled_miller_array"], run_xtriage_analysis=False)
d["resolution_plots"].update(
plotter.generate_resolution_dependent_plots())
if d["resolution_plots"]["cc_one_half"]["data"][2]:
cc_anom = d["resolution_plots"]["cc_one_half"]["data"][2]["y"]
significance = d["resolution_plots"]["cc_one_half"]["data"][3][
"y"]
sig = flex.double(cc_anom) > flex.double(significance)
max_anom = 0
for i, v in enumerate(sig):
if v:
max_anom = i
else:
break
d_min = uctbx.d_star_sq_as_d(
plotter.binner.limits())[max_anom + 1]
else:
d_min = 0.0
d["misc_plots"].update(plotter.generate_miscellanous_plots())
intensities_anom = self.data[
"scaled_miller_array"].as_anomalous_array()
intensities_anom = intensities_anom.map_to_asu().customized_copy(
info=self.data["scaled_miller_array"].info())
anom_plotter = AnomalousPlotter(intensities_anom,
strong_cutoff=d_min)
d["anom_plots"].update(anom_plotter.make_plots())
d["image_range_tables"] = self.data["image_range_tables"]
return d
def test_ResolutionPlotsAndStats(iobs):
i_obs_anom = iobs.as_anomalous_array()
iobs_anom = i_obs_anom.map_to_asu().customized_copy(info=iobs.info())
n_bins = 2
result = dataset_statistics(iobs,
assert_is_not_unique_set_under_symmetry=False,
n_bins=n_bins)
anom_result = dataset_statistics(
iobs_anom,
assert_is_not_unique_set_under_symmetry=False,
anomalous=True,
n_bins=n_bins,
)
plotter = ResolutionPlotsAndStats(result, anom_result)
assert plotter.d_star_sq_ticktext == [
"1.26", "1.19", "1.13", "1.08", "1.04"
]
assert plotter.d_star_sq_tickvals == pytest.approx(
[0.6319, 0.7055, 0.7792, 0.8528, 0.9264], 1e-4)
tables = plotter.statistics_tables()
assert len(tables) == 2 # overall and per resolution
# test plots individually
d = plotter.cc_one_half_plot()
assert len(d["cc_one_half"]["data"]) == 4
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"])
d = plotter.i_over_sig_i_plot()
assert len(d["i_over_sig_i"]["data"]) == 1
assert len(d["i_over_sig_i"]["data"][0]["y"]) == n_bins
d = plotter.r_pim_plot()
assert len(d["r_pim"]["data"]) == 1
assert len(d["r_pim"]["data"][0]["y"]) == n_bins
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
# now try centric options and sigma tau for cc_one_half
plotter = ResolutionPlotsAndStats(result, anom_result, is_centric=True)
d = plotter.cc_one_half_plot(method="sigma_tau")
assert len(d["cc_one_half"]["data"]) == 4
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:2])
assert d["cc_one_half"]["data"][2] == {} # no anomalous plots
assert d["cc_one_half"]["data"][3] == {} # no anomalous plots
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
assert d["completeness"]["data"][1] == {}
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
assert d["multiplicity_vs_resolution"]["data"][1] == {}
plots = plotter.make_all_plots()
assert list(plots.keys()) == [
"cc_one_half",
"i_over_sig_i",
"completeness",
"multiplicity_vs_resolution",
"r_pim",
]
for plot in plots.values():
assert plot["layout"]["xaxis"][
"ticktext"] == plotter.d_star_sq_ticktext
assert plot["layout"]["xaxis"][
"tickvals"] == plotter.d_star_sq_tickvals
filenames = (
"/Users/whi10850/Documents/test_data/multi_example/integrated_files/inc_test/1/",
"/Users/whi10850/Documents/test_data/multi_example/integrated_files/inc_test/2/",
)
cc_one_half_data = []
i_over_sig_data = []
for filepath in filenames:
experiments = load.experiment_list(filepath + "scaled_experiments.json",
check_format=False)
reflections = flex.reflection_table.from_pickle(filepath + "scaled.pickle")
print("memory: %s" %
int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
arr = scaled_data_as_miller_array([reflections], experiments)
norm_stats, anom_stats = merging_stats_from_scaled_array(arr)
plotter = ResolutionPlotsAndStats(norm_stats, anom_stats)
resolution_plots = plotter.make_all_plots()
cc_one_half_data.append(resolution_plots["cc_one_half"])
i_over_sig_data.append(resolution_plots["i_over_sig_i"])
del experiments
del reflections
gc.collect()
print("added data for %s" % filepath)
data = {"cc_one_half": cc_one_half_data, "i_over_sigma": i_over_sig_data}
with open("incremental_data.json", "w") as f:
json.dump(data, f, indent=True)
def test_ResolutionPlotsAndStats(iobs):
i_obs_anom = iobs.as_anomalous_array()
iobs_anom = i_obs_anom.map_to_asu().customized_copy(info=iobs.info())
n_bins = 2
result = dataset_statistics(
iobs, assert_is_not_unique_set_under_symmetry=False, n_bins=n_bins
)
anom_result = dataset_statistics(
iobs_anom,
assert_is_not_unique_set_under_symmetry=False,
anomalous=True,
n_bins=n_bins,
)
plotter = ResolutionPlotsAndStats(result, anom_result)
assert plotter.d_star_sq_ticktext == ["1.74", "1.53", "1.38", "1.27", "1.18"]
assert plotter.d_star_sq_tickvals == pytest.approx(
[
0.32984033277048164,
0.42706274943714834,
0.524285166103815,
0.6215075827704818,
0.7187299994371485,
],
1e-4,
)
tables = plotter.statistics_tables()
assert len(tables) == 2 # overall and per resolution
# test plots individually
d = plotter.cc_one_half_plot()
assert len(d["cc_one_half"]["data"]) == 6
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:4])
d["cc_one_half"]["data"][0]["x"] == [
0.5 * (uctbx.d_as_d_star_sq(b.d_max) + uctbx.d_as_d_star_sq(b.d_min))
for b in result.bins
]
d = plotter.i_over_sig_i_plot()
assert len(d["i_over_sig_i"]["data"]) == 1
assert len(d["i_over_sig_i"]["data"][0]["y"]) == n_bins
d = plotter.r_pim_plot()
assert len(d["r_pim"]["data"]) == 1
assert len(d["r_pim"]["data"][0]["y"]) == n_bins
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
# now try centric options and sigma tau for cc_one_half
plotter = ResolutionPlotsAndStats(result, anom_result, is_centric=True)
d = plotter.cc_one_half_plot(method="sigma_tau")
assert len(d["cc_one_half"]["data"]) == 6
assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:2])
assert d["cc_one_half"]["data"][2] == {} # no anomalous plots
assert d["cc_one_half"]["data"][3] == {} # no anomalous plots
assert d["cc_one_half"]["data"][4] == {} # no cc_fit
assert d["cc_one_half"]["data"][5] == {} # no d_min
d = plotter.completeness_plot()
assert len(d["completeness"]["data"]) == 2
assert len(d["completeness"]["data"][0]["y"]) == n_bins
assert d["completeness"]["data"][1] == {}
d = plotter.multiplicity_vs_resolution_plot()
assert len(d["multiplicity_vs_resolution"]["data"]) == 2
assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins
assert d["multiplicity_vs_resolution"]["data"][1] == {}
plots = plotter.make_all_plots()
assert list(plots.keys()) == [
"cc_one_half",
"i_over_sig_i",
"completeness",
"multiplicity_vs_resolution",
"r_pim",
]
for plot in plots.values():
assert plot["layout"]["xaxis"]["ticktext"] == plotter.d_star_sq_ticktext
assert plot["layout"]["xaxis"]["tickvals"] == plotter.d_star_sq_tickvals