def test_compute_delta_cchalf(dials_regression):
"""Test compute delta cchalf on an integrated mtz."""
filename = os.path.join(
dials_regression, "delta_cchalf_test_data", "test.XDS_ASCII.mtz"
)
params = phil_scope.extract()
params.nbins = 1
script = CCHalfFromMTZ(params, filename)
script.run()
cchalf_i = script.results_summary["per_dataset_delta_cc_half_values"][
"delta_cc_half_values"
]
assert abs(100 * script.results_summary["mean_cc_half"] - 94.582) < 1e-3
assert (
abs(100 * script.results_summary["mean_cc_half"] - 100 * cchalf_i[1] - 79.587)
< 1e-3
)
assert (
abs(100 * script.results_summary["mean_cc_half"] - 100 * cchalf_i[0] - 94.238)
< 1e-3
)
def test_exclusion_in_CCHalfFromDials():
"""Test the exclusion of image groups."""
# Same input as above, but mock DeltaCCHalf algorithm to just test
# interpretation of results and setting of excluded regions. With the
# input, test that outlier edges are correctly removed.
params = phil_scope.extract()
params.mode = "image_group"
expts = generated_exp(n=2)
refls = generated_refl()
def mock_algorithm(*_):
"""Mock a result from DeltaCCHalf"""
algo = mock.Mock()
algo.run.return_value = None
algo.results_summary = {
"per_dataset_delta_cc_half_values": {
"delta_cc_half_values": [-5.0, -2.0, 4.0, -5.0],
"datasets": [0, 1, 2, 3],
},
"dataset_removal": {
"cutoff_value": -1.0
},
}
return algo
with mock.patch(
"dials.algorithms.statistics.cc_half_algorithm.DeltaCCHalf",
side_effect=mock_algorithm,
):
script = CCHalfFromDials(params, expts, refls)
script.run()
assert script.datasetid_to_groups == {
"0": [],
"1": [2]
} # all but 3 removed
expts = script.experiments
assert list(expts.identifiers()) == ["1"]
assert expts[0].scan.get_valid_image_ranges(
expts.identifiers()[0]) == [(1, 10)]
assert script.results_summary["dataset_removal"][
"experiment_ids_fully_removed"] == [0]
assert script.results_summary["dataset_removal"][
"experiments_fully_removed"] == ["0"]
def test_setup_of_CCHalfFromDials():
"""Test the correct setup in image group mode.
Test for the case of outliers at the end of images, and image ranges not
equaling a multiple of the grouping."""
params = phil_scope.extract()
params.mode = "image_group"
expts = generated_exp(n=2)
refls = generated_refl()
# Expected behaviour is that the outliers will not be included in the
# image range, and that all groups will have at least 10 images in.
script = CCHalfFromDials(params, expts, refls)
assert script.group_to_datasetid_and_range == {
0: ("0", (5, 14)),
1: ("0", (15, 25)),
2: ("1", (1, 10)),
3: ("1", (11, 23)),
}
assert script.datasetid_to_groups == {"0": [0, 1], "1": [2, 3]}
def run(self):
"""Run cycles of scaling and filtering."""
with ScalingHTMLContextManager(self):
start_time = time.time()
results = AnalysisResults()
for counter in range(
1, self.params.filtering.deltacchalf.max_cycles + 1):
self.run_scaling_cycle()
if counter == 1:
results.initial_expids_and_image_ranges = [
(exp.identifier,
exp.scan.get_image_range()) if exp.scan else None
for exp in self.experiments
]
delta_cc_params = deltacc_phil_scope.extract()
delta_cc_params.mode = self.params.filtering.deltacchalf.mode
delta_cc_params.group_size = (
self.params.filtering.deltacchalf.group_size)
delta_cc_params.stdcutoff = self.params.filtering.deltacchalf.stdcutoff
logger.info("\nPerforming a round of filtering.\n")
# need to reduce to single table.
joined_reflections = flex.reflection_table()
for table in self.reflections:
joined_reflections.extend(table)
script = deltaccscript(delta_cc_params, self.experiments,
joined_reflections)
script.run()
valid_image_ranges = get_valid_image_ranges(self.experiments)
results.expids_and_image_ranges = [
(exp.identifier,
valid_image_ranges[i]) if exp.scan else None
for i, exp in enumerate(self.experiments)
]
self.experiments = script.experiments
self.params.dataset_selection.use_datasets = None
self.params.dataset_selection.exclude_datasets = None
results = log_cycle_results(results, self, script)
logger.info(
"Cycle %s of filtering, n_reflections removed this cycle: %s",
counter,
results.get_last_cycle_results()["n_removed"],
)
# Test termination conditions
latest_results = results.get_last_cycle_results()
if latest_results["n_removed"] == 0:
logger.info(
"Finishing scaling and filtering as no data removed in this cycle."
)
if self.params.scaling_options.full_matrix:
self.reflections = parse_multiple_datasets(
[script.filtered_reflection_table])
results = self._run_final_scale_cycle(results)
else:
self.reflections = [script.filtered_reflection_table]
results.finish(termination_reason="no_more_removed")
break
# Need to split reflections for further processing.
self.reflections = parse_multiple_datasets(
[script.filtered_reflection_table])
if (latest_results["cumul_percent_removed"] >
self.params.filtering.deltacchalf.max_percent_removed):
logger.info(
"Finishing scale and filtering as have now removed more than the limit."
)
results = self._run_final_scale_cycle(results)
results.finish(termination_reason="max_percent_removed")
break
if self.params.filtering.deltacchalf.min_completeness:
if (latest_results["merging_stats"]["completeness"] <
self.params.filtering.deltacchalf.min_completeness
):
logger.info(
"Finishing scaling and filtering as completeness now below cutoff."
)
results = self._run_final_scale_cycle(results)
results.finish(
termination_reason="below_completeness_limit")
break
if counter == self.params.filtering.deltacchalf.max_cycles:
logger.info("Finishing as reached max number of cycles.")
results = self._run_final_scale_cycle(results)
results.finish(termination_reason="max_cycles")
break
# If not finished then need to create new scaler to try again
self._create_model_and_scaler()
self.filtering_results = results
# Print summary of results
logger.info(results)
with open(self.params.filtering.output.scale_and_filter_results,
"w") as f:
json.dump(self.filtering_results.to_dict(), f, indent=2)
# All done!
logger.info("\nTotal time taken: %.4fs ", time.time() - start_time)
logger.info("%s%s%s", "\n", "=" * 80, "\n")
