def run(self, args=None):
""" Perform the integration. """
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from dials.util import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(args=args,
show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry("more than 1 reflection file was given")
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry("no experiment list was specified")
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
if __name__ == "__main__":
# Configure logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
for abs_params in params.absorption_correction:
if abs_params.apply:
if not (params.integration.debug.output
and not params.integration.debug.separate_files):
raise Sorry(
"Shoeboxes must be saved to integration intermediates to apply an absorption correction. "
+
"Set integration.debug.output=True, integration.debug.separate_files=False and "
+
"integration.debug.delete_shoeboxes=True to temporarily store shoeboxes."
)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info("Using external mask: %s" % mask.filename)
for tile in mask.data:
logger.info(" Mask has %d pixels masked" %
tile.data().count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.info("=" * 80)
logger.info("")
logger.info("Experiments")
logger.info("")
logger.info("Models for experiment %d" % i)
logger.info("")
logger.info(str(exp.beam))
logger.info(str(exp.detector))
if exp.goniometer:
logger.info(str(exp.goniometer))
if exp.scan:
logger.info(str(exp.scan))
logger.info(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
# Check pixels don't belong to neighbours
if reference is not None:
if exp.goniometer is not None and exp.scan is not None:
self.filter_reference_pixels(reference, experiments)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments, reference, params.scan_range)
# Modify experiment list if exclude images is set
experiments = self.exclude_images(experiments, params.exclude_images)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding,
)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise Sorry("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif len(unmatched) != 0:
logger.info("")
logger.info("*" * 80)
logger.info(
"Warning: %d reference spots were not matched to predictions"
% (len(unmatched)))
logger.info("*" * 80)
logger.info("")
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src["id"] == eid)
tmp["id"] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference["id"] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions"
% expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and reference is not None
and "shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments,
reference)
else:
experiments = ProfileModelFactory.create(params, experiments)
for expr in experiments:
if expr.profile is None:
raise Sorry("No profile information in experiment list")
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Correct integrated intensities for absorption correction, if necessary
for abs_params in params.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction, )
experiments, reflections = multi_kapton_correction(
experiments,
reflections,
abs_params.fuller_kapton,
logger=logger)()
if params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
from dxtbx.model.experiment_list import ExperimentList
sig_filter = SignificanceFilter(params)
filtered_refls = sig_filter(experiments, reflections)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
logger.info(
"Removed %d reflections out of %d when applying significance filter"
% (len(reflections) - len(filtered_refls), len(reflections)))
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
logger.info(
"Removed experiment %d which has no reflections left after applying significance filter"
% expt_id)
if len(accepted_refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
experiments = accepted_expts
reflections = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if params.integration.debug.delete_shoeboxes and "shoebox" in reflections:
del reflections["shoebox"]
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))
return experiments, reflections
def run_with_preparsed(self, params, options):
"""Run combine_experiments, but allow passing in of parameters"""
# Try to load the models and data
if not params.input.experiments:
print("No Experiments found in the input")
self.parser.print_help()
return
if not params.input.reflections:
print("No reflection data found in the input")
self.parser.print_help()
return
if len(params.input.reflections) != len(params.input.experiments):
sys.exit(
"The number of input reflections files does not match the "
"number of input experiments")
flat_exps = flatten_experiments(params.input.experiments)
ref_beam = params.reference_from_experiment.beam
ref_goniometer = params.reference_from_experiment.goniometer
ref_scan = params.reference_from_experiment.scan
ref_crystal = params.reference_from_experiment.crystal
ref_detector = params.reference_from_experiment.detector
if ref_beam is not None:
try:
ref_beam = flat_exps[ref_beam].beam
except IndexError:
sys.exit(f"{ref_beam} is not a valid experiment ID")
if ref_goniometer is not None:
try:
ref_goniometer = flat_exps[ref_goniometer].goniometer
except IndexError:
sys.exit(f"{ref_goniometer} is not a valid experiment ID")
if ref_scan is not None:
try:
ref_scan = flat_exps[ref_scan].scan
except IndexError:
sys.exit(f"{ref_scan} is not a valid experiment ID")
if ref_crystal is not None:
try:
ref_crystal = flat_exps[ref_crystal].crystal
except IndexError:
sys.exit(f"{ref_crystal} is not a valid experiment ID")
if ref_detector is not None:
assert not params.reference_from_experiment.average_detector
try:
ref_detector = flat_exps[ref_detector].detector
except IndexError:
sys.exit(f"{ref_detector} is not a valid experiment ID")
elif params.reference_from_experiment.average_detector:
# Average all of the detectors together
def average_detectors(target, panelgroups, depth):
# Recursive function to do the averaging
if (params.reference_from_experiment.average_hierarchy_level is
None or depth == params.reference_from_experiment.
average_hierarchy_level):
n = len(panelgroups)
sum_fast = matrix.col((0.0, 0.0, 0.0))
sum_slow = matrix.col((0.0, 0.0, 0.0))
sum_ori = matrix.col((0.0, 0.0, 0.0))
# Average the d matrix vectors
for pg in panelgroups:
sum_fast += matrix.col(pg.get_local_fast_axis())
sum_slow += matrix.col(pg.get_local_slow_axis())
sum_ori += matrix.col(pg.get_local_origin())
sum_fast /= n
sum_slow /= n
sum_ori /= n
# Re-orthagonalize the slow and the fast vectors by rotating around the cross product
c = sum_fast.cross(sum_slow)
a = sum_fast.angle(sum_slow, deg=True) / 2
sum_fast = sum_fast.rotate_around_origin(c,
a - 45,
deg=True)
sum_slow = sum_slow.rotate_around_origin(c,
-(a - 45),
deg=True)
target.set_local_frame(sum_fast, sum_slow, sum_ori)
if target.is_group():
# Recurse
for i, target_pg in enumerate(target):
average_detectors(target_pg,
[pg[i] for pg in panelgroups],
depth + 1)
ref_detector = flat_exps[0].detector
average_detectors(ref_detector.hierarchy(),
[e.detector.hierarchy() for e in flat_exps], 0)
combine = CombineWithReference(
beam=ref_beam,
goniometer=ref_goniometer,
scan=ref_scan,
crystal=ref_crystal,
detector=ref_detector,
params=params,
)
# set up global experiments and reflections lists
reflections = flex.reflection_table()
global_id = 0
skipped_expts_min_refl = 0
skipped_expts_max_refl = 0
experiments = ExperimentList()
# loop through the input, building up the global lists
nrefs_per_exp = []
for ref_wrapper, exp_wrapper in zip(params.input.reflections,
params.input.experiments):
refs = ref_wrapper.data
exps = exp_wrapper.data
# Record initial mapping of ids for updating later.
ids_map = dict(refs.experiment_identifiers())
# Keep track of mapping of imageset_ids old->new within this experimentlist
imageset_result_map = {}
for k in refs.experiment_identifiers().keys():
del refs.experiment_identifiers()[k]
for i, exp in enumerate(exps):
sel = refs["id"] == i
sub_ref = refs.select(sel)
n_sub_ref = len(sub_ref)
if (params.output.min_reflections_per_experiment is not None
and n_sub_ref <
params.output.min_reflections_per_experiment):
skipped_expts_min_refl += 1
continue
if (params.output.max_reflections_per_experiment is not None
and n_sub_ref >
params.output.max_reflections_per_experiment):
skipped_expts_max_refl += 1
continue
nrefs_per_exp.append(n_sub_ref)
sub_ref["id"] = flex.int(len(sub_ref), global_id)
# now update identifiers if set.
if i in ids_map:
sub_ref.experiment_identifiers()[global_id] = ids_map[i]
if params.output.delete_shoeboxes and "shoebox" in sub_ref:
del sub_ref["shoebox"]
try:
experiments.append(combine(exp))
except ComparisonError as e:
# When we failed tolerance checks, give a useful error message
(path,
index) = find_experiment_in(exp, params.input.experiments)
sys.exit(
"Model didn't match reference within required tolerance for experiment {} in {}:"
"\n{}\nAdjust tolerances or set compare_models=False to ignore differences."
.format(index, path, str(e)))
# Rewrite imageset_id, if the experiment has and imageset
if exp.imageset and "imageset_id" in sub_ref:
# Get the index of the imageset for this experiment and record how it changed
new_imageset_id = experiments.imagesets().index(
experiments[-1].imageset)
old_imageset_id = exps.imagesets().index(exp.imageset)
imageset_result_map[old_imageset_id] = new_imageset_id
# Check for invalid(?) imageset_id indices... and leave if they are wrong
if len(set(sub_ref["imageset_id"])) != 1:
logger.warning(
"Warning: Experiment %d reflections appear to have come from multiple imagesets - output may be incorrect",
i,
)
else:
sub_ref["imageset_id"] = flex.int(
len(sub_ref), new_imageset_id)
reflections.extend(sub_ref)
global_id += 1
# Include unindexed reflections, if we can safely remap their imagesets
if "imageset_id" in reflections:
unindexed_refs = refs.select(refs["id"] == -1)
for old_id in set(unindexed_refs["imageset_id"]):
subs = unindexed_refs.select(
unindexed_refs["imageset_id"] == old_id)
subs["imageset_id"] = flex.int(len(subs),
imageset_result_map[old_id])
reflections.extend(subs)
if (params.output.min_reflections_per_experiment is not None
and skipped_expts_min_refl > 0):
print(
"Removed {} experiments with fewer than {} reflections".format(
skipped_expts_min_refl,
params.output.min_reflections_per_experiment))
if (params.output.max_reflections_per_experiment is not None
and skipped_expts_max_refl > 0):
print(
"Removed {} experiments with more than {} reflections".format(
skipped_expts_max_refl,
params.output.max_reflections_per_experiment))
# print number of reflections per experiment
header = ["Experiment", "Number of reflections"]
rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_exp)]
print(tabulate(rows, header))
# save a random subset if requested
if (params.output.n_subset is not None
and len(experiments) > params.output.n_subset):
subset_exp = ExperimentList()
subset_refls = flex.reflection_table()
if params.output.n_subset_method == "random":
n_picked = 0
indices = list(range(len(experiments)))
if reflections.experiment_identifiers().keys():
indices_to_sel = []
while n_picked < params.output.n_subset:
idx = indices.pop(random.randint(0, len(indices) - 1))
indices_to_sel.append(idx)
n_picked += 1
# make sure select in order.
for idx in sorted(indices_to_sel):
subset_exp.append(experiments[idx])
subset_refls = reflections.select(subset_exp)
subset_refls.reset_ids()
else:
while n_picked < params.output.n_subset:
idx = indices.pop(random.randint(0, len(indices) - 1))
subset_exp.append(experiments[idx])
refls = reflections.select(reflections["id"] == idx)
refls["id"] = flex.int(len(refls), n_picked)
subset_refls.extend(refls)
n_picked += 1
print(
"Selecting a random subset of {} experiments out of {} total."
.format(params.output.n_subset, len(experiments)))
elif params.output.n_subset_method == "n_refl":
if params.output.n_refl_panel_list is None:
refls_subset = reflections
else:
sel = flex.bool(len(reflections), False)
for p in params.output.n_refl_panel_list:
sel |= reflections["panel"] == p
refls_subset = reflections.select(sel)
refl_counts = flex.int()
for expt_id in range(len(experiments)):
refl_counts.append(
(refls_subset["id"] == expt_id).count(True))
sort_order = flex.sort_permutation(refl_counts, reverse=True)
if reflections.experiment_identifiers().keys():
for idx in sorted(sort_order[:params.output.n_subset]):
subset_exp.append(experiments[idx])
subset_refls = reflections.select(subset_exp)
subset_refls.reset_ids()
else:
for expt_id, idx in enumerate(
sort_order[:params.output.n_subset]):
subset_exp.append(experiments[idx])
refls = reflections.select(reflections["id"] == idx)
refls["id"] = flex.int(len(refls), expt_id)
subset_refls.extend(refls)
print(
"Selecting a subset of {} experiments with highest number of reflections out of {} total."
.format(params.output.n_subset, len(experiments)))
elif params.output.n_subset_method == "significance_filter":
params.output.significance_filter.enable = True
sig_filter = SignificanceFilter(params.output)
refls_subset = sig_filter(experiments, reflections)
refl_counts = flex.int()
for expt_id in range(len(experiments)):
refl_counts.append(
(refls_subset["id"] == expt_id).count(True))
sort_order = flex.sort_permutation(refl_counts, reverse=True)
if reflections.experiment_identifiers().keys():
for idx in sorted(sort_order[:params.output.n_subset]):
subset_exp.append(experiments[idx])
subset_refls = reflections.select(subset_exp)
subset_refls.reset_ids()
else:
for expt_id, idx in enumerate(
sort_order[:params.output.n_subset]):
subset_exp.append(experiments[idx])
refls = reflections.select(reflections["id"] == idx)
refls["id"] = flex.int(len(refls), expt_id)
subset_refls.extend(refls)
experiments = subset_exp
reflections = subset_refls
def save_in_batches(experiments,
reflections,
exp_name,
refl_name,
batch_size=1000):
for i, indices in enumerate(
splitit(list(range(len(experiments))),
(len(experiments) // batch_size) + 1)):
batch_expts = ExperimentList()
batch_refls = flex.reflection_table()
if reflections.experiment_identifiers().keys():
for sub_idx in indices:
batch_expts.append(experiments[sub_idx])
batch_refls = reflections.select(batch_expts)
batch_refls.reset_ids()
else:
for sub_id, sub_idx in enumerate(indices):
batch_expts.append(experiments[sub_idx])
sub_refls = reflections.select(
reflections["id"] == sub_idx)
sub_refls["id"] = flex.int(len(sub_refls), sub_id)
batch_refls.extend(sub_refls)
exp_filename = os.path.splitext(exp_name)[0] + "_%03d.expt" % i
ref_filename = os.path.splitext(
refl_name)[0] + "_%03d.refl" % i
self._save_output(batch_expts, batch_refls, exp_filename,
ref_filename)
def combine_in_clusters(experiments_l, reflections_l, exp_name,
refl_name, end_count):
result = []
for cluster, experiment in enumerate(experiments_l):
cluster_expts = ExperimentList()
cluster_refls = flex.reflection_table()
for i, expts in enumerate(experiment):
refls = reflections_l[cluster][i]
if refls.experiment_identifiers().keys():
identifier = refls.experiment_identifiers().values()[0]
id_val = refls.experiment_identifiers().keys()[0]
del refls.experiment_identifiers()[id_val]
refls["id"] = flex.int(len(refls), i)
refls.experiment_identifiers()[i] = identifier
refls.assert_experiment_identifiers_are_consistent(
experiment[i:i + 1])
else:
refls["id"] = flex.int(len(refls), i)
cluster_expts.append(expts)
cluster_refls.extend(refls)
exp_filename = os.path.splitext(exp_name)[0] + (
"_cluster%d.expt" % (end_count - cluster))
ref_filename = os.path.splitext(refl_name)[0] + (
"_cluster%d.refl" % (end_count - cluster))
result.append(
(cluster_expts, cluster_refls, exp_filename, ref_filename))
return result
# cluster the resulting experiments if requested
if params.clustering.use:
clustered = Cluster(
experiments,
reflections,
dendrogram=params.clustering.dendrogram,
threshold=params.clustering.threshold,
n_max=params.clustering.max_crystals,
)
n_clusters = len(clustered.clustered_frames)
def not_too_many(keeps):
if params.clustering.max_clusters is not None:
return len(keeps) < params.clustering.max_clusters
return True
keep_frames = []
sorted_keys = sorted(clustered.clustered_frames.keys())
while len(clustered.clustered_frames) > 0 and not_too_many(
keep_frames):
keep_frames.append(
clustered.clustered_frames.pop(sorted_keys.pop(-1)))
if params.clustering.exclude_single_crystal_clusters:
keep_frames = [k for k in keep_frames if len(k) > 1]
clustered_experiments = [
ExperimentList([f.experiment for f in frame_cluster])
for frame_cluster in keep_frames
]
clustered_reflections = [[f.reflections for f in frame_cluster]
for frame_cluster in keep_frames]
list_of_combined = combine_in_clusters(
clustered_experiments,
clustered_reflections,
params.output.experiments_filename,
params.output.reflections_filename,
n_clusters,
)
for saveable_tuple in list_of_combined:
if params.output.max_batch_size is None:
self._save_output(*saveable_tuple)
else:
save_in_batches(*saveable_tuple,
batch_size=params.output.max_batch_size)
else:
if params.output.max_batch_size is None:
self._save_output(
experiments,
reflections,
params.output.experiments_filename,
params.output.reflections_filename,
)
else:
save_in_batches(
experiments,
reflections,
params.output.experiments_filename,
params.output.reflections_filename,
batch_size=params.output.max_batch_size,
)
return
def run_integration(params, experiments, reference=None):
"""Perform the integration.
Returns:
experiments: The integrated experiments
reflections: The integrated reflections
report(optional): An integration report.
Raises:
ValueError: For a number of bad inputs
RuntimeError: If the profile model creation fails
"""
predicted = None
rubbish = None
for abs_params in params.absorption_correction:
if abs_params.apply:
if not (params.integration.debug.output
and not params.integration.debug.separate_files):
raise ValueError(
"Shoeboxes must be saved to integration intermediates to apply an absorption correction. "
+
"Set integration.debug.output=True, integration.debug.separate_files=False and "
+
"integration.debug.delete_shoeboxes=True to temporarily store shoeboxes."
)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info("Using external mask: %s", mask.filename)
for tile in mask.data:
logger.info(" Mask has %d pixels masked",
tile.data().count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
summary = "\n".join((
"",
"=" * 80,
"",
"Experiments",
"",
"Models for experiment %d" % i,
"",
str(exp.beam),
str(exp.detector),
))
if exp.goniometer:
summary += str(exp.goniometer) + "\n"
if exp.scan:
summary += str(exp.scan) + "\n"
summary += str(exp.crystal)
logger.info(summary)
logger.info("\n".join(("", "=" * 80, "")))
logger.info(heading("Initialising"))
# Load the data
if reference:
reference, rubbish = process_reference(reference)
# Check pixels don't belong to neighbours
if exp.goniometer is not None and exp.scan is not None:
reference = filter_reference_pixels(reference, experiments)
# Modify experiment list if scan range is set.
experiments, reference = split_for_scan_range(experiments, reference,
params.scan_range)
# Modify experiment list if exclude images is set
if params.exclude_images:
for experiment in experiments:
for index in params.exclude_images:
experiment.imageset.mark_for_rejection(index, True)
# Predict the reflections
logger.info("\n".join(("", "=" * 80, "")))
logger.info(heading("Predicting reflections"))
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding,
)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise ValueError("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif unmatched:
msg = (
"Warning: %d reference spots were not matched to predictions" %
unmatched.size())
border = "\n".join(("", "*" * 80, ""))
logger.info("".join((border, msg, border)))
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
old_id = eid
new_id = good_expt_count
tmp = src.select(src["id"] == old_id)
tmp["id"] = flex.int(len(tmp), good_expt_count)
if old_id in tmp.experiment_identifiers():
identifier = tmp.experiment_identifiers()[old_id]
del tmp.experiment_identifiers()[old_id]
tmp.experiment_identifiers()[new_id] = identifier
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference["id"] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions",
expt_id,
)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = sample_predictions(experiments, predicted, params)
# Compute the profile model - either load existing or compute
# can raise RuntimeError
experiments = ProfileModelFactory.create(params, experiments, reference)
for expr in experiments:
if expr.profile is None:
raise ValueError("No profile information in experiment list")
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = create_integrator(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Remove unintegrated reflections
if not params.output.output_unintegrated_reflections:
keep = reflections.get_flags(reflections.flags.integrated, all=False)
logger.info(
"Removing %d unintegrated reflections of %d total",
keep.count(False),
keep.size(),
)
reflections = reflections.select(keep)
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Correct integrated intensities for absorption correction, if necessary
for abs_params in params.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction, )
experiments, reflections = multi_kapton_correction(
experiments,
reflections,
abs_params.fuller_kapton,
logger=logger)()
if params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
sig_filter = SignificanceFilter(params)
filtered_refls = sig_filter(experiments, reflections)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
logger.info(
"Removed %d reflections out of %d when applying significance filter",
(reflections.size() - filtered_refls.size()),
reflections.size(),
)
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if refls:
accepted_expts.append(expt)
current_id = expt_id
new_id = len(accepted_expts) - 1
refls["id"] = flex.int(len(refls), new_id)
if expt.identifier:
del refls.experiment_identifiers()[current_id]
refls.experiment_identifiers()[new_id] = expt.identifier
accepted_refls.extend(refls)
else:
logger.info(
"Removed experiment %d which has no reflections left after applying significance filter",
expt_id,
)
if not accepted_refls:
raise ValueError(
"No reflections left after applying significance filter")
experiments = accepted_expts
reflections = accepted_refls
# Write a report if requested
report = None
if params.output.report is not None:
report = integrator.report()
return experiments, reflections, report
def run_with_preparsed(self, params, options):
"""Run combine_experiments, but allow passing in of parameters"""
from dials.util.options import flatten_experiments
# Try to load the models and data
if len(params.input.experiments) == 0:
print("No Experiments found in the input")
self.parser.print_help()
return
if len(params.input.reflections) == 0:
print("No reflection data found in the input")
self.parser.print_help()
return
try:
assert len(params.input.reflections) == len(
params.input.experiments)
except AssertionError:
raise Sorry(
"The number of input reflections files does not match the "
"number of input experiments")
flat_exps = flatten_experiments(params.input.experiments)
ref_beam = params.reference_from_experiment.beam
ref_goniometer = params.reference_from_experiment.goniometer
ref_scan = params.reference_from_experiment.scan
ref_crystal = params.reference_from_experiment.crystal
ref_detector = params.reference_from_experiment.detector
if ref_beam is not None:
try:
ref_beam = flat_exps[ref_beam].beam
except IndexError:
raise Sorry("{} is not a valid experiment ID".format(ref_beam))
if ref_goniometer is not None:
try:
ref_goniometer = flat_exps[ref_goniometer].goniometer
except IndexError:
raise Sorry(
"{} is not a valid experiment ID".format(ref_goniometer))
if ref_scan is not None:
try:
ref_scan = flat_exps[ref_scan].scan
except IndexError:
raise Sorry("{} is not a valid experiment ID".format(ref_scan))
if ref_crystal is not None:
try:
ref_crystal = flat_exps[ref_crystal].crystal
except IndexError:
raise Sorry(
"{} is not a valid experiment ID".format(ref_crystal))
if ref_detector is not None:
assert not params.reference_from_experiment.average_detector
try:
ref_detector = flat_exps[ref_detector].detector
except IndexError:
raise Sorry(
"{} is not a valid experiment ID".format(ref_detector))
elif params.reference_from_experiment.average_detector:
# Average all of the detectors together
from scitbx.matrix import col
def average_detectors(target, panelgroups, depth):
# Recursive function to do the averaging
if (params.reference_from_experiment.average_hierarchy_level is
None or depth == params.reference_from_experiment.
average_hierarchy_level):
n = len(panelgroups)
sum_fast = col((0.0, 0.0, 0.0))
sum_slow = col((0.0, 0.0, 0.0))
sum_ori = col((0.0, 0.0, 0.0))
# Average the d matrix vectors
for pg in panelgroups:
sum_fast += col(pg.get_local_fast_axis())
sum_slow += col(pg.get_local_slow_axis())
sum_ori += col(pg.get_local_origin())
sum_fast /= n
sum_slow /= n
sum_ori /= n
# Re-orthagonalize the slow and the fast vectors by rotating around the cross product
c = sum_fast.cross(sum_slow)
a = sum_fast.angle(sum_slow, deg=True) / 2
sum_fast = sum_fast.rotate(c, a - 45, deg=True)
sum_slow = sum_slow.rotate(c, -(a - 45), deg=True)
target.set_local_frame(sum_fast, sum_slow, sum_ori)
if target.is_group():
# Recurse
for i, target_pg in enumerate(target):
average_detectors(target_pg,
[pg[i] for pg in panelgroups],
depth + 1)
ref_detector = flat_exps[0].detector
average_detectors(ref_detector.hierarchy(),
[e.detector.hierarchy() for e in flat_exps], 0)
combine = CombineWithReference(
beam=ref_beam,
goniometer=ref_goniometer,
scan=ref_scan,
crystal=ref_crystal,
detector=ref_detector,
params=params,
)
# set up global experiments and reflections lists
from dials.array_family import flex
reflections = flex.reflection_table()
global_id = 0
skipped_expts = 0
from dxtbx.model.experiment_list import ExperimentList
experiments = ExperimentList()
# loop through the input, building up the global lists
nrefs_per_exp = []
for ref_wrapper, exp_wrapper in zip(params.input.reflections,
params.input.experiments):
refs = ref_wrapper.data
exps = exp_wrapper.data
for i, exp in enumerate(exps):
sel = refs["id"] == i
sub_ref = refs.select(sel)
n_sub_ref = len(sub_ref)
if (params.output.min_reflections_per_experiment is not None
and n_sub_ref <
params.output.min_reflections_per_experiment):
skipped_expts += 1
continue
nrefs_per_exp.append(n_sub_ref)
sub_ref["id"] = flex.int(len(sub_ref), global_id)
if params.output.delete_shoeboxes and "shoebox" in sub_ref:
del sub_ref["shoebox"]
reflections.extend(sub_ref)
try:
experiments.append(combine(exp))
except ComparisonError as e:
# When we failed tolerance checks, give a useful error message
(path,
index) = find_experiment_in(exp, params.input.experiments)
raise Sorry(
"Model didn't match reference within required tolerance for experiment {} in {}:"
"\n{}\nAdjust tolerances or set compare_models=False to ignore differences."
.format(index, path, str(e)))
global_id += 1
if (params.output.min_reflections_per_experiment is not None
and skipped_expts > 0):
print("Removed {0} experiments with fewer than {1} reflections".
format(skipped_expts,
params.output.min_reflections_per_experiment))
# print number of reflections per experiment
from libtbx.table_utils import simple_table
header = ["Experiment", "Number of reflections"]
rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_exp)]
st = simple_table(rows, header)
print(st.format())
# save a random subset if requested
if (params.output.n_subset is not None
and len(experiments) > params.output.n_subset):
subset_exp = ExperimentList()
subset_refls = flex.reflection_table()
if params.output.n_subset_method == "random":
n_picked = 0
indices = list(range(len(experiments)))
while n_picked < params.output.n_subset:
idx = indices.pop(random.randint(0, len(indices) - 1))
subset_exp.append(experiments[idx])
refls = reflections.select(reflections["id"] == idx)
refls["id"] = flex.int(len(refls), n_picked)
subset_refls.extend(refls)
n_picked += 1
print(
"Selecting a random subset of {0} experiments out of {1} total."
.format(params.output.n_subset, len(experiments)))
elif params.output.n_subset_method == "n_refl":
if params.output.n_refl_panel_list is None:
refls_subset = reflections
else:
sel = flex.bool(len(reflections), False)
for p in params.output.n_refl_panel_list:
sel |= reflections["panel"] == p
refls_subset = reflections.select(sel)
refl_counts = flex.int()
for expt_id in range(len(experiments)):
refl_counts.append(
len(refls_subset.select(
refls_subset["id"] == expt_id)))
sort_order = flex.sort_permutation(refl_counts, reverse=True)
for expt_id, idx in enumerate(
sort_order[:params.output.n_subset]):
subset_exp.append(experiments[idx])
refls = reflections.select(reflections["id"] == idx)
refls["id"] = flex.int(len(refls), expt_id)
subset_refls.extend(refls)
print(
"Selecting a subset of {0} experiments with highest number of reflections out of {1} total."
.format(params.output.n_subset, len(experiments)))
elif params.output.n_subset_method == "significance_filter":
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
params.output.significance_filter.enable = True
sig_filter = SignificanceFilter(params.output)
refls_subset = sig_filter(experiments, reflections)
refl_counts = flex.int()
for expt_id in range(len(experiments)):
refl_counts.append(
len(refls_subset.select(
refls_subset["id"] == expt_id)))
sort_order = flex.sort_permutation(refl_counts, reverse=True)
for expt_id, idx in enumerate(
sort_order[:params.output.n_subset]):
subset_exp.append(experiments[idx])
refls = reflections.select(reflections["id"] == idx)
refls["id"] = flex.int(len(refls), expt_id)
subset_refls.extend(refls)
experiments = subset_exp
reflections = subset_refls
def save_in_batches(experiments,
reflections,
exp_name,
refl_name,
batch_size=1000):
from dxtbx.command_line.image_average import splitit
for i, indices in enumerate(
splitit(list(range(len(experiments))),
(len(experiments) // batch_size) + 1)):
batch_expts = ExperimentList()
batch_refls = flex.reflection_table()
for sub_id, sub_idx in enumerate(indices):
batch_expts.append(experiments[sub_idx])
sub_refls = reflections.select(
reflections["id"] == sub_idx)
sub_refls["id"] = flex.int(len(sub_refls), sub_id)
batch_refls.extend(sub_refls)
exp_filename = os.path.splitext(exp_name)[0] + "_%03d.expt" % i
ref_filename = os.path.splitext(
refl_name)[0] + "_%03d.refl" % i
self._save_output(batch_expts, batch_refls, exp_filename,
ref_filename)
def combine_in_clusters(experiments_l, reflections_l, exp_name,
refl_name, end_count):
result = []
for cluster, experiment in enumerate(experiments_l):
cluster_expts = ExperimentList()
cluster_refls = flex.reflection_table()
for i, expts in enumerate(experiment):
refls = reflections_l[cluster][i]
refls["id"] = flex.int(len(refls), i)
cluster_expts.append(expts)
cluster_refls.extend(refls)
exp_filename = os.path.splitext(exp_name)[0] + (
"_cluster%d.expt" % (end_count - cluster))
ref_filename = os.path.splitext(refl_name)[0] + (
"_cluster%d.refl" % (end_count - cluster))
result.append(
(cluster_expts, cluster_refls, exp_filename, ref_filename))
return result
# cluster the resulting experiments if requested
if params.clustering.use:
clustered = Cluster(
experiments,
reflections,
dendrogram=params.clustering.dendrogram,
threshold=params.clustering.threshold,
n_max=params.clustering.max_crystals,
)
n_clusters = len(clustered.clustered_frames)
def not_too_many(keeps):
if params.clustering.max_clusters is not None:
return len(keeps) < params.clustering.max_clusters
return True
keep_frames = []
sorted_keys = sorted(clustered.clustered_frames.keys())
while len(clustered.clustered_frames) > 0 and not_too_many(
keep_frames):
keep_frames.append(
clustered.clustered_frames.pop(sorted_keys.pop(-1)))
if params.clustering.exclude_single_crystal_clusters:
keep_frames = [k for k in keep_frames if len(k) > 1]
clustered_experiments = [[f.experiment for f in frame_cluster]
for frame_cluster in keep_frames]
clustered_reflections = [[f.reflections for f in frame_cluster]
for frame_cluster in keep_frames]
list_of_combined = combine_in_clusters(
clustered_experiments,
clustered_reflections,
params.output.experiments_filename,
params.output.reflections_filename,
n_clusters,
)
for saveable_tuple in list_of_combined:
if params.output.max_batch_size is None:
self._save_output(*saveable_tuple)
else:
save_in_batches(*saveable_tuple,
batch_size=params.output.max_batch_size)
else:
if params.output.max_batch_size is None:
self._save_output(
experiments,
reflections,
params.output.experiments_filename,
params.output.reflections_filename,
)
else:
save_in_batches(
experiments,
reflections,
params.output.experiments_filename,
params.output.reflections_filename,
batch_size=params.output.max_batch_size,
)
return
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# Select only those reflections which were integrated
if 'intensity.prf.variance' in integrated:
selection = integrated.get_flags(
integrated.flags.integrated,
all=True)
else:
selection = integrated.get_flags(
integrated.flags.integrated_sum)
integrated = integrated.select(selection)
len_all = len(integrated)
integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels))
print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all)
# verify sigmas are sensible
if 'intensity.prf.value' in integrated:
if (integrated['intensity.prf.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
if 'intensity.sum.value' in integrated:
if (integrated['intensity.sum.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
# apply detector gain to summation variances
integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain
if 'background.sum.value' in integrated:
if (integrated['background.sum.variance'] < 0).count(True) > 0:
raise Sorry("Found negative variances")
if (integrated['background.sum.variance'] == 0).count(True) > 0:
print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True))
integrated = integrated.select(integrated['background.sum.variance'] > 0)
# apply detector gain to background summation variances
integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import multi_kapton_correction
experiments, integrated = multi_kapton_correction(experiments, integrated,
abs_params.fuller_kapton, logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
sig_filter = SignificanceFilter(self.params)
refls = sig_filter(experiments, integrated)
logger.info("Removed %d reflections out of %d when applying significance filter"%(len(integrated)-len(refls), len(integrated)))
if len(refls) == 0:
raise Sorry("No reflections left after applying significance filter")
integrated = refls
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, '_ML_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def run(args):
user_phil = []
input_dirs = []
input_paths = []
for arg in args:
if os.path.isdir(arg):
input_dirs.append(arg)
continue
elif os.path.exists(arg):
input_paths.append(arg)
continue
try:
user_phil.append(parse(arg))
except Exception as e:
raise Sorry("Unrecognized argument %s" % arg)
params = phil_scope.fetch(sources=user_phil).extract()
sf_params = sf_scope.extract()
sf_params.significance_filter.isigi_cutoff = params.i_sigi_cutoff
sf_params.significance_filter.d_min = params.d_min
def get_paths(dirname):
absolute = lambda name: os.path.join(dirname, name)
names = os.listdir(dirname)
return map(absolute, names)
files_dict = {dirname: get_paths(dirname) for dirname in input_dirs}
if params.run_tags_from_filenames:
for path in input_paths:
filename = os.path.basename(path)
try:
run = int(
filename.split("idx-")[1].split("-")[0].split("run")[1])
except Exception:
run = int(
filename.split("idx-")[1].split("-")[0].split("r")[1])
except Exception:
run = None
try:
files_dict[run].append(path)
except KeyError:
files_dict[run] = [path]
else:
files_dict[None] = input_paths
all_results = []
runs = []
# iterate through grouped file paths and look for processing results
for run, files in files_dict.iteritems():
if len(files) == 0: continue
runs.append(run)
timestamps = flex.double()
two_theta_low = flex.double()
two_theta_high = flex.double()
n_strong = flex.int()
resolutions = flex.double()
n_lattices = flex.int()
for i, path in enumerate(sorted(files)):
root = os.path.dirname(path)
filename = os.path.basename(path)
split_fn = filename.split('_')
if len(split_fn) <= 0 or split_fn[-1] != "strong.pickle":
continue
base = os.path.join(root, "_".join(split_fn[:-1]))
print filename
strong_name = base + "_strong.pickle"
if not os.path.exists(strong_name):
print "Couldn't log %s, strong pickle not found" % filename
continue
# Read the spotfinding results
strong = easy_pickle.load(strong_name)
print "N strong reflections: %d" % len(strong)
timestamps.append(i)
n_strong.append(len(strong))
two_theta_low.append(0)
two_theta_high.append(0)
# Read indexing results if possible
experiments_name = base + "_integrated_experiments.json"
indexed_name = base + "_integrated.pickle"
if not os.path.exists(experiments_name) or not os.path.exists(
indexed_name):
print "Frame didn't index"
resolutions.append(0)
n_lattices.append(0)
continue
experiments = ExperimentListFactory.from_json_file(
experiments_name, check_format=False)
n_lattices.append(len(experiments))
reflections = easy_pickle.load(indexed_name)
reflections = reflections.select(reflections['intensity.sum.value']
> 0) # positive reflections only
best_d_min = None
for expt_id, experiment in enumerate(experiments):
refls = reflections.select(reflections['id'] == expt_id)
refls['id'] = flex.int(len(refls), 0)
sig_filter = SignificanceFilter(sf_params)
sig_filter(experiments[expt_id:expt_id + 1], refls)
if best_d_min is None or sig_filter.best_d_min < best_d_min:
best_d_min = sig_filter.best_d_min
resolutions.append(best_d_min or 0)
all_results.append((timestamps, two_theta_low, two_theta_high,
n_strong, resolutions, n_lattices))
plot_multirun_stats(all_results, runs, params.d_min, n_strong_cutoff=params.n_strong_cutoff, \
i_sigi_cutoff=params.i_sigi_cutoff, run_tags=params.run_tags, \
minimalist=params.minimalist, interactive=True, compress_runs=True, \
title=params.title)
print "Frame didn't index"
resolutions.append(0)
n_lattices.append(0)
continue
experiments = ExperimentListFactory.from_json_file(
experiments_name, check_format=False)
n_lattices.append(len(experiments))
reflections = easy_pickle.load(indexed_name)
reflections = reflections.select(reflections['intensity.sum.value']
> 0) # positive reflections only
best_d_min = None
for expt_id, experiment in enumerate(experiments):
refls = reflections.select(reflections['id'] == expt_id)
refls['id'] = flex.int(len(refls), 0)
sig_filter = SignificanceFilter(sf_params)
sig_filter(experiments[expt_id:expt_id + 1], refls)
if best_d_min is None or sig_filter.best_d_min < best_d_min:
best_d_min = sig_filter.best_d_min
resolutions.append(best_d_min or 0)
all_results.append((timestamps, two_theta_low, two_theta_high,
n_strong, resolutions, n_lattices))
plot_multirun_stats(all_results, runs, params.d_min, n_strong_cutoff=params.n_strong_cutoff, \
i_sigi_cutoff=params.i_sigi_cutoff, run_tags=params.run_tags, \
minimalist=params.minimalist, interactive=True, compress_runs=True, \
title=params.title)
if __name__ == "__main__":
def integrate(self, experiments, indexed):
if self.params.skip_hopper:
return super(Hopper_Processor,
self).integrate(experiments, indexed)
st = time.time()
logger.info("*" * 80)
logger.info("Integrating Reflections")
logger.info("*" * 80)
indexed, _ = self.process_reference(indexed)
if self.params.integration.integration_only_overrides.trusted_range:
for detector in experiments.detectors():
for panel in detector:
panel.set_trusted_range(
self.params.integration.integration_only_overrides.
trusted_range)
if self.params.dispatch.coset:
from xfel.util.sublattice_helper import integrate_coset
integrate_coset(self, experiments, indexed)
# Get the integrator from the input parameters
logger.info("Configuring integrator from input parameters")
from dials.algorithms.integration.integrator import create_integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments,
indexed)
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
for expt_id, expt in enumerate(experiments):
if (self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is
None or expt.profile.sigma_b() <
self.params.profile.gaussian_rs.parameters.sigma_b_cutoff):
refls = indexed.select(indexed["id"] == expt_id)
refls["id"] = flex.int(len(refls), len(new_experiments))
# refls.reset_ids()
del refls.experiment_identifiers()[expt_id]
refls.experiment_identifiers()[len(
new_experiments)] = expt.identifier
new_reflections.extend(refls)
new_experiments.append(expt)
else:
logger.info("Rejected expt %d with sigma_b %f" %
(expt_id, expt.profile.sigma_b()))
experiments = new_experiments
indexed = new_reflections
if len(experiments) == 0:
raise RuntimeError("No experiments after filtering by sigma_b")
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
# NOTE: this is the only changed needed to dials.stills_process
# TODO: multi xtal
# TODO: add in normal dials predictions as an option
predicted, model = predictions.get_predicted_from_pandas(
self.stage1_df,
self.params.diffBragg,
self.observed,
experiments[0].identifier,
self.device_id,
spectrum_override=self.stage1_modeler.SIM.beam.spectrum)
if self.params.refine_predictions:
experiments, rnd2_refls = self.refine(experiments,
predicted,
refining_predictions=True,
best=self.stage1_df)
# TODO: match rnd2_refls with indexed.refl and re-save indexed.refl
predicted, model = predictions.get_predicted_from_pandas(
self.stage1_df,
self.params.diffBragg,
self.observed,
experiments[0].identifier,
self.device_id,
spectrum_override=self.stage1_modeler.SIM.beam.spectrum)
predicted.match_with_reference(indexed)
integrator = create_integrator(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
if self.params.partial_correct:
integrated = predictions.normalize_by_partiality(
integrated,
model,
default_F=self.params.diffBragg.predictions.default_Famplitude,
gain=self.params.diffBragg.refiner.adu_per_photon)
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply:
if abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction, )
elif abs_params.algorithm == "kapton_2019":
from dials.algorithms.integration.kapton_2019_correction import (
multi_kapton_correction, )
experiments, integrated = multi_kapton_correction(
experiments,
integrated,
abs_params.fuller_kapton,
logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
sig_filter = SignificanceFilter(self.params)
filtered_refls = sig_filter(experiments, integrated)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
logger.info(
"Removed %d reflections out of %d when applying significance filter",
len(integrated) - len(filtered_refls),
len(integrated),
)
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
logger.info(
"Removed experiment %d which has no reflections left after applying significance filter",
expt_id,
)
if len(accepted_refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
experiments = accepted_expts
integrated = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
del integrated["shoebox"]
if self.params.output.composite_output:
if (self.params.output.integrated_experiments_filename
or self.params.output.integrated_filename):
assert (self.params.output.integrated_experiments_filename
is not None
and self.params.output.integrated_filename is not None)
n = len(self.all_integrated_experiments)
self.all_integrated_experiments.extend(experiments)
for i, experiment in enumerate(experiments):
refls = integrated.select(integrated["id"] == i)
refls["id"] = flex.int(len(refls), n)
del refls.experiment_identifiers()[i]
refls.experiment_identifiers()[n] = experiment.identifier
self.all_integrated_reflections.extend(refls)
n += 1
else:
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
experiments.as_json(
self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated,
self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import (
calc_2D_rmsd_and_displacements, )
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n" % rmsd_indexed
for i in range(6):
bright_integrated = integrated.select(
(integrated["intensity.sum.value"] /
flex.sqrt(integrated["intensity.sum.variance"])) >= i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(
bright_integrated)
else:
rmsd_integrated = 0
log_str += (
"N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
% (i, len(bright_integrated), rmsd_integrated))
for crystal_model in experiments.crystals():
if hasattr(crystal_model, "get_domain_size_ang"):
log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format(
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg(),
)
logger.info(log_str)
logger.info("")
logger.info("Time Taken = %f seconds", time.time() - st)
return integrated
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed, _ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments,
indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments,
predicted)
# Integrate the reflections
integrated = integrator.integrate()
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import multi_kapton_correction
experiments, integrated = multi_kapton_correction(
experiments,
integrated,
abs_params.fuller_kapton,
logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
sig_filter = SignificanceFilter(self.params)
refls = sig_filter(experiments, integrated)
logger.info(
"Removed %d reflections out of %d when applying significance filter"
% (len(integrated) - len(refls), len(integrated)))
if len(refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
integrated = refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and 'shoebox' in integrated:
del integrated['shoebox']
if self.params.output.composite_output:
if self.params.output.integrated_experiments_filename or self.params.output.integrated_filename:
assert self.params.output.integrated_experiments_filename is not None and self.params.output.integrated_filename is not None
from dials.array_family import flex
n = len(self.all_integrated_experiments)
self.all_integrated_experiments.extend(experiments)
for i, experiment in enumerate(experiments):
refls = integrated.select(integrated['id'] == i)
refls['id'] = flex.int(len(refls), n)
self.all_integrated_reflections.extend(refls)
n += 1
else:
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
from dxtbx.model.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(
self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated,
self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n" % (rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select(
(integrated['intensity.sum.value'] /
flex.sqrt(integrated['intensity.sum.variance'])) >= i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(
bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % (
i, len(bright_integrated), rmsd_integrated)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, 'get_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f" % (
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg())
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def integrate(self, experiments, indexed):
# TODO: Figure out if this is necessary and/or how to do this better
indexed, _ = self.process_reference(indexed)
if self.params.integration.integration_only_overrides.trusted_range:
for detector in experiments.detectors():
for panel in detector:
panel.set_trusted_range(
self.params.integration.integration_only_overrides.trusted_range
)
# Get the integrator from the input parameters
from dials.algorithms.integration.integrator import create_integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
for expt_id, expt in enumerate(experiments):
if (
self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is None
or expt.profile.sigma_b()
< self.params.profile.gaussian_rs.parameters.sigma_b_cutoff
):
refls = indexed.select(indexed["id"] == expt_id)
refls["id"] = flex.int(len(refls), len(new_experiments))
# refls.reset_ids()
del refls.experiment_identifiers()[expt_id]
refls.experiment_identifiers()[len(new_experiments)] = expt.identifier
new_reflections.extend(refls)
new_experiments.append(expt)
else:
# TODO: this can be done better, also
print(
"Rejected expt %d with sigma_b %f"
% (expt_id, expt.profile.sigma_b())
)
experiments = new_experiments
indexed = new_reflections
if len(experiments) == 0:
raise RuntimeError("No experiments after filtering by sigma_b")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static,
)
predicted.match_with_reference(indexed)
integrator = create_integrator(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply:
if abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction,
)
elif abs_params.algorithm == "kapton_2019":
from dials.algorithms.integration.kapton_2019_correction import (
multi_kapton_correction,
)
experiments, integrated = multi_kapton_correction(
experiments, integrated, abs_params.fuller_kapton, logger=logger
)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter,
)
sig_filter = SignificanceFilter(self.params)
filtered_refls = sig_filter(experiments, integrated)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
print(
"Removed experiment %d which has no reflections left after applying significance filter",
expt_id,
)
if len(accepted_refls) == 0:
raise Sorry("No reflections left after applying significance filter")
experiments = accepted_expts
integrated = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
del integrated["shoebox"]
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
experiments.as_json(self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(
integrated, self.params.output.integrated_filename
)
self.write_integration_pickles(integrated, experiments)
# TODO: Figure out what this is
from dials.algorithms.indexing.stills_indexer import (
calc_2D_rmsd_and_displacements,
)
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = f"RMSD indexed (px): {rmsd_indexed:f}\n"
for i in range(6):
bright_integrated = integrated.select(
(
integrated["intensity.sum.value"]
/ flex.sqrt(integrated["intensity.sum.variance"])
)
>= i
)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += (
"N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
% (i, len(bright_integrated), rmsd_integrated)
)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, "get_domain_size_ang"):
log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format(
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg(),
)
print(log_str)
return integrated