def run(args):
usage = "dials.estimate_resolution [options] (scaled.expt scaled.refl | scaled_unmerged.mtz)"
import libtbx.load_env
if libtbx.env.dispatcher_name == "dials.resolutionizer":
warnings.warn(
"dials.resolutionizer is now deprecated, please use dials.estimate_resolution instead"
)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options, unhandled = parser.parse_args(
args=args, return_unhandled=True, show_diff_phil=True
)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments
)
if (not reflections or not experiments) and not unhandled:
parser.print_help()
return
if reflections and experiments and unhandled:
sys.exit(
"Must provide either scaled unmerged mtz OR dials-format scaled reflections and experiments files"
)
# Configure the logging
log.config(logfile=params.output.log, verbosity=options.verbose)
logger.info(dials_version())
if len(unhandled) == 1:
scaled_unmerged = unhandled[0]
m = resolution_analysis.Resolutionizer.from_unmerged_mtz(
scaled_unmerged, params.resolution
)
else:
reflections = parse_multiple_datasets(reflections)
m = resolution_analysis.Resolutionizer.from_reflections_and_experiments(
reflections, experiments, params.resolution
)
plots = m.resolution_auto()
if params.output.html:
output_html_report(plots, params.output.html)
if params.output.json:
with open(params.output.json, "w") as fh:
json.dump(plots, fh)
return plots
def protk_experiments_and_reflections(dials_data):
data_dir = dials_data("multi_crystal_proteinase_k")
# Load experiments
experiments = ExperimentList()
for expt_file in sorted(f.strpath
for f in data_dir.listdir("experiments*.json")):
experiments.extend(load.experiment_list(expt_file, check_format=False))
# Load reflection tables
reflections = [
flex.reflection_table.from_file(refl_file) for refl_file in sorted(
f.strpath for f in data_dir.listdir("reflections*.pickle"))
]
# Setup experiment identifiers
reflections = parse_multiple_datasets(reflections)
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
# Combine into single ExperimentList
reflections_all = flex.reflection_table()
for i, (expt, refl) in enumerate(zip(experiments, reflections)):
reflections_all.extend(refl)
reflections_all.assert_experiment_identifiers_are_consistent(experiments)
return experiments, reflections_all
def run(args=None):
usage = "dials.cosym [options] models.expt observations.refl"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options, args = parser.parse_args(args=args,
show_diff_phil=False,
return_unhandled=True)
# Configure the logging
log.config(verbosity=options.verbose, logfile=params.output.log)
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
if params.seed is not None:
flex.set_random_seed(params.seed)
np.random.seed(params.seed)
random.seed(params.seed)
if not params.input.experiments or not params.input.reflections:
parser.print_help()
sys.exit()
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
reflections = parse_multiple_datasets(reflections)
if len(experiments) != len(reflections):
raise Sorry(
"Mismatched number of experiments and reflection tables found: %s & %s."
% (len(experiments), len(reflections)))
try:
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
cosym_instance = cosym(experiments=experiments,
reflections=reflections,
params=params)
except ValueError as e:
raise Sorry(e)
if params.output.html or params.output.json:
register_default_cosym_observers(cosym_instance)
cosym_instance.run()
cosym_instance.export()
def run(args):
usage = "dials.symmetry [options] models.expt observations.refl"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, _, args = parser.parse_args(args=args,
show_diff_phil=False,
return_unhandled=True)
# Configure the logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
if params.seed is not None:
import random
flex.set_random_seed(params.seed)
random.seed(params.seed)
if not params.input.experiments or not params.input.reflections:
parser.print_help()
sys.exit()
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
reflections = parse_multiple_datasets(reflections)
if len(experiments) != len(reflections):
raise Sorry(
"Mismatched number of experiments and reflection tables found: %s & %s."
% (len(experiments), len(reflections)))
try:
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
symmetry(experiments, reflections, params=params)
except ValueError as e:
raise Sorry(e)
def run(args=None):
usage = "dials.estimate_resolution [options] (scaled.expt scaled.refl | scaled_unmerged.mtz)"
parser = ArgumentParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
params, options, unhandled = parser.parse_args(args=args,
return_unhandled=True,
show_diff_phil=True)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
if (not reflections or not experiments) and not unhandled:
parser.print_help()
return
if reflections and experiments and unhandled:
sys.exit(
"Must provide either scaled unmerged mtz OR dials-format scaled reflections and experiments files"
)
# Configure the logging
log.config(logfile=params.output.log, verbosity=options.verbose)
logger.info(dials_version())
if len(unhandled) == 1:
scaled_unmerged = unhandled[0]
m = resolution_analysis.Resolutionizer.from_unmerged_mtz(
scaled_unmerged, params.resolution)
else:
reflections = parse_multiple_datasets(reflections)
if len(experiments) != len(reflections):
sys.exit(
f"Mismatched number of experiments and reflection tables found: {len(experiments)} & {len(reflections)}."
)
m = resolution_analysis.Resolutionizer.from_reflections_and_experiments(
reflections, experiments, params.resolution)
plots = m.resolution_auto()
if params.output.html:
output_html_report(plots, params.output.html)
if params.output.json:
with open(params.output.json, "w") as fh:
json.dump(plots, fh)
return plots
def run(args=None):
"""Run symmetry analysis from the command-line."""
usage = "dials.symmetry [options] models.expt observations.refl"
parser = ArgumentParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options, args = parser.parse_args(args=args,
show_diff_phil=False,
return_unhandled=True)
# Configure the logging
log.config(verbosity=options.verbose, logfile=params.output.log)
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
if params.seed is not None:
flex.set_random_seed(params.seed)
random.seed(params.seed)
if not params.input.experiments or not params.input.reflections:
parser.print_help()
sys.exit()
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
reflections = parse_multiple_datasets(reflections)
if len(experiments) != len(reflections):
sys.exit(
"Mismatched number of experiments and reflection tables found: %s & %s."
% (len(experiments), len(reflections)))
try:
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
symmetry(experiments, reflections, params=params)
except ValueError as e:
sys.exit(e)
def test_parse_multiple_datasets():
"""Test the namesake function. This expects a list of reflection tables, and
selects on the column named 'id'."""
rt1 = flex.reflection_table()
rt1["id"] = flex.int([0, 0, 0])
rt1.experiment_identifiers()[0] = "0"
rt2 = flex.reflection_table()
rt2["id"] = flex.int([2, 2, 4, 4])
rt2.experiment_identifiers()[2] = "2"
rt2.experiment_identifiers()[4] = "4"
single_tables = parse_multiple_datasets([rt2])
assert len(single_tables) == 2
assert list(set(single_tables[0]["id"])) == [2]
assert list(set(single_tables[1]["id"])) == [4]
single_tables = parse_multiple_datasets([rt1, rt2])
assert list(set(single_tables[0]["id"])) == [0]
assert list(set(single_tables[1]["id"])) == [2]
assert list(set(single_tables[2]["id"])) == [4]
assert len(single_tables) == 3
single_tables = parse_multiple_datasets([rt1])
assert len(single_tables) == 1
assert list(set(single_tables[0]["id"])) == [0]
# if a duplicate id is given, then this should be detected and new ids
# determined for all datasets.
rt3 = flex.reflection_table()
rt3["id"] = flex.int([2, 2])
rt3.experiment_identifiers()[2] = "5"
single_tables = parse_multiple_datasets([rt1, rt2, rt3])
assert len(single_tables) == 4
assert list(set(single_tables[0]["id"])) == [0]
assert single_tables[0].experiment_identifiers()[0] == "0"
assert list(set(single_tables[1]["id"])) == [1]
assert single_tables[1].experiment_identifiers()[1] == "2"
assert list(set(single_tables[2]["id"])) == [2]
assert single_tables[2].experiment_identifiers()[2] == "4"
assert list(set(single_tables[3]["id"])) == [3]
assert single_tables[3].experiment_identifiers()[3] == "5"
def delta_cc_half_analysis(self):
# transform models into miller arrays
intensities, batches = filtered_arrays_from_experiments_reflections(
self._data_manager.experiments,
parse_multiple_datasets([self._data_manager.reflections]),
outlier_rejection_after_filter=False,
partiality_threshold=0.99,
return_batches=True,
)
result = DeltaCcHalf(intensities, batches)
d = {}
d.update(result.histogram())
d.update(result.normalised_scores())
return d, result.get_table(html=True)
def run(args):
usage = (
"dials.resolutionizer [options] (scaled.expt scaled.refl | scaled_unmerged.mtz)"
)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options, unhandled = parser.parse_args(return_unhandled=True,
show_diff_phil=True)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
if (not reflections or not experiments) and not unhandled:
parser.print_help()
return
if reflections and experiments and unhandled:
sys.exit(
"Must provide either scaled unmerged mtz OR dials-format scaled reflections and experiments files"
)
# Configure the logging
log.config(logfile=params.output.log, verbosity=options.verbose)
logger.info(dials_version())
if len(unhandled) == 1:
scaled_unmerged = unhandled[0]
m = resolutionizer.Resolutionizer.from_unmerged_mtz(
scaled_unmerged, params.resolutionizer)
else:
reflections = parse_multiple_datasets(reflections)
m = resolutionizer.Resolutionizer.from_reflections_and_experiments(
reflections, experiments, params.resolutionizer)
m.resolution_auto()
def run(args=None):
"""Run assign experiment identifiers from the command line."""
usage = (
"""Usage: dials.assign_experiment_identifiers observations.refl models.expt"""
)
parser = ArgumentParser(
usage=usage,
read_experiments=True,
read_reflections=True,
phil=phil_scope,
check_format=False,
epilog=help_message,
)
params, _ = parser.parse_args(args=args, show_diff_phil=False)
if not params.input.experiments or not params.input.reflections:
parser.print_help()
sys.exit()
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
reflections = parse_multiple_datasets(reflections)
if len(experiments) != len(reflections):
raise Sorry(
"Mismatched number of experiments and reflection tables found: %s & %s."
% (len(experiments), len(reflections)))
try:
experiments, reflections = assign_unique_identifiers(
experiments, reflections, params.identifiers)
except ValueError as e:
raise Sorry(e)
print(f"assigned identifiers: {list(experiments.identifiers())}")
experiments.as_file(params.output.experiments)
joint_table = flex.reflection_table()
for reflection_table in reflections:
joint_table.extend(reflection_table)
joint_table.as_file(params.output.reflections)
def run():
# The script usage
usage = ("usage: xia2.multi_crystal_analysis [options] [param.phil] "
"models.expt observations.refl")
# Create the parser
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
# Parse the command line
params, options = parser.parse_args(show_diff_phil=False)
# Configure the logging
for name in ("xia2", "dials"):
log.config(verbosity=options.verbose,
logfile=params.output.log,
name=name)
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Try to load the models and data
if len(params.input.experiments) == 0:
logger.info("No Experiments found in the input")
parser.print_help()
return
if len(params.input.reflections) == 0:
logger.info("No reflection data found in the input")
parser.print_help()
return
try:
assert len(params.input.reflections) == len(params.input.experiments)
except AssertionError:
raise Sorry("The number of input reflections files does not match the "
"number of input experiments")
if params.seed is not None:
flex.set_random_seed(params.seed)
random.seed(params.seed)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
reflections = parse_multiple_datasets(reflections)
joint_table = flex.reflection_table()
for i in range(len(reflections)):
joint_table.extend(reflections[i])
reflections = joint_table
MultiCrystalReport(params,
experiments=experiments,
reflections=reflections).report()
def prepare_input(params, experiments, reflections):
"""Perform checks on the data and prepare the data for scaling.
Raises:
ValueError - a range of checks are made, a ValueError may be raised
for a number of reasons.
"""
#### First exclude any datasets, before the dataset is split into
#### individual reflection tables and expids set.
if (params.dataset_selection.exclude_datasets
or params.dataset_selection.use_datasets):
experiments, reflections = select_datasets_on_ids(
experiments,
reflections,
params.dataset_selection.exclude_datasets,
params.dataset_selection.use_datasets,
)
ids = flex.size_t()
for r in reflections:
ids.extend(r.experiment_identifiers().keys())
logger.info(
"\nDataset ids for retained datasets are: %s \n",
",".join(str(i) for i in ids),
)
#### Split the reflections tables into a list of reflection tables,
#### with one table per experiment.
logger.info("Checking for the existence of a reflection table \n"
"containing multiple datasets \n")
reflections = parse_multiple_datasets(reflections)
logger.info(
"Found %s reflection tables & %s experiments in total.",
len(reflections),
len(experiments),
)
if len(experiments) != len(reflections):
raise ValueError(
"Mismatched number of experiments and reflection tables found.")
#### Assign experiment identifiers.
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
ids = itertools.chain.from_iterable(r.experiment_identifiers().keys()
for r in reflections)
logger.info("\nDataset ids are: %s \n", ",".join(str(i) for i in ids))
for r in reflections:
r.unset_flags(flex.bool(len(r), True), r.flags.bad_for_scaling)
r.unset_flags(flex.bool(r.size(), True), r.flags.scaled)
reflections, experiments = exclude_image_ranges_for_scaling(
reflections, experiments, params.exclude_images)
#### Allow checking of consistent indexing, useful for
#### targeted / incremental scaling.
if params.scaling_options.check_consistent_indexing:
logger.info("Running dials.cosym to check consistent indexing:\n")
cosym_params = cosym_phil_scope.extract()
cosym_params.nproc = params.scaling_options.nproc
cosym_instance = cosym(experiments, reflections, cosym_params)
cosym_instance.run()
experiments = cosym_instance.experiments
reflections = cosym_instance.reflections
logger.info("Finished running dials.cosym, continuing with scaling.\n")
#### Make sure all experiments in same space group
sgs = [
expt.crystal.get_space_group().type().number() for expt in experiments
]
if len(set(sgs)) > 1:
raise ValueError("""The experiments have different space groups:
space group numbers found: %s
Please reanalyse the data so that space groups are consistent,
(consider using dials.reindex, dials.symmetry or dials.cosym) or
remove incompatible experiments (using the option exclude_datasets=)"""
% ", ".join(map(str, set(sgs))))
logger.info(
"Space group being used during scaling is %s",
experiments[0].crystal.get_space_group().info(),
)
#### If doing targeted scaling, extract data and append an experiment
#### and reflection table to the lists
if params.scaling_options.target_model:
logger.info("Extracting data from structural model.")
exp, reflection_table = create_datastructures_for_structural_model(
reflections, experiments, params.scaling_options.target_model)
experiments.append(exp)
reflections.append(reflection_table)
elif params.scaling_options.target_mtz:
logger.info("Extracting data from merged mtz.")
exp, reflection_table = create_datastructures_for_target_mtz(
experiments, params.scaling_options.target_mtz)
experiments.append(exp)
reflections.append(reflection_table)
#### Perform any non-batch cutting of the datasets, including the target dataset
best_unit_cell = params.reflection_selection.best_unit_cell
if best_unit_cell is None:
best_unit_cell = determine_best_unit_cell(experiments)
for reflection in reflections:
if params.cut_data.d_min or params.cut_data.d_max:
d = best_unit_cell.d(reflection["miller_index"])
if params.cut_data.d_min:
sel = d < params.cut_data.d_min
reflection.set_flags(sel,
reflection.flags.user_excluded_in_scaling)
if params.cut_data.d_max:
sel = d > params.cut_data.d_max
reflection.set_flags(sel,
reflection.flags.user_excluded_in_scaling)
if params.cut_data.partiality_cutoff and "partiality" in reflection:
reflection.set_flags(
reflection["partiality"] < params.cut_data.partiality_cutoff,
reflection.flags.user_excluded_in_scaling,
)
return params, experiments, reflections
def export_mtz(integrated_data, experiment_list, params):
"""Export data from integrated_data corresponding to experiment_list to an
MTZ file hklout."""
# if mtz filename is auto, then choose scaled.mtz or integrated.mtz
if params.mtz.hklout in (None, Auto, "auto"):
if ("intensity.scale.value"
in integrated_data) and ("intensity.scale.variance"
in integrated_data):
params.mtz.hklout = "scaled.mtz"
logger.info(
"Data appears to be scaled, setting mtz.hklout = 'scaled.mtz'")
else:
params.mtz.hklout = "integrated.mtz"
logger.info(
"Data appears to be unscaled, setting mtz.hklout = 'integrated.mtz'"
)
# First get the experiment identifier information out of the data
expids_in_table = integrated_data.experiment_identifiers()
if not list(expids_in_table.keys()):
reflection_tables = parse_multiple_datasets([integrated_data])
experiment_list, refl_list = assign_unique_identifiers(
experiment_list, reflection_tables)
integrated_data = flex.reflection_table()
for reflections in refl_list:
integrated_data.extend(reflections)
expids_in_table = integrated_data.experiment_identifiers()
integrated_data.assert_experiment_identifiers_are_consistent(
experiment_list)
expids_in_list = list(experiment_list.identifiers())
# Convert experiment_list to a real python list or else identity assumptions
# fail like:
# assert experiment_list[0] is experiment_list[0]
# And assumptions about added attributes break
experiment_list = list(experiment_list)
# Validate multi-experiment assumptions
if len(experiment_list) > 1:
# All experiments should match crystals, or else we need multiple crystals/datasets
if not all(x.crystal == experiment_list[0].crystal
for x in experiment_list[1:]):
logger.warning(
"Experiment crystals differ. Using first experiment crystal for file-level data."
)
wavelengths = match_wavelengths(experiment_list)
if len(wavelengths.keys()) > 1:
logger.info(
"Multiple wavelengths found: \n%s",
"\n".join(" Wavlength: %.5f, experiment numbers: %s " %
(k, ",".join(map(str, v)))
for k, v in wavelengths.items()),
)
else:
wavelengths = OrderedDict(
{experiment_list[0].beam.get_wavelength(): [0]})
# also only work correctly with one panel (for the moment)
if any(len(experiment.detector) != 1 for experiment in experiment_list):
logger.warning("Ignoring multiple panels in output MTZ")
best_unit_cell = params.mtz.best_unit_cell
if best_unit_cell is None:
best_unit_cell = determine_best_unit_cell(experiment_list)
integrated_data["d"] = best_unit_cell.d(integrated_data["miller_index"])
# Clean up the data with the passed in options
integrated_data = filter_reflection_table(
integrated_data,
intensity_choice=params.intensity,
partiality_threshold=params.mtz.partiality_threshold,
combine_partials=params.mtz.combine_partials,
min_isigi=params.mtz.min_isigi,
filter_ice_rings=params.mtz.filter_ice_rings,
d_min=params.mtz.d_min,
)
# get batch offsets and image ranges - even for scanless experiments
batch_offsets = [
expt.scan.get_batch_offset() for expt in experiment_list
if expt.scan is not None
]
unique_offsets = set(batch_offsets)
if len(set(unique_offsets)) <= 1:
logger.debug("Calculating new batches")
batch_offsets = calculate_batch_offsets(experiment_list)
batch_starts = [
e.scan.get_image_range()[0] if e.scan else 0
for e in experiment_list
]
effective_offsets = [
o + s for o, s in zip(batch_offsets, batch_starts)
]
unique_offsets = set(effective_offsets)
else:
logger.debug("Keeping existing batches")
image_ranges = get_image_ranges(experiment_list)
if len(unique_offsets) != len(batch_offsets):
raise ValueError("Duplicate batch offsets detected: %s" % ", ".join(
str(item)
for item, count in Counter(batch_offsets).items() if count > 1))
# Create the mtz file
mtz_writer = UnmergedMTZWriter(
experiment_list[0].crystal.get_space_group())
# FIXME TODO for more than one experiment into an MTZ file:
#
# - add an epoch (or recover an epoch) from the scan and add this as an extra
# column to the MTZ file for scaling, so we know that the two lattices were
# integrated at the same time
# ✓ decide a sensible BATCH increment to apply to the BATCH value between
# experiments and add this
for id_ in expids_in_table.keys():
# Grab our subset of the data
loc = expids_in_list.index(
expids_in_table[id_]) # get strid and use to find loc in list
experiment = experiment_list[loc]
if len(list(wavelengths.keys())) > 1:
for i, (wl, exps) in enumerate(wavelengths.items()):
if loc in exps:
wavelength = wl
dataset_id = i + 1
break
else:
wavelength = list(wavelengths.keys())[0]
dataset_id = 1
reflections = integrated_data.select(integrated_data["id"] == id_)
batch_offset = batch_offsets[loc]
image_range = image_ranges[loc]
reflections = assign_batches_to_reflections([reflections],
[batch_offset])[0]
experiment.data = dict(reflections)
s0n = matrix.col(experiment.beam.get_s0()).normalize().elems
logger.debug("Beam vector: %.4f %.4f %.4f" % s0n)
mtz_writer.add_batch_list(
image_range,
experiment,
wavelength,
dataset_id,
batch_offset=batch_offset,
force_static_model=params.mtz.force_static_model,
)
# Create the batch offset array. This gives us an experiment (id)-dependent
# batch offset to calculate the correct batch from image number.
experiment.data["batch_offset"] = flex.int(len(experiment.data["id"]),
batch_offset)
# Calculate whether we have a ROT value for this experiment, and set the column
_, _, z = experiment.data["xyzcal.px"].parts()
if experiment.scan:
experiment.data[
"ROT"] = experiment.scan.get_angle_from_array_index(z)
else:
experiment.data["ROT"] = z
mtz_writer.add_crystal(
crystal_name=params.mtz.crystal_name,
project_name=params.mtz.project_name,
unit_cell=best_unit_cell,
)
# Note: add unit cell here as may have changed basis since creating mtz.
# For multi-wave unmerged mtz, we add an empty dataset for each wavelength,
# but only write the data into the final dataset (for unmerged the batches
# link the unmerged data to the individual wavelengths).
for wavelength in wavelengths:
mtz_writer.add_empty_dataset(wavelength)
# Combine all of the experiment data columns before writing
combined_data = {
k: v.deep_copy()
for k, v in experiment_list[0].data.items()
}
for experiment in experiment_list[1:]:
for k, v in experiment.data.items():
combined_data[k].extend(v)
# ALL columns must be the same length
assert len({len(v)
for v in combined_data.values()
}) == 1, "Column length mismatch"
assert len(combined_data["id"]) == len(
integrated_data["id"]), "Lost rows in split/combine"
# Write all the data and columns to the mtz file
mtz_writer.write_columns(combined_data)
logger.info("Saving {} integrated reflections to {}".format(
len(combined_data["id"]), params.mtz.hklout))
mtz_file = mtz_writer.mtz_file
mtz_file.write(params.mtz.hklout)
return mtz_file
def run(args=None,
phil=phil_scope): # type: (List[str], libtbx.phil.scope) -> None
usage = "dials.missing_reflections [options] scaled.expt scaled.refl"
parser = OptionParser(
usage=usage,
phil=phil,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
params, options = parser.parse_args(args=args, show_diff_phil=False)
# Configure the logging.
dials.util.log.config(options.verbose)
logger.info(dials_version())
# Log the difference between the PHIL scope definition and the active PHIL scope,
# which will include the parsed user inputs.
diff_phil = parser.diff_phil.as_str()
if diff_phil:
logger.info("The following parameters have been modified:\n%s",
diff_phil)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if not experiments or not reflections:
parser.print_help()
return
if len(reflections) != 1 and len(experiments) != len(reflections):
sys.exit(
"Number of experiments must equal the number of reflection tables")
from dials.util.multi_dataset_handling import (
assign_unique_identifiers,
parse_multiple_datasets,
)
reflections = parse_multiple_datasets(reflections)
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
if all("inverse_scale_factor" in refl for refl in reflections):
# Assume all arrays have been scaled
miller_array = scaled_data_as_miller_array(reflections,
experiments,
anomalous_flag=False)
else:
# Else get the integrated intensities
miller_arrays = filtered_arrays_from_experiments_reflections(
experiments,
reflections,
)
miller_array = miller_arrays[0]
for ma in miller_arrays[1:]:
miller_array = miller_array.concatenate(ma)
if params.d_min or params.d_max:
miller_array = miller_array.resolution_filter(d_min=params.d_min,
d_max=params.d_max)
# Print overall summary of input miller array
s = io.StringIO()
ma_unique = miller_array.unique_under_symmetry()
ma_unique.show_comprehensive_summary(f=s)
logger.info(f"\n{s.getvalue()}")
# Get the regions of missing reflections
complete_set, unique_ms = missing_reflections.connected_components(
miller_array)
unique_ms = [
ms for ms in unique_ms if ms.size() >= params.min_component_size
]
# Print some output for user
if len(unique_ms):
logger.info(
"The following connected regions of missing reflections have been identified:"
)
n_expected = complete_set.size()
rows = []
for ms in unique_ms:
d_max, d_min = (uctbx.d_star_sq_as_d(ds2)
for ds2 in ms.min_max_d_star_sq())
rows.append([
ms.size(),
f"{100 * ms.size() / n_expected:.1f}",
f"{d_max:.2f}-{d_min:.2f}",
])
logger.info(
tabulate(
rows,
headers=["# reflections", "% missing",
"Resolution range (Å)"]))
else:
logger.info("No connected regions of missing reflections identified")
def run(self):
"""Run cycles of scaling and filtering."""
with ScalingHTMLContextManager(self):
start_time = time.time()
results = AnalysisResults()
for counter in range(
1, self.params.filtering.deltacchalf.max_cycles + 1):
self.run_scaling_cycle()
if counter == 1:
results.initial_expids_and_image_ranges = [
(exp.identifier,
exp.scan.get_image_range()) if exp.scan else None
for exp in self.experiments
]
delta_cc_params = deltacc_phil_scope.extract()
delta_cc_params.mode = self.params.filtering.deltacchalf.mode
delta_cc_params.group_size = (
self.params.filtering.deltacchalf.group_size)
delta_cc_params.stdcutoff = self.params.filtering.deltacchalf.stdcutoff
logger.info("\nPerforming a round of filtering.\n")
# need to reduce to single table.
joined_reflections = flex.reflection_table()
for table in self.reflections:
joined_reflections.extend(table)
script = deltaccscript(delta_cc_params, self.experiments,
joined_reflections)
script.run()
valid_image_ranges = get_valid_image_ranges(self.experiments)
results.expids_and_image_ranges = [
(exp.identifier,
valid_image_ranges[i]) if exp.scan else None
for i, exp in enumerate(self.experiments)
]
self.experiments = script.experiments
self.params.dataset_selection.use_datasets = None
self.params.dataset_selection.exclude_datasets = None
results = log_cycle_results(results, self, script)
logger.info(
"Cycle %s of filtering, n_reflections removed this cycle: %s",
counter,
results.get_last_cycle_results()["n_removed"],
)
# Test termination conditions
latest_results = results.get_last_cycle_results()
if latest_results["n_removed"] == 0:
logger.info(
"Finishing scaling and filtering as no data removed in this cycle."
)
if self.params.scaling_options.full_matrix:
self.reflections = parse_multiple_datasets(
[script.filtered_reflection_table])
results = self._run_final_scale_cycle(results)
else:
self.reflections = [script.filtered_reflection_table]
results.finish(termination_reason="no_more_removed")
break
# Need to split reflections for further processing.
self.reflections = parse_multiple_datasets(
[script.filtered_reflection_table])
if (latest_results["cumul_percent_removed"] >
self.params.filtering.deltacchalf.max_percent_removed):
logger.info(
"Finishing scale and filtering as have now removed more than the limit."
)
results = self._run_final_scale_cycle(results)
results.finish(termination_reason="max_percent_removed")
break
if self.params.filtering.deltacchalf.min_completeness:
if (latest_results["merging_stats"]["completeness"] <
self.params.filtering.deltacchalf.min_completeness
):
logger.info(
"Finishing scaling and filtering as completeness now below cutoff."
)
results = self._run_final_scale_cycle(results)
results.finish(
termination_reason="below_completeness_limit")
break
if counter == self.params.filtering.deltacchalf.max_cycles:
logger.info("Finishing as reached max number of cycles.")
results = self._run_final_scale_cycle(results)
results.finish(termination_reason="max_cycles")
break
# If not finished then need to create new scaler to try again
self._create_model_and_scaler()
self.filtering_results = results
# Print summary of results
logger.info(results)
with open(self.params.filtering.output.scale_and_filter_results,
"w") as f:
json.dump(self.filtering_results.to_dict(), f, indent=2)
# All done!
logger.info("\nTotal time taken: %.4fs ", time.time() - start_time)
logger.info("%s%s%s", "\n", "=" * 80, "\n")
def run(args=sys.argv[1:]):
# Create the parser
parser = OptionParser(
# usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
# epilog=help_message,
)
# Parse the command line
params, options, args = parser.parse_args(args=args,
show_diff_phil=False,
return_unhandled=True)
# Configure the logging
xia2.Handlers.Streams.setup_logging(logfile=params.output.log,
verbose=options.verbose)
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
if params.unit_cell is not None:
unit_cell = params.unit_cell
crystal_symmetry = crystal.symmetry(unit_cell=unit_cell)
else:
crystal_symmetry = None
if len(params.input.experiments):
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
reflections = parse_multiple_datasets(reflections)
if len(experiments) != len(reflections):
raise sys.exit(
"Mismatched number of experiments and reflection tables found: %s & %s."
% (len(experiments), len(reflections)))
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
# transform models into miller arrays
intensities, batches = filtered_arrays_from_experiments_reflections(
experiments,
reflections,
outlier_rejection_after_filter=False,
partiality_threshold=0.99,
return_batches=True,
)
if args and os.path.isfile(args[0]):
result = any_reflection_file(args[0])
unmerged_intensities = None
batches_all = None
for ma in result.as_miller_arrays(merge_equivalents=False,
crystal_symmetry=crystal_symmetry):
if ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]:
assert ma.anomalous_flag()
unmerged_intensities = ma
elif ma.info().labels == ["I", "SIGI"]:
assert not ma.anomalous_flag()
unmerged_intensities = ma
elif ma.info().labels == ["BATCH"]:
batches_all = ma
assert batches_all is not None
assert unmerged_intensities is not None
sel = unmerged_intensities.sigmas() > 0
unmerged_intensities = unmerged_intensities.select(sel).set_info(
unmerged_intensities.info())
batches_all = batches_all.select(sel)
id_to_batches = None
if len(params.batch) > 0:
id_to_batches = {}
for b in params.batch:
assert b.id is not None
assert b.range is not None
assert b.id not in id_to_batches, "Duplicate batch id: %s" % b.id
id_to_batches[b.id] = b.range
separate = separate_unmerged(unmerged_intensities,
batches_all,
id_to_batches=id_to_batches)
batches = list(separate.batches.values())
intensities = list(separate.intensities.values())
result = DeltaCcHalf(
intensities,
batches,
n_bins=params.n_bins,
d_min=params.d_min,
cc_one_half_method=params.cc_one_half_method,
group_size=params.group_size,
)
logger.info(tabulate(result.get_table(), headers="firstrow"))
hist_filename = "delta_cc_hist.png"
logger.info("Saving histogram to %s" % hist_filename)
result.plot_histogram(hist_filename)
normalised_scores_filename = "normalised_scores.png"
logger.info("Saving normalised scores to %s" % normalised_scores_filename)
result.plot_normalised_scores(normalised_scores_filename)
Citations.cite("delta_cc_half")
for citation in Citations.get_citations_acta():
logger.info(citation)
def run(args=None,
phil=phil_scope): # type: (List[str], libtbx.phil.scope) -> None
"""
Check command-line input and call other functions to do the legwork.
Run the script, parsing arguments found in 'args' and using the PHIL scope
defined in 'phil'.
Args:
args: The arguments supplied by the user (default: sys.argv[1:])
phil: The PHIL scope definition (default: phil_scope, the master PHIL scope
for this program).
"""
usage = "dials.command_name [options] imported.expt strong.refl"
parser = OptionParser(
usage=usage,
phil=phil,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
params, options = parser.parse_args(args=args, show_diff_phil=False)
# Configure the logging.
dials.util.log.config(options.verbose)
# Log the difference between the PHIL scope definition and the active PHIL scope,
# which will include the parsed user inputs.
diff_phil = parser.diff_phil.as_str()
if diff_phil:
logger.info("The following parameters have been modified:\n%s",
diff_phil)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if not experiments or not reflections:
parser.print_help()
return
if len(reflections) != 1 and len(experiments) != len(reflections):
sys.exit(
"Number of experiments must equal the number of reflection tables")
from dials.util.multi_dataset_handling import (
assign_unique_identifiers,
parse_multiple_datasets,
)
reflections = parse_multiple_datasets(reflections)
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
# Do whatever this program is supposed to do.
do_connected_components(
experiments,
reflections,
min_component_size=params.min_component_size,
)
def run(self):
"""Execute the script."""
start_time = time()
# Parse the command line
params, _ = self.parser.parse_args(show_diff_phil=False)
# set up global reflections list
reflections = flex.reflection_table()
# loop through the input, building up the global lists
reflections_list = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
reflections_list = parse_multiple_datasets(reflections_list)
for refs in reflections_list:
reflections.extend(refs)
# Try to load the models and data
nexp = len(experiments)
if nexp == 0:
print("No Experiments found in the input")
self.parser.print_help()
return
if not reflections:
print("No reflection data found in the input")
self.parser.print_help()
return
self.check_input(reflections)
# Configure the logging
log.config(logfile=params.output.log)
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Convert to P 1?
if params.refinement.triclinic:
reflections, experiments = self.convert_to_P1(reflections, experiments)
# Combine crystals?
if params.refinement.combine_crystal_models and len(experiments) > 1:
logger.info("Combining {0} crystal models".format(len(experiments)))
experiments = self.combine_crystals(experiments)
# Filter integrated centroids?
if params.refinement.filter_integrated_centroids:
reflections = self.filter_integrated_centroids(reflections)
# Filter data if scaled to remove outliers
if "inverse_scale_factor" in reflections:
try:
reflections = filter_reflection_table(reflections, ["scale"])
except ValueError as e:
logger.warn(e)
logger.info(
"Filtering on scaled data failed, proceeding with integrated data."
)
# Get the refiner
logger.info("Configuring refiner")
refiner = self.create_refiner(params, reflections, experiments)
# Refine the geometry
if nexp == 1:
logger.info("Performing refinement of a single Experiment...")
else:
logger.info("Performing refinement of {} Experiments...".format(nexp))
refiner.run()
# get the refined experiments
experiments = refiner.get_experiments()
crystals = experiments.crystals()
if len(crystals) == 1:
# output the refined model for information
logger.info("")
logger.info("Final refined crystal model:")
logger.info(crystals[0])
logger.info(self.cell_param_table(crystals[0]))
# Save the refined experiments to file
output_experiments_filename = params.output.experiments
logger.info(
"Saving refined experiments to {}".format(output_experiments_filename)
)
experiments.as_file(output_experiments_filename)
# Create correlation plots
if params.output.correlation_plot.filename is not None:
create_correlation_plots(refiner, params.output)
if params.output.cif is not None:
self.generate_cif(crystals[0], refiner, filename=params.output.cif)
if params.output.p4p is not None:
self.generate_p4p(
crystals[0], experiments[0].beam, filename=params.output.p4p
)
if params.output.mmcif is not None:
self.generate_mmcif(crystals[0], refiner, filename=params.output.mmcif)
# Log the total time taken
logger.info("\nTotal time taken: {:.2f}s".format(time() - start_time))
def run(args: List[str] = None, phil: libtbx.phil.scope = phil_scope) -> None:
"""
Run dials.anvil_correction as from the command line.
Take integrated experiment lists and reflection tables and correct the all the
integrated intensities for the estimated attenuation by the diamond anvils.
Args:
args: The arguments supplied by the user (default: sys.argv[1:]).
phil: The PHIL scope definition (default: phil_scope, the master PHIL scope
for this program).
"""
usage = "dials.anvil_correction [options] integrated.expt integrated.refl"
parser = OptionParser(
usage=usage,
phil=phil,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
params, options = parser.parse_args(args=args, show_diff_phil=False)
# Log the difference between the PHIL scope definition and the active PHIL scope,
# which will include the parsed user inputs.
diff_phil = parser.diff_phil.as_str()
if diff_phil:
logger.info("The following parameters have been modified:\n%s",
diff_phil)
# Check that at least one reflection table and experiment list have been provided.
input_errors = []
if not params.input.experiments:
input_errors.append(
"Please provide at least one valid experiment list (.expt) file.")
if not params.input.reflections:
input_errors.append(
"Please provide at least one valid reflection table (.refl) file.")
if input_errors:
sys.exit("\n".join([parser.format_help()] + input_errors))
if not np.linalg.norm(params.anvil.normal):
sys.exit(
"It seems you have provided a surface normal vector with zero length."
)
# Check that the anvil surface normal really is normalised.
dac_norm = params.anvil.normal / np.linalg.norm(params.anvil.normal)
# Configure the logging.
dials.util.log.config(options.verbose, logfile=params.output.log)
# These functions are commonly used to collate the input.
experiments = flatten_experiments(params.input.experiments)
reflections_list = flatten_reflections(params.input.reflections)
# Work around parse_multiple_datasets dropping unindexed reflections.
unindexed = flex.reflection_table()
for r_table in reflections_list:
unindexed.extend(r_table.select(r_table["id"] == -1))
# Get a single reflection table per experiment object.
reflections_list = parse_multiple_datasets(reflections_list)
reflections_list = sort_tables_to_experiments_order(
reflections_list, experiments)
# Record the density of diamond in g·cm⁻³ (for consistency with NIST tables,
# https://doi.org/10.18434/T4D01F).
density = params.anvil.density / 1000 # g·cm⁻³
# Correct for the attenuation of the incident and diffracted beams by the diamond
# anvil cell.
logger.info(
"Correcting integrated reflection intensities for attenuation by the diamond "
"anvil cell.")
for experiment, reflections in zip(experiments, reflections_list):
correct_intensities_for_dac_attenuation(experiment, reflections,
dac_norm,
params.anvil.thickness,
density)
logger.info("Done.")
# Do optional experiment list file output here.
if params.output.experiments:
logger.info("Writing the experiment list to %s",
params.output.experiments)
experiments.as_file(params.output.experiments)
logger.info("Writing the reflection table to %s",
params.output.reflections)
# Collate reflections into a single reflection table and save it to file.
reflections = unindexed
for r_table in reflections_list:
reflections.extend(r_table)
del reflections_list
reflections.as_file(params.output.reflections)
def run(args):
usage = ("xia2.multiplex [options] [param.phil] "
"models1.expt models2.expt observations1.refl "
"observations2.refl...")
# Create the parser
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
# Parse the command line
params, options = parser.parse_args(args=args, show_diff_phil=False)
# Configure the logging
xia2.Handlers.Streams.setup_logging(logfile=params.output.log,
verbose=options.verbose)
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Try to load the models and data
if len(params.input.experiments) == 0:
logger.info("No Experiments found in the input")
parser.print_help()
return
if len(params.input.reflections) == 0:
logger.info("No reflection data found in the input")
parser.print_help()
return
try:
assert len(params.input.reflections) == len(params.input.experiments)
except AssertionError:
raise sys.exit(
"The number of input reflections files does not match the "
"number of input experiments")
if params.seed is not None:
flex.set_random_seed(params.seed)
random.seed(params.seed)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) < 2:
sys.exit("xia2.multiplex requires a minimum of two experiments")
reflections = parse_multiple_datasets(reflections)
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
reflections, experiments = exclude_image_ranges_for_scaling(
reflections, experiments, params.exclude_images)
reflections_all = flex.reflection_table()
assert len(reflections) == 1 or len(reflections) == len(experiments)
for i, (expt, refl) in enumerate(zip(experiments, reflections)):
reflections_all.extend(refl)
reflections_all.assert_experiment_identifiers_are_consistent(experiments)
if params.identifiers is not None:
identifiers = []
for identifier in params.identifiers:
identifiers.extend(identifier.split(","))
params.identifiers = identifiers
try:
ScaleAndMerge.MultiCrystalScale(experiments, reflections_all, params)
except ValueError as e:
sys.exit(str(e))
