def __init__(self, experiments, reflections, params=None):
if params is None:
params = phil_scope.extract()
self._params = params
# transform models into miller arrays
n_datasets = len(experiments)
datasets = filtered_arrays_from_experiments_reflections(
experiments,
reflections,
outlier_rejection_after_filter=True,
partiality_threshold=params.partiality_threshold,
)
if len(datasets) != n_datasets:
raise ValueError(
"""Some datasets have no reflection after prefiltering, please check
input data and filtering settings e.g partiality_threshold""")
result = determine_space_group(
datasets,
normalisation=self._params.normalisation,
d_min=self._params.d_min,
min_i_mean_over_sigma_mean=self._params.min_i_mean_over_sigma_mean,
lattice_symmetry_max_delta=self._params.lattice_symmetry_max_delta,
relative_length_tolerance=self._params.relative_length_tolerance,
absolute_angle_tolerance=self._params.absolute_angle_tolerance,
)
logger.info(result)
if params.output.json is not None:
result.as_json(filename=params.output.json)
self._export_experiments_reflections(experiments, reflections, result)
def test_filtered_arrays_from_experiments_reflections_with_batches(dials_data):
x4wide_dir = dials_data("x4wide_processed")
refl = flex.reflection_table.from_file(
x4wide_dir.join("AUTOMATIC_DEFAULT_scaled.refl").strpath)
expts = ExperimentListFactory.from_serialized_format(
x4wide_dir.join("AUTOMATIC_DEFAULT_scaled.expt").strpath,
check_format=False)
miller_arrays, batches = filtered_arrays_from_experiments_reflections(
expts, [refl], return_batches=True)
assert len(miller_arrays) == len(batches) == 1
assert miller_arrays[0].size() == batches[0].size() == 62920
assert len(set(batches[0].data())) == 88
assert flex.min(batches[0].data()) == 2
assert flex.max(batches[0].data()) == 89
def delta_cc_half_analysis(self):
# transform models into miller arrays
intensities, batches = filtered_arrays_from_experiments_reflections(
self._data_manager.experiments,
parse_multiple_datasets([self._data_manager.reflections]),
outlier_rejection_after_filter=False,
partiality_threshold=0.99,
return_batches=True,
)
result = DeltaCcHalf(intensities, batches)
d = {}
d.update(result.histogram())
d.update(result.normalised_scores())
return d, result.get_table(html=True)
def __init__(self, experiments, reflections, params=None):
super(cosym, self).__init__(
events=["run_cosym", "performed_unit_cell_clustering"])
if params is None:
params = phil_scope.extract()
self.params = params
self._experiments, self._reflections = self._filter_min_reflections(
experiments, reflections)
# map experiments and reflections to primitive setting
self._experiments, self._reflections = self._map_to_primitive(
self._experiments, self._reflections)
if len(self._experiments) > 1:
# perform unit cell clustering
identifiers = self._unit_cell_clustering(self._experiments)
if len(identifiers) < len(self._experiments):
logger.info(
"Selecting subset of %i datasets for cosym analysis: %s",
len(identifiers),
str(identifiers),
)
self._experiments, self._reflections = select_datasets_on_ids(
self._experiments,
self._reflections,
use_datasets=identifiers)
self._experiments, self._reflections = self._map_to_minimum_cell(
self._experiments, self._reflections)
# transform models into miller arrays
datasets = filtered_arrays_from_experiments_reflections(
self.experiments,
self.reflections,
outlier_rejection_after_filter=False,
partiality_threshold=params.partiality_threshold,
)
self.cosym_analysis = CosymAnalysis(datasets, self.params)
def symmetry(experiments, reflection_tables, params=None):
"""
Run symmetry analysis
Args:
experiments: An experiment list.
reflection_tables: A list of reflection tables.
params: The dials.symmetry phil scope.
"""
result = None
if params is None:
params = phil_scope.extract()
refls_for_sym = []
if params.laue_group is Auto:
logger.info("=" * 80)
logger.info("")
logger.info("Performing Laue group analysis")
logger.info("")
# Transform models into miller arrays
n_datasets = len(experiments)
# Map experiments and reflections to minimum cell
# Eliminate reflections that are systematically absent due to centring
# of the lattice, otherwise they would lead to non-integer miller indices
# when reindexing to a primitive setting
cb_ops = change_of_basis_ops_to_minimum_cell(
experiments,
params.lattice_symmetry_max_delta,
params.relative_length_tolerance,
params.absolute_angle_tolerance,
)
reflection_tables = eliminate_sys_absent(experiments,
reflection_tables)
experiments, reflection_tables = apply_change_of_basis_ops(
experiments, reflection_tables, cb_ops)
refls_for_sym = get_subset_for_symmetry(experiments, reflection_tables,
params.exclude_images)
datasets = filtered_arrays_from_experiments_reflections(
experiments,
refls_for_sym,
outlier_rejection_after_filter=True,
partiality_threshold=params.partiality_threshold,
)
if len(datasets) != n_datasets:
raise ValueError(
"""Some datasets have no reflection after prefiltering, please check
input data and filtering settings e.g partiality_threshold""")
datasets = [
ma.as_anomalous_array().merge_equivalents().array()
for ma in datasets
]
result = LaueGroupAnalysis(
datasets,
normalisation=params.normalisation,
d_min=params.d_min,
min_i_mean_over_sigma_mean=params.min_i_mean_over_sigma_mean,
lattice_symmetry_max_delta=params.lattice_symmetry_max_delta,
relative_length_tolerance=params.relative_length_tolerance,
absolute_angle_tolerance=params.absolute_angle_tolerance,
best_monoclinic_beta=params.best_monoclinic_beta,
)
logger.info("")
logger.info(result)
if params.output.json is not None:
d = result.as_dict()
d["cb_op_inp_min"] = [str(cb_op) for cb_op in cb_ops]
# Copy the "input_symmetry" to "min_cell_symmetry" as it isn't technically
# the input symmetry to dials.symmetry
d["min_cell_symmetry"] = d["input_symmetry"]
del d["input_symmetry"]
json_str = json.dumps(d, indent=2)
with open(params.output.json, "w") as f:
f.write(json_str)
# Change of basis operator from input unit cell to best unit cell
cb_op_inp_best = result.best_solution.subgroup["cb_op_inp_best"]
# Get the best space group.
best_subsym = result.best_solution.subgroup["best_subsym"]
best_space_group = best_subsym.space_group(
).build_derived_acentric_group()
logger.info(
tabulate(
[[
str(best_subsym.space_group_info()),
str(best_space_group.info())
]],
["Patterson group", "Corresponding MX group"],
))
# Reindex the input data
experiments, reflection_tables = _reindex_experiments_reflections(
experiments, reflection_tables, best_space_group, cb_op_inp_best)
elif params.laue_group is not None:
if params.change_of_basis_op is not None:
cb_op = sgtbx.change_of_basis_op(params.change_of_basis_op)
else:
cb_op = sgtbx.change_of_basis_op()
# Reindex the input data
experiments, reflection_tables = _reindex_experiments_reflections(
experiments, reflection_tables, params.laue_group.group(), cb_op)
if params.systematic_absences.check:
logger.info("=" * 80)
logger.info("")
logger.info("Analysing systematic absences")
logger.info("")
# Get the laue class from the current space group.
space_group = experiments[0].crystal.get_space_group()
laue_group = str(space_group.build_derived_patterson_group().info())
logger.info("Laue group: %s", laue_group)
if laue_group in ("I m -3", "I m m m"):
if laue_group == "I m -3":
logger.info(
"""Space groups I 2 3 & I 21 3 cannot be distinguished with systematic absence
analysis, due to lattice centering.
Using space group I 2 3, space group I 21 3 is equally likely.\n""")
if laue_group == "I m m m":
logger.info(
"""Space groups I 2 2 2 & I 21 21 21 cannot be distinguished with systematic absence
analysis, due to lattice centering.
Using space group I 2 2 2, space group I 21 21 21 is equally likely.\n""")
elif laue_group not in laue_groups_for_absence_analysis:
logger.info("No absences to check for this laue group\n")
else:
if not refls_for_sym:
refls_for_sym = get_subset_for_symmetry(
experiments, reflection_tables, params.exclude_images)
if (params.d_min is Auto) and (result is not None):
d_min = result.intensities.resolution_range()[1]
elif params.d_min is Auto:
d_min = resolution_filter_from_reflections_experiments(
refls_for_sym,
experiments,
params.min_i_mean_over_sigma_mean,
params.min_cc_half,
)
else:
d_min = params.d_min
# combine before sys abs test - only triggers if laue_group=None and
# multiple input files.
if len(reflection_tables) > 1:
joint_reflections = flex.reflection_table()
for table in refls_for_sym:
joint_reflections.extend(table)
else:
joint_reflections = refls_for_sym[0]
merged_reflections = prepare_merged_reflection_table(
experiments, joint_reflections, d_min)
run_systematic_absences_checks(
experiments,
merged_reflections,
float(params.systematic_absences.significance_level),
)
logger.info(
"Saving reindexed experiments to %s in space group %s",
params.output.experiments,
str(experiments[0].crystal.get_space_group().info()),
)
experiments.as_file(params.output.experiments)
if params.output.reflections is not None:
if len(reflection_tables) > 1:
joint_reflections = flex.reflection_table()
for table in reflection_tables:
joint_reflections.extend(table)
else:
joint_reflections = reflection_tables[0]
logger.info(
"Saving %s reindexed reflections to %s",
len(joint_reflections),
params.output.reflections,
)
joint_reflections.as_file(params.output.reflections)
if params.output.html and params.systematic_absences.check:
ScrewAxisObserver().generate_html_report(params.output.html)
def test_filtered_arrays_from_experiments_reflections():
"""Test the creating of a miller array from crystal and reflection table."""
refl = generate_integrated_test_reflections()
refl["miller_index"] = flex.miller_index([(1, 0, 0), (2, 0, 0), (3, 0, 0),
(4, 0, 0), (5, 0, 0), (6, 0, 0)])
experiments = ExperimentList()
exp_dict = {
"__id__": "crystal",
"real_space_a": [1.0, 0.0, 0.0],
"real_space_b": [0.0, 1.0, 0.0],
"real_space_c": [0.0, 0.0, 2.0],
"space_group_hall_symbol": " C 2y",
}
crystal = Crystal.from_dict(exp_dict)
experiments.append(Experiment(crystal=crystal))
miller_set = filtered_arrays_from_experiments_reflections(
experiments, [refl])[0]
assert isinstance(miller_set, miller.set)
assert list(miller_set.data()) == [4.6, 2.4, 2.5] # same as calling filter
# for export on scale intensity reducer.
# now try for prf
del refl["intensity.scale.value"]
miller_set = filtered_arrays_from_experiments_reflections(
experiments, [refl])[0]
assert isinstance(miller_set, miller.set)
assert list(miller_set.data()) == [1.0, 2.0, 3.0] # same as calling filter
# for export on prf + sum intensity reducer.
# now just for sum
del refl["intensity.prf.value"]
miller_set = filtered_arrays_from_experiments_reflections(
experiments, [refl])[0]
assert isinstance(miller_set, miller.set)
assert list(miller_set.data()) == [11.0, 12.0, 13.0,
14.0] # same as calling
# filter for export on prf intensity reducer.
# Now try with a bad dataset - should be filtered.
refl = generate_integrated_test_reflections()
refl["miller_index"] = flex.miller_index([(1, 0, 0), (2, 0, 0), (3, 0, 0),
(4, 0, 0), (5, 0, 0), (6, 0, 0)])
# Trigger filtering on prf/sum, but when prf is bad - should proceed with sum
refl.unset_flags(flex.bool(6, True), refl.flags.integrated_prf)
del refl["intensity.scale.value"]
refl2 = generate_integrated_test_reflections()
refl2["partiality"] = flex.double(6, 0.0)
experiments = ExperimentList()
experiments.append(Experiment(crystal=crystal))
experiments.append(Experiment(crystal=crystal))
miller_sets = filtered_arrays_from_experiments_reflections(
experiments, [refl, refl2], outlier_rejection_after_filter=True)
assert len(miller_sets) == 1
with pytest.raises(AssertionError):
miller_sets = filtered_arrays_from_experiments_reflections(
experiments, [refl, refl2], return_batches=True)
experiments = ExperimentList()
experiments.append(Experiment(crystal=crystal))
experiments.append(Experiment(crystal=crystal))
refl2 = generate_integrated_test_reflections()
refl2["partiality"] = flex.double(6, 0.0)
with pytest.raises(ValueError):
refl["partiality"] = flex.double(6, 0.0)
_ = filtered_arrays_from_experiments_reflections(
experiments, [refl, refl2])
def run(args):
import libtbx.load_env
from dials.util import Sorry
usage = "dials.reindex [options] indexed.expt indexed.refl"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0 and len(reflections) == 0:
parser.print_help()
return
if params.change_of_basis_op is None:
raise Sorry("Please provide a change_of_basis_op.")
reference_crystal = None
if params.reference.experiments is not None:
from dxtbx.serialize import load
reference_experiments = load.experiment_list(
params.reference.experiments, check_format=False)
assert len(reference_experiments.crystals()) == 1
reference_crystal = reference_experiments.crystals()[0]
if params.reference.reflections is not None:
# First check that we have everything as expected for the reference reindexing
# Currently only supports reindexing one dataset at a time
if params.reference.experiments is None:
raise Sorry(
"""For reindexing against a reference dataset, a reference
experiments file must also be specified with the option: reference= """)
if not os.path.exists(params.reference.reflections):
raise Sorry("Could not locate reference dataset reflection file")
if len(experiments) != 1 or len(reflections) != 1:
raise Sorry(
"Only one dataset can be reindexed to a reference at a time")
reference_reflections = flex.reflection_table().from_file(
params.reference.reflections)
test_reflections = reflections[0]
if (reference_crystal.get_space_group().type().number() !=
experiments.crystals()[0].get_space_group().type().number()):
raise Sorry("Space group of input does not match reference")
# Set some flags to allow filtering, if wanting to reindex against
# reference with data that has not yet been through integration
if (test_reflections.get_flags(
test_reflections.flags.integrated_sum).count(True) == 0):
assert (
"intensity.sum.value"
in test_reflections), "No 'intensity.sum.value' in reflections"
test_reflections.set_flags(
flex.bool(test_reflections.size(), True),
test_reflections.flags.integrated_sum,
)
if (reference_reflections.get_flags(
reference_reflections.flags.integrated_sum).count(True) == 0):
assert ("intensity.sum.value" in test_reflections
), "No 'intensity.sum.value in reference reflections"
reference_reflections.set_flags(
flex.bool(reference_reflections.size(), True),
reference_reflections.flags.integrated_sum,
)
# Make miller array of the two datasets
try:
test_miller_set = filtered_arrays_from_experiments_reflections(
experiments, [test_reflections])[0]
except ValueError:
raise Sorry(
"No reflections remain after filtering the test dataset")
try:
reference_miller_set = filtered_arrays_from_experiments_reflections(
reference_experiments, [reference_reflections])[0]
except ValueError:
raise Sorry(
"No reflections remain after filtering the reference dataset")
from dials.algorithms.symmetry.reindex_to_reference import (
determine_reindex_operator_against_reference, )
change_of_basis_op = determine_reindex_operator_against_reference(
test_miller_set, reference_miller_set)
elif len(experiments) and params.change_of_basis_op is libtbx.Auto:
if reference_crystal is not None:
if len(experiments.crystals()) > 1:
raise Sorry("Only one crystal can be processed at a time")
from dials.algorithms.indexing.compare_orientation_matrices import (
difference_rotation_matrix_axis_angle, )
cryst = experiments.crystals()[0]
R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle(
cryst, reference_crystal)
print("Change of basis op: %s" % change_of_basis_op)
print("Rotation matrix to transform input crystal to reference::")
print(R.mathematica_form(format="%.3f", one_row_per_line=True))
print(
"Rotation of %.3f degrees" % angle,
"about axis (%.3f, %.3f, %.3f)" % axis,
)
elif len(reflections):
assert len(reflections) == 1
# always re-map reflections to reciprocal space
refl_copy = flex.reflection_table()
for i, imageset in enumerate(experiments.imagesets()):
if "imageset_id" in reflections[0]:
sel = reflections[0]["imageset_id"] == i
else:
sel = reflections[0]["id"] == i
refl = reflections[0].select(sel)
refl.centroid_px_to_mm(imageset.get_detector(),
imageset.get_scan())
refl.map_centroids_to_reciprocal_space(
imageset.get_detector(),
imageset.get_beam(),
imageset.get_goniometer(),
)
refl_copy.extend(refl)
# index the reflection list using the input experiments list
refl_copy["id"] = flex.int(len(refl_copy), -1)
index = AssignIndicesGlobal(tolerance=0.2)
index(refl_copy, experiments)
hkl_expt = refl_copy["miller_index"]
hkl_input = reflections[0]["miller_index"]
change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt)
# reset experiments list since we don't want to reindex this
experiments = []
else:
change_of_basis_op = sgtbx.change_of_basis_op(
params.change_of_basis_op)
if len(experiments):
for crystal in experiments.crystals():
cryst_orig = copy.deepcopy(crystal)
cryst_reindexed = cryst_orig.change_basis(change_of_basis_op)
if params.space_group is not None:
a, b, c = cryst_reindexed.get_real_space_vectors()
A_varying = [
cryst_reindexed.get_A_at_scan_point(i)
for i in range(cryst_reindexed.num_scan_points)
]
cryst_reindexed = Crystal(
a, b, c, space_group=params.space_group.group())
cryst_reindexed.set_A_at_scan_points(A_varying)
crystal.update(cryst_reindexed)
print("Old crystal:")
print(cryst_orig)
print()
print("New crystal:")
print(cryst_reindexed)
print()
print("Saving reindexed experimental models to %s" %
params.output.experiments)
experiments.as_file(params.output.experiments)
if len(reflections):
assert len(reflections) == 1
reflections = reflections[0]
miller_indices = reflections["miller_index"]
if params.hkl_offset is not None:
h, k, l = miller_indices.as_vec3_double().parts()
h += params.hkl_offset[0]
k += params.hkl_offset[1]
l += params.hkl_offset[2]
miller_indices = flex.miller_index(h.iround(), k.iround(),
l.iround())
non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices(
miller_indices)
if non_integral_indices.size() > 0:
print(
"Removing %i/%i reflections (change of basis results in non-integral indices)"
% (non_integral_indices.size(), miller_indices.size()))
sel = flex.bool(miller_indices.size(), True)
sel.set_selected(non_integral_indices, False)
miller_indices_reindexed = change_of_basis_op.apply(
miller_indices.select(sel))
reflections["miller_index"].set_selected(sel, miller_indices_reindexed)
reflections["miller_index"].set_selected(~sel, (0, 0, 0))
print("Saving reindexed reflections to %s" % params.output.reflections)
easy_pickle.dump(params.output.reflections, reflections)
def __init__(self,
experiments,
reflections,
uuid_cache_in,
params=None,
do_plot=False,
i_plot=None,
output_dir=None):
super(dials_cl_cosym_wrapper, self).__init__(
events=["run_cosym", "performed_unit_cell_clustering"])
if params is None:
params = phil_scope.extract()
self.params = params
self._reflections = []
for refl, expt in zip(reflections, experiments):
sel = get_selection_for_valid_image_ranges(refl, expt)
self._reflections.append(refl.select(sel))
self._experiments, self._reflections = self._filter_min_reflections(
experiments, self._reflections, uuid_cache_in)
self.ids_to_identifiers_map = {}
for table in self._reflections:
self.ids_to_identifiers_map.update(table.experiment_identifiers())
self.identifiers_to_ids_map = {
value: key
for key, value in self.ids_to_identifiers_map.items()
}
if len(self._experiments) > 1:
# perform unit cell clustering
identifiers = self._unit_cell_clustering(self._experiments)
if len(identifiers) < len(self._experiments):
logger.info(
"Selecting subset of %i datasets for cosym analysis: %s" %
(len(identifiers), str(identifiers)))
self._experiments, self._reflections = select_datasets_on_identifiers(
self._experiments,
self._reflections,
use_datasets=identifiers)
self.uuid_cache = [
self.uuid_cache[int(id)] for id in identifiers
]
# Map experiments and reflections to minimum cell
cb_ops = change_of_basis_ops_to_minimum_cell(
self._experiments,
params.lattice_symmetry_max_delta,
params.relative_length_tolerance,
params.absolute_angle_tolerance,
)
in_cb_ops = len(cb_ops)
exclude = [
expt.identifier for expt, cb_op in zip(self._experiments, cb_ops)
if not cb_op
]
if len(exclude):
logger.info(
"Rejecting {} datasets from cosym analysis "\
"(couldn't determine consistent cb_op to minimum cell):\n"\
"{}".format(len(exclude), exclude)
)
self._experiments, self._reflections = select_datasets_on_identifiers(
self._experiments, self._reflections, exclude_datasets=exclude)
cb_ops = list(filter(None, cb_ops))
ex_cb_ops = len(cb_ops)
#Normally we expect that all the cb_ops are the same (applicable for PSI with P63)
assertion_dict = {}
for cb_op in cb_ops:
key_ = cb_op.as_hkl()
assertion_dict[key_] = assertion_dict.get(key_, 0)
assertion_dict[key_] += 1
if len(assertion_dict) != 1:
# unexpected, there is normally only 1 cb operator to minimum cell
from libtbx.mpi4py import MPI
mpi_rank = MPI.COMM_WORLD.Get_rank()
mpi_size = MPI.COMM_WORLD.Get_size()
print(
"RANK %02d, # experiments %d, after exclusion %d, unexpectedly there are %d unique cb_ops: %s"
% (mpi_rank, in_cb_ops, ex_cb_ops, len(assertion_dict),
", ".join([
"%s:%d" % (key, assertion_dict[key])
for key in assertion_dict
])))
# revisit with different use cases later
# In fact we need all cb_ops to match because the user might supply
# a custom reindexing operator and we need to consistently tranform
# it from the conventional basis into the minimum basis. Therefore,
# force them all to match, but make sure user is aware.
if not params.single_cb_op_to_minimum:
raise RuntimeError(
'There are >1 different cb_ops to minimum and \
cosym.single_cb_op_to_minimum is not True')
else:
best_cb_op_str = max(assertion_dict, key=assertion_dict.get)
best_cb_op = None
for cb_op in cb_ops:
if cb_op.as_hkl() == best_cb_op_str:
best_cb_op = cb_op
break
assert best_cb_op is not None
cb_ops = [best_cb_op] * len(cb_ops)
self.cb_op_to_minimum = cb_ops
# Eliminate reflections that are systematically absent due to centring
# of the lattice, otherwise they would lead to non-integer miller indices
# when reindexing to a primitive setting
self._reflections = eliminate_sys_absent(self._experiments,
self._reflections)
self._experiments, self._reflections = apply_change_of_basis_ops(
self._experiments, self._reflections, cb_ops)
# transform models into miller arrays
datasets = filtered_arrays_from_experiments_reflections(
self.experiments,
self.reflections,
outlier_rejection_after_filter=False,
partiality_threshold=params.partiality_threshold,
)
datasets = [
ma.as_anomalous_array().merge_equivalents().array()
for ma in datasets
]
# opportunity here to subclass as defined above, instead of the dials-implemented version
self.cosym_analysis = CosymAnalysis(
datasets,
self.params,
do_plot=do_plot,
i_plot=i_plot,
plot_fname=self.params.plot.filename,
plot_format=self.params.plot.format,
output_dir=output_dir,
cb_op=cb_ops[0])
def run(args=None,
phil=phil_scope): # type: (List[str], libtbx.phil.scope) -> None
usage = "dials.missing_reflections [options] scaled.expt scaled.refl"
parser = OptionParser(
usage=usage,
phil=phil,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=__doc__,
)
params, options = parser.parse_args(args=args, show_diff_phil=False)
# Configure the logging.
dials.util.log.config(options.verbose)
logger.info(dials_version())
# Log the difference between the PHIL scope definition and the active PHIL scope,
# which will include the parsed user inputs.
diff_phil = parser.diff_phil.as_str()
if diff_phil:
logger.info("The following parameters have been modified:\n%s",
diff_phil)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if not experiments or not reflections:
parser.print_help()
return
if len(reflections) != 1 and len(experiments) != len(reflections):
sys.exit(
"Number of experiments must equal the number of reflection tables")
from dials.util.multi_dataset_handling import (
assign_unique_identifiers,
parse_multiple_datasets,
)
reflections = parse_multiple_datasets(reflections)
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
if all("inverse_scale_factor" in refl for refl in reflections):
# Assume all arrays have been scaled
miller_array = scaled_data_as_miller_array(reflections,
experiments,
anomalous_flag=False)
else:
# Else get the integrated intensities
miller_arrays = filtered_arrays_from_experiments_reflections(
experiments,
reflections,
)
miller_array = miller_arrays[0]
for ma in miller_arrays[1:]:
miller_array = miller_array.concatenate(ma)
if params.d_min or params.d_max:
miller_array = miller_array.resolution_filter(d_min=params.d_min,
d_max=params.d_max)
# Print overall summary of input miller array
s = io.StringIO()
ma_unique = miller_array.unique_under_symmetry()
ma_unique.show_comprehensive_summary(f=s)
logger.info(f"\n{s.getvalue()}")
# Get the regions of missing reflections
complete_set, unique_ms = missing_reflections.connected_components(
miller_array)
unique_ms = [
ms for ms in unique_ms if ms.size() >= params.min_component_size
]
# Print some output for user
if len(unique_ms):
logger.info(
"The following connected regions of missing reflections have been identified:"
)
n_expected = complete_set.size()
rows = []
for ms in unique_ms:
d_max, d_min = (uctbx.d_star_sq_as_d(ds2)
for ds2 in ms.min_max_d_star_sq())
rows.append([
ms.size(),
f"{100 * ms.size() / n_expected:.1f}",
f"{d_max:.2f}-{d_min:.2f}",
])
logger.info(
tabulate(
rows,
headers=["# reflections", "% missing",
"Resolution range (Å)"]))
else:
logger.info("No connected regions of missing reflections identified")
def run(args=sys.argv[1:]):
# Create the parser
parser = OptionParser(
# usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
# epilog=help_message,
)
# Parse the command line
params, options, args = parser.parse_args(args=args,
show_diff_phil=False,
return_unhandled=True)
# Configure the logging
xia2.Handlers.Streams.setup_logging(logfile=params.output.log,
verbose=options.verbose)
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
if params.unit_cell is not None:
unit_cell = params.unit_cell
crystal_symmetry = crystal.symmetry(unit_cell=unit_cell)
else:
crystal_symmetry = None
if len(params.input.experiments):
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
reflections = parse_multiple_datasets(reflections)
if len(experiments) != len(reflections):
raise sys.exit(
"Mismatched number of experiments and reflection tables found: %s & %s."
% (len(experiments), len(reflections)))
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
# transform models into miller arrays
intensities, batches = filtered_arrays_from_experiments_reflections(
experiments,
reflections,
outlier_rejection_after_filter=False,
partiality_threshold=0.99,
return_batches=True,
)
if args and os.path.isfile(args[0]):
result = any_reflection_file(args[0])
unmerged_intensities = None
batches_all = None
for ma in result.as_miller_arrays(merge_equivalents=False,
crystal_symmetry=crystal_symmetry):
if ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]:
assert ma.anomalous_flag()
unmerged_intensities = ma
elif ma.info().labels == ["I", "SIGI"]:
assert not ma.anomalous_flag()
unmerged_intensities = ma
elif ma.info().labels == ["BATCH"]:
batches_all = ma
assert batches_all is not None
assert unmerged_intensities is not None
sel = unmerged_intensities.sigmas() > 0
unmerged_intensities = unmerged_intensities.select(sel).set_info(
unmerged_intensities.info())
batches_all = batches_all.select(sel)
id_to_batches = None
if len(params.batch) > 0:
id_to_batches = {}
for b in params.batch:
assert b.id is not None
assert b.range is not None
assert b.id not in id_to_batches, "Duplicate batch id: %s" % b.id
id_to_batches[b.id] = b.range
separate = separate_unmerged(unmerged_intensities,
batches_all,
id_to_batches=id_to_batches)
batches = list(separate.batches.values())
intensities = list(separate.intensities.values())
result = DeltaCcHalf(
intensities,
batches,
n_bins=params.n_bins,
d_min=params.d_min,
cc_one_half_method=params.cc_one_half_method,
group_size=params.group_size,
)
logger.info(tabulate(result.get_table(), headers="firstrow"))
hist_filename = "delta_cc_hist.png"
logger.info("Saving histogram to %s" % hist_filename)
result.plot_histogram(hist_filename)
normalised_scores_filename = "normalised_scores.png"
logger.info("Saving normalised scores to %s" % normalised_scores_filename)
result.plot_normalised_scores(normalised_scores_filename)
Citations.cite("delta_cc_half")
for citation in Citations.get_citations_acta():
logger.info(citation)
def __init__(self, experiments, reflections, params=None):
super(cosym, self).__init__(
events=["run_cosym", "performed_unit_cell_clustering"])
if params is None:
params = phil_scope.extract()
self.params = params
self._reflections = []
for refl, expt in zip(reflections, experiments):
sel = get_selection_for_valid_image_ranges(refl, expt)
self._reflections.append(refl.select(sel))
self._experiments, self._reflections = self._filter_min_reflections(
experiments, self._reflections)
self.ids_to_identifiers_map = {}
for table in self._reflections:
self.ids_to_identifiers_map.update(table.experiment_identifiers())
self.identifiers_to_ids_map = {
value: key
for key, value in self.ids_to_identifiers_map.items()
}
if len(self._experiments) > 1:
# perform unit cell clustering
identifiers = self._unit_cell_clustering(self._experiments)
if len(identifiers) < len(self._experiments):
logger.info(
"Selecting subset of %i datasets for cosym analysis: %s",
len(identifiers),
str(identifiers),
)
self._experiments, self._reflections = select_datasets_on_identifiers(
self._experiments,
self._reflections,
use_datasets=identifiers)
# Map experiments and reflections to minimum cell
cb_ops = change_of_basis_ops_to_minimum_cell(
self._experiments,
params.lattice_symmetry_max_delta,
params.relative_length_tolerance,
params.absolute_angle_tolerance,
)
exclude = [
expt.identifier for expt, cb_op in zip(self._experiments, cb_ops)
if not cb_op
]
if len(exclude):
logger.info(
f"Rejecting {len(exclude)} datasets from cosym analysis "
f"(couldn't determine consistent cb_op to minimum cell):\n"
f"{exclude}", )
self._experiments, self._reflections = select_datasets_on_identifiers(
self._experiments, self._reflections, exclude_datasets=exclude)
cb_ops = list(filter(None, cb_ops))
# Eliminate reflections that are systematically absent due to centring
# of the lattice, otherwise they would lead to non-integer miller indices
# when reindexing to a primitive setting
self._reflections = eliminate_sys_absent(self._experiments,
self._reflections)
self._experiments, self._reflections = apply_change_of_basis_ops(
self._experiments, self._reflections, cb_ops)
# transform models into miller arrays
datasets = filtered_arrays_from_experiments_reflections(
self.experiments,
self.reflections,
outlier_rejection_after_filter=False,
partiality_threshold=params.partiality_threshold,
)
datasets = [
ma.as_anomalous_array().merge_equivalents().array()
for ma in datasets
]
self.cosym_analysis = CosymAnalysis(datasets, self.params)
def run(args=None):
import libtbx.load_env
from dials.util import Sorry
usage = "dials.reindex [options] indexed.expt indexed.refl"
parser = ArgumentParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(args, show_diff_phil=True)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments
)
if len(experiments) == 0 and len(reflections) == 0:
parser.print_help()
return
if params.change_of_basis_op is None:
raise Sorry("Please provide a change_of_basis_op.")
reference_crystal = None
if params.reference.experiments is not None:
from dxtbx.serialize import load
reference_experiments = load.experiment_list(
params.reference.experiments, check_format=False
)
if len(reference_experiments.crystals()) == 1:
reference_crystal = reference_experiments.crystals()[0]
else:
# first check sg all same
sgs = [
expt.crystal.get_space_group().type().number() for expt in experiments
]
if len(set(sgs)) > 1:
raise Sorry(
"""The reference experiments have different space groups:
space group numbers found: %s
Please reanalyse the data so that space groups are consistent,
(consider using dials.reindex, dials.symmetry or dials.cosym)"""
% ", ".join(map(str, set(sgs)))
)
reference_crystal = reference_experiments.crystals()[0]
reference_crystal.unit_cell = determine_best_unit_cell(
reference_experiments
)
if params.reference.reflections is not None:
# First check that we have everything as expected for the reference reindexing
if params.reference.experiments is None:
raise Sorry(
"""For reindexing against a reference dataset, a reference
experiments file must also be specified with the option: reference.experiments= """
)
if not os.path.exists(params.reference.reflections):
raise Sorry("Could not locate reference dataset reflection file")
reference_reflections = flex.reflection_table().from_file(
params.reference.reflections
)
test_reflections = reflections[0]
if (
reference_crystal.get_space_group().type().number()
!= experiments.crystals()[0].get_space_group().type().number()
):
raise Sorry("Space group of input does not match reference")
# Set some flags to allow filtering, if wanting to reindex against
# reference with data that has not yet been through integration
if (
test_reflections.get_flags(test_reflections.flags.integrated_sum).count(
True
)
== 0
):
assert (
"intensity.sum.value" in test_reflections
), "No 'intensity.sum.value' in reflections"
test_reflections.set_flags(
flex.bool(test_reflections.size(), True),
test_reflections.flags.integrated_sum,
)
if (
reference_reflections.get_flags(
reference_reflections.flags.integrated_sum
).count(True)
== 0
):
assert (
"intensity.sum.value" in test_reflections
), "No 'intensity.sum.value in reference reflections"
reference_reflections.set_flags(
flex.bool(reference_reflections.size(), True),
reference_reflections.flags.integrated_sum,
)
# Make miller array of the two datasets
try:
test_miller_set = filtered_arrays_from_experiments_reflections(
experiments, [test_reflections]
)[0]
except ValueError:
raise Sorry("No reflections remain after filtering the test dataset")
try:
reference_miller_set = filtered_arrays_from_experiments_reflections(
reference_experiments, [reference_reflections]
)[0]
except ValueError:
raise Sorry("No reflections remain after filtering the reference dataset")
from dials.algorithms.symmetry.reindex_to_reference import (
determine_reindex_operator_against_reference,
)
change_of_basis_op = determine_reindex_operator_against_reference(
test_miller_set, reference_miller_set
)
elif len(experiments) and params.change_of_basis_op is libtbx.Auto:
if reference_crystal is not None:
if len(experiments.crystals()) > 1:
raise Sorry("Only one crystal can be processed at a time")
from dials.algorithms.indexing.compare_orientation_matrices import (
difference_rotation_matrix_axis_angle,
)
cryst = experiments.crystals()[0]
R, axis, angle, change_of_basis_op = difference_rotation_matrix_axis_angle(
cryst, reference_crystal
)
print(f"Change of basis op: {change_of_basis_op}")
print("Rotation matrix to transform input crystal to reference::")
print(R.mathematica_form(format="%.3f", one_row_per_line=True))
print(
f"Rotation of {angle:.3f} degrees",
"about axis (%.3f, %.3f, %.3f)" % axis,
)
elif len(reflections):
assert len(reflections) == 1
# always re-map reflections to reciprocal space
refl = reflections.deep_copy()
refl.centroid_px_to_mm(experiments)
refl.map_centroids_to_reciprocal_space(experiments)
# index the reflection list using the input experiments list
refl["id"] = flex.int(len(refl), -1)
index = AssignIndicesGlobal(tolerance=0.2)
index(refl, experiments)
hkl_expt = refl["miller_index"]
hkl_input = reflections[0]["miller_index"]
change_of_basis_op = derive_change_of_basis_op(hkl_input, hkl_expt)
# reset experiments list since we don't want to reindex this
experiments = []
else:
change_of_basis_op = sgtbx.change_of_basis_op(params.change_of_basis_op)
if len(experiments):
space_group = params.space_group
if space_group is not None:
space_group = space_group.group()
try:
experiments = reindex_experiments(
experiments, change_of_basis_op, space_group=space_group
)
except RuntimeError as e:
# Only catch specific errors here
if "Unsuitable value for rational rotation matrix." in str(e):
original_message = str(e).split(":")[-1].strip()
sys.exit(f"Error: {original_message} Is your change_of_basis_op valid?")
raise
print(f"Saving reindexed experimental models to {params.output.experiments}")
experiments.as_file(params.output.experiments)
if len(reflections):
assert len(reflections) == 1
reflections = reflections[0]
miller_indices = reflections["miller_index"]
if params.hkl_offset is not None:
h, k, l = miller_indices.as_vec3_double().parts()
h += params.hkl_offset[0]
k += params.hkl_offset[1]
l += params.hkl_offset[2]
miller_indices = flex.miller_index(h.iround(), k.iround(), l.iround())
non_integral_indices = change_of_basis_op.apply_results_in_non_integral_indices(
miller_indices
)
if non_integral_indices.size() > 0:
print(
"Removing %i/%i reflections (change of basis results in non-integral indices)"
% (non_integral_indices.size(), miller_indices.size())
)
sel = flex.bool(miller_indices.size(), True)
sel.set_selected(non_integral_indices, False)
miller_indices_reindexed = change_of_basis_op.apply(miller_indices.select(sel))
reflections["miller_index"].set_selected(sel, miller_indices_reindexed)
reflections["miller_index"].set_selected(~sel, (0, 0, 0))
print(f"Saving reindexed reflections to {params.output.reflections}")
reflections.as_file(params.output.reflections)
def __init__(self,
experiments,
reflections,
uuid_cache_in,
params=None,
do_plot=False,
i_plot=None,
output_dir=None):
super(dials_cl_cosym_wrapper, self).__init__(
events=["run_cosym", "performed_unit_cell_clustering"])
if params is None:
params = phil_scope.extract()
self.params = params
self._reflections = []
for refl, expt in zip(reflections, experiments):
sel = get_selection_for_valid_image_ranges(refl, expt)
self._reflections.append(refl.select(sel))
self._experiments, self._reflections = self._filter_min_reflections(
experiments, self._reflections, uuid_cache_in)
self.ids_to_identifiers_map = {}
for table in self._reflections:
self.ids_to_identifiers_map.update(table.experiment_identifiers())
self.identifiers_to_ids_map = {
value: key
for key, value in self.ids_to_identifiers_map.items()
}
if len(self._experiments) > 1:
# perform unit cell clustering
identifiers = self._unit_cell_clustering(self._experiments)
if len(identifiers) < len(self._experiments):
logger.info(
"Selecting subset of %i datasets for cosym analysis: %s" %
(len(identifiers), str(identifiers)))
self._experiments, self._reflections = select_datasets_on_identifiers(
self._experiments,
self._reflections,
use_datasets=identifiers)
# Map experiments and reflections to minimum cell
cb_ops = change_of_basis_ops_to_minimum_cell(
self._experiments,
params.lattice_symmetry_max_delta,
params.relative_length_tolerance,
params.absolute_angle_tolerance,
)
exclude = [
expt.identifier for expt, cb_op in zip(self._experiments, cb_ops)
if not cb_op
]
if len(exclude):
logger.info(
"Rejecting {} datasets from cosym analysis "\
"(couldn't determine consistent cb_op to minimum cell):\n"\
"{}".format(len(exclude), exclude)
)
self._experiments, self._reflections = select_datasets_on_identifiers(
self._experiments, self._reflections, exclude_datasets=exclude)
cb_ops = list(filter(None, cb_ops))
# Eliminate reflections that are systematically absent due to centring
# of the lattice, otherwise they would lead to non-integer miller indices
# when reindexing to a primitive setting
self._reflections = eliminate_sys_absent(self._experiments,
self._reflections)
self._experiments, self._reflections = apply_change_of_basis_ops(
self._experiments, self._reflections, cb_ops)
# transform models into miller arrays
datasets = filtered_arrays_from_experiments_reflections(
self.experiments,
self.reflections,
outlier_rejection_after_filter=False,
partiality_threshold=params.partiality_threshold,
)
datasets = [
ma.as_anomalous_array().merge_equivalents().array()
for ma in datasets
]
# opportunity here to subclass as defined above, instead of the dials-implemented version
self.cosym_analysis = CosymAnalysis(
datasets,
self.params,
do_plot=do_plot,
i_plot=i_plot,
plot_fname=self.params.plot.filename,
plot_format=self.params.plot.format,
output_dir=output_dir)
#Fixed in subclass: parent class apparently erases the knowledge of input-to-minimum cb_ops.
# without storing the op in self, we can never trace back to input setting.
self.cb_op_to_minimum = cb_ops
#Not sure yet, we may be assuming that all the cb_ops are the same (applicable for PSI with P63)
assertion_set = set(cb_ops)
assert len(
assertion_set
) == 1 # guarantees all are the same; revisit with different use cases later
