def test_slice_reflections():
r = flex.reflection_table()
r["id"] = flex.int([0, 0, 0, 1, 1, 1, 2, 2, 2])
image_number = [0, 1, 2, 0, 1, 2, 0, 1, 2]
r["xyzobs.px.value"] = flex.vec3_double(
zip([0] * len(r), [0] * len(r), image_number))
sliced_r = slice_reflections(r, [(1, 2), (1, 1), (2, 3)])
assert list(sliced_r["id"]) == [0, 0, 1, 2, 2]
assert list(
sliced_r["xyzobs.px.value"].parts()[2]) == [0.0, 1.0, 0.0, 1.0, 2.0]
def run(args):
usage = "dials.search_beam_position [options] imported.expt strong.refl"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(args, show_diff_phil=False)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
if len(experiments) == 0 or len(reflections) == 0:
parser.print_help()
exit(0)
# Configure the logging
log.config(logfile=params.output.log)
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
if params.seed is not None:
flex.set_random_seed(params.seed)
random.seed(params.seed)
if params.nproc is libtbx.Auto:
params.nproc = libtbx.introspection.number_of_processors()
imagesets = experiments.imagesets()
# Split all the refln tables by ID, corresponding to the respective imagesets
reflections = [
refl_unique_id for refl in reflections
for refl_unique_id in refl.split_by_experiment_id()
]
assert len(imagesets) > 0
assert len(reflections) == len(imagesets)
if params.image_range is not None and len(params.image_range) > 0:
reflections = [
slice_reflections(refl, params.image_range) for refl in reflections
]
for i in range(params.n_macro_cycles):
if params.n_macro_cycles > 1:
logger.info("Starting macro cycle %i" % (i + 1))
experiments = discover_better_experimental_model(
experiments,
reflections,
params,
nproc=params.nproc,
d_min=params.d_min,
mm_search_scope=params.mm_search_scope,
wide_search_binning=params.wide_search_binning,
plot_search_scope=params.plot_search_scope,
)
logger.info("")
logger.info("Saving optimised experiments to %s" %
params.output.experiments)
experiments.as_file(params.output.experiments)
def index(experiments, reflections, params):
"""
Index the input experiments and reflections.
Args:
experiments: The experiments to index
reflections (list): A list of reflection tables containing strong spots
params: An instance of the indexing phil scope
Returns:
(tuple): tuple containing:
experiments: The indexed experiment list
reflections (dials.array_family.flex.reflection_table):
The indexed reflections
"""
if experiments.crystals()[0] is not None:
known_crystal_models = experiments.crystals()
else:
known_crystal_models = None
if len(reflections) == 0:
raise ValueError("No reflection lists found in input")
elif len(reflections) == 1:
reflections[0]["imageset_id"] = reflections[0]["id"]
elif len(reflections) > 1:
assert len(reflections) == len(experiments)
for i in range(len(reflections)):
reflections[i]["imageset_id"] = flex.int(len(reflections[i]), i)
if i > 0:
reflections[0].extend(reflections[i])
reflections = reflections[0]
# If there are scan and goniometer objects present but the oscillation angle is zero
# then expt.scan and expt.goniometer to None, as the behaviour of some downstream
# algorithms depend on the presence/absence of these objects
for expt in experiments:
if (expt.goniometer is not None and expt.scan is not None
and expt.scan.is_still()):
expt.imageset = ImageSetFactory.imageset_from_anyset(expt.imageset)
expt.goniometer = None
expt.scan = None
if params.indexing.image_range:
reflections = slice_reflections(reflections,
params.indexing.image_range)
if len(experiments) == 1 or params.indexing.joint_indexing:
indexed_experiments, indexed_reflections = _index_experiments(
experiments,
reflections,
copy.deepcopy(params),
known_crystal_models=known_crystal_models,
)
else:
indexed_experiments = ExperimentList()
indexed_reflections = flex.reflection_table()
with concurrent.futures.ProcessPoolExecutor(
max_workers=params.indexing.nproc) as pool:
futures = []
for i_expt, expt in enumerate(experiments):
refl = reflections.select(reflections["imageset_id"] == i_expt)
refl["imageset_id"] = flex.size_t(len(refl), 0)
futures.append(
pool.submit(
_index_experiments,
ExperimentList([expt]),
refl,
copy.deepcopy(params),
known_crystal_models=known_crystal_models,
))
for future in concurrent.futures.as_completed(futures):
try:
idx_expts, idx_refl = future.result()
except Exception as e:
print(e)
else:
if idx_expts is None:
continue
for j_expt, _ in enumerate(idx_expts):
sel = idx_refl["id"] == j_expt
idx_refl["id"].set_selected(
sel,
len(indexed_experiments) + j_expt)
idx_refl["imageset_id"] = flex.size_t(
len(idx_refl), i_expt)
indexed_reflections.extend(idx_refl)
indexed_experiments.extend(idx_expts)
return indexed_experiments, indexed_reflections
def test_trim_scans_to_observations(dials_data):
# Use 4 scan data for this test
data_dir = dials_data("l_cysteine_dials_output")
experiments = ExperimentListFactory.from_json_file(
(data_dir / "indexed.expt").strpath, check_format=False)
reflections = flex.reflection_table.from_file(
(data_dir / "indexed.refl").strpath)
# Check the image and oscillation range are what we expect
image_ranges = [e.scan.get_image_range() for e in experiments]
osc_ranges = [e.scan.get_oscillation_range() for e in experiments]
for a, b in zip(image_ranges, [(1, 1700), (1, 1700), (1, 1700),
(1, 1800)]):
assert a == b
for a, b in zip(osc_ranges, [(-145.0, 25.0), (-145.0, 25.0),
(-145.0, 25.0), (0.0, 180.0)]):
assert a == pytest.approx(b)
# If image range unchanged, nothing should happen
trim_expt = _trim_scans_to_observations(deepcopy(experiments), reflections)
new_im_ranges = [e.scan.get_image_range() for e in trim_expt]
for a, b in zip(image_ranges, new_im_ranges):
assert a == b
# Slice 20 images off head and tail
sliced_ranges = [(r[0] + 20, r[1] - 20) for r in image_ranges]
sliced = slice_reflections(reflections, sliced_ranges)
# Now trimmed scans should have array ranges equal to their min, max
# shoebox z coords
trim_expt = _trim_scans_to_observations(deepcopy(experiments), sliced)
new_array_ranges = [e.scan.get_array_range() for e in trim_expt]
for i, e in enumerate(trim_expt):
refs = sliced.select(sliced["id"] == i)
bb = refs["shoebox"].bounding_boxes()
z_min, z_max = bb.parts()[4:]
assert new_array_ranges[i] == (min(z_min), max(z_max))
# Oscillation ranges should be trimmed so that the associated angle is the
# same in the original and trimmed scans
new_osc_ranges = [e.scan.get_oscillation_range() for e in trim_expt]
for exp, r1, r2 in zip(experiments, new_array_ranges, new_osc_ranges):
assert exp.scan.get_angle_from_array_index(r1[0]) == pytest.approx(
r2[0])
assert exp.scan.get_angle_from_array_index(r1[1]) == pytest.approx(
r2[1])
# Now delete shoebox data. Trimmed scans will be wider than the observed
# range by >0.5 deg at each end
del sliced["shoebox"]
trim_expt = _trim_scans_to_observations(deepcopy(experiments), sliced)
new_array_ranges = [e.scan.get_array_range() for e in trim_expt]
for i, e in enumerate(trim_expt):
refs = sliced.select(sliced["id"] == i)
z = refs["xyzobs.px.value"].parts()[2]
im_width = e.scan.get_oscillation()[1]
assert ((min(z) - new_array_ranges[i][0]) / im_width) > 0.5
assert ((new_array_ranges[i][1] - max(z)) / im_width) > 0.5
# Oscillation ranges should be trimmed so that the associated angle is the
# same in the original and trimmed scans
new_osc_ranges = [e.scan.get_oscillation_range() for e in trim_expt]
for exp, r1, r2 in zip(experiments, new_array_ranges, new_osc_ranges):
assert exp.scan.get_angle_from_array_index(r1[0]) == pytest.approx(
r2[0])
assert exp.scan.get_angle_from_array_index(r1[1]) == pytest.approx(
r2[1])
def run(self):
"""Execute the script."""
from dials.util.options import reflections_and_experiments_from_files
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
# Try to load the models and data
slice_exps = len(experiments) > 0
slice_refs = len(reflections) > 0
# Catch case of nothing to do
if not slice_exps and not slice_refs:
print("No suitable input provided")
self.parser.print_help()
return
if reflections:
if len(reflections) > 1:
raise Sorry(
"Only one reflections list can be imported at present")
reflections = reflections[0]
# calculate frame numbers if needed
if experiments:
reflections = calculate_frame_numbers(reflections, experiments)
# if we still don't have the right column give up
if "xyzobs.px.value" not in reflections:
raise Sorry(
"These reflections do not have frame numbers set, and "
"there are no experiments provided to calculate these.")
# set trivial case where no scan range is provided at all
if not params.image_range:
params.image_range = [None]
# check if slicing into blocks
if params.block_size is not None:
if slice_exps:
if len(experiments) > 1:
raise Sorry(
"For slicing into blocks please provide a single "
"scan only")
scan = experiments[0].scan
# Having extracted the scan, calculate the blocks
params.image_range = calculate_block_ranges(
scan, params.block_size)
# Do the slicing then recombine
if slice_exps:
sliced = [
slice_experiments(experiments, [sr])[0]
for sr in params.image_range
]
sliced_experiments = ExperimentList()
for exp in sliced:
sliced_experiments.append(exp)
# slice reflections if present
if slice_refs:
sliced = [
slice_reflections(reflections, [sr])
for sr in params.image_range
]
sliced_reflections = sliced[0]
for i, rt in enumerate(sliced[1:]):
rt["id"] += i + 1 # set id
sliced_reflections.extend(rt)
else:
# slice each dataset into the requested subset
if slice_exps:
sliced_experiments = slice_experiments(experiments,
params.image_range)
if slice_refs:
sliced_reflections = slice_reflections(reflections,
params.image_range)
# Save sliced experiments
if slice_exps:
output_experiments_filename = params.output.experiments_filename
if output_experiments_filename is None:
# take first filename as template
bname = basename(params.input.experiments[0].filename)
bname = splitext(bname)[0]
if not bname:
bname = "experiments"
if len(params.image_range
) == 1 and params.image_range[0] is not None:
ext = "_{0}_{1}.expt".format(*params.image_range[0])
else:
ext = "_sliced.expt"
output_experiments_filename = bname + ext
print("Saving sliced experiments to {}".format(
output_experiments_filename))
sliced_experiments.as_file(output_experiments_filename)
# Save sliced reflections
if slice_refs:
output_reflections_filename = params.output.reflections_filename
if output_reflections_filename is None:
# take first filename as template
bname = basename(params.input.reflections[0].filename)
bname = splitext(bname)[0]
if not bname:
bname = "reflections"
if len(params.image_range
) == 1 and params.image_range[0] is not None:
ext = "_{0}_{1}.refl".format(*params.image_range[0])
else:
ext = "_sliced.refl"
output_reflections_filename = bname + ext
print("Saving sliced reflections to {0}".format(
output_reflections_filename))
sliced_reflections.as_file(output_reflections_filename)
return
def index(experiments, reflections, params):
"""
Index the input experiments and reflections.
Args:
experiments: The experiments to index
reflections (list): A list of reflection tables containing strong spots
params: An instance of the indexing phil scope
Returns:
(tuple): tuple containing:
experiments: The indexed experiment list
reflections (dials.array_family.flex.reflection_table):
The indexed reflections
Raises:
ValueError: `reflections` is an empty list or `experiments` contains a
combination of sequence and stills data.
dials.algorithms.indexing.DialsIndexError: Indexing failed.
"""
if experiments.crystals()[0] is not None:
known_crystal_models = experiments.crystals()
else:
known_crystal_models = None
if len(reflections) == 0:
raise ValueError("No reflection lists found in input")
elif len(reflections) == 1:
if "imageset_id" not in reflections[0]:
reflections[0]["imageset_id"] = reflections[0]["id"]
elif len(reflections) > 1:
assert len(reflections) == len(experiments)
for i in range(len(reflections)):
reflections[i]["imageset_id"] = flex.int(len(reflections[i]), i)
if i > 0:
reflections[0].extend(reflections[i])
reflections = reflections[0]
if params.indexing.image_range:
reflections = slice_reflections(reflections, params.indexing.image_range)
if len(experiments) == 1 or params.indexing.joint_indexing:
indexed_experiments, indexed_reflections = _index_experiments(
experiments,
reflections,
copy.deepcopy(params),
known_crystal_models=known_crystal_models,
)
else:
indexed_experiments = ExperimentList()
indexed_reflections = flex.reflection_table()
with concurrent.futures.ProcessPoolExecutor(
max_workers=params.indexing.nproc
) as pool:
futures = []
for i_expt, expt in enumerate(experiments):
refl = reflections.select(reflections["imageset_id"] == i_expt)
refl["imageset_id"] = flex.size_t(len(refl), 0)
futures.append(
pool.submit(
_index_experiments,
ExperimentList([expt]),
refl,
copy.deepcopy(params),
known_crystal_models=known_crystal_models,
)
)
tables_list = []
for future in concurrent.futures.as_completed(futures):
try:
idx_expts, idx_refl = future.result()
except Exception as e:
print(e)
else:
if idx_expts is None:
continue
# Update the experiment ids by incrementing by the number of indexed
# experiments already in the list
##FIXME below, is i_expt correct - or should it be the
# index of the 'future'?
idx_refl["imageset_id"] = flex.size_t(idx_refl.size(), i_expt)
tables_list.append(idx_refl)
indexed_experiments.extend(idx_expts)
tables_list = renumber_table_id_columns(tables_list)
for table in tables_list:
indexed_reflections.extend(table)
return indexed_experiments, indexed_reflections
def run(args):
usage = "dials.search_beam_position [options] imported.expt strong.refl"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=False)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0 or len(reflections) == 0:
parser.print_help()
exit(0)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
if params.seed is not None:
flex.set_random_seed(params.seed)
random.seed(params.seed)
imagesets = experiments.imagesets()
# Split all the refln tables by ID, corresponding to the respective imagesets
reflections = [
refl_unique_id for refl in reflections
for refl_unique_id in refl.split_by_experiment_id()
]
assert len(imagesets) > 0
assert len(reflections) == len(imagesets)
if params.image_range is not None and len(params.image_range) > 0:
reflections = [
slice_reflections(refl, params.image_range) for refl in reflections
]
dps_params = dps_phil_scope.extract()
# for development, we want an exhaustive plot of beam probability map:
dps_params.indexing.plot_search_scope = params.plot_search_scope
dps_params.indexing.mm_search_scope = params.mm_search_scope
for i in range(params.n_macro_cycles):
if params.n_macro_cycles > 1:
logger.info("Starting macro cycle %i" % (i + 1))
new_detector, new_beam = discover_better_experimental_model(
imagesets,
reflections,
params,
dps_params,
nproc=params.nproc,
wide_search_binning=params.wide_search_binning,
)
for experiment in experiments:
experiment.beam = new_beam
experiment.detector = new_detector
logger.info("")
logger.info("Saving optimized experiments to %s" %
params.output.experiments)
dump.experiment_list(experiments, params.output.experiments)
def __init__(self, experiments, reflections, params):
self._params = params
if experiments.crystals()[0] is not None:
known_crystal_models = experiments.crystals()
else:
known_crystal_models = None
if len(reflections) == 0:
raise Sorry("No reflection lists found in input")
elif len(reflections) == 1:
reflections[0]["imageset_id"] = reflections[0]["id"]
elif len(reflections) > 1:
assert len(reflections) == len(experiments)
for i in range(len(reflections)):
reflections[i]["imageset_id"] = flex.int(
len(reflections[i]), i)
if i > 0:
reflections[0].extend(reflections[i])
reflections = reflections[0]
for expt in experiments:
if (expt.goniometer is not None and expt.scan is not None
and expt.scan.get_oscillation()[1] == 0):
expt.goniometer = None
expt.scan = None
if self._params.indexing.image_range:
reflections = slice_reflections(reflections,
self._params.indexing.image_range)
if len(experiments) == 1 or self._params.indexing.joint_indexing:
try:
self._indexed_experiments, self._indexed_reflections = index_experiments(
experiments,
reflections,
copy.deepcopy(params),
known_crystal_models=known_crystal_models,
)
except DialsIndexError as e:
raise Sorry(e.message)
else:
self._indexed_experiments = ExperimentList()
self._indexed_reflections = flex.reflection_table()
import concurrent.futures
with concurrent.futures.ProcessPoolExecutor(
max_workers=params.indexing.nproc) as pool:
futures = []
for i_expt, expt in enumerate(experiments):
refl = reflections.select(
reflections["imageset_id"] == i_expt)
refl["imageset_id"] = flex.size_t(len(refl), 0)
futures.append(
pool.submit(
index_experiments,
ExperimentList([expt]),
refl,
copy.deepcopy(params),
known_crystal_models=known_crystal_models,
))
for future in concurrent.futures.as_completed(futures):
try:
idx_expts, idx_refl = future.result()
except Exception as e:
print(e)
else:
if idx_expts is None:
continue
for j_expt, _ in enumerate(idx_expts):
sel = idx_refl["id"] == j_expt
idx_refl["id"].set_selected(
sel,
len(self._indexed_experiments) + j_expt)
idx_refl["imageset_id"] = flex.size_t(
len(idx_refl), i_expt)
self._indexed_reflections.extend(idx_refl)
self._indexed_experiments.extend(idx_expts)
