def test_experimentlist_change_basis(dials_data):
experiments = ExperimentList()
for i in range(4):
experiments.extend(
ExperimentList.from_file(
dials_data("vmxi_proteinase_k_sweeps") /
("experiments_%i.expt" % i),
check_format=False,
))
reindexed_uc = (68.368, 103.968, 68.368, 90.000, 90.000, 90.000)
reindexed_sg = sgtbx.space_group_info("P 4 2 2 (b,c,a)").group()
cb_op = sgtbx.change_of_basis_op("-a,-c,-b")
for cb_op in (cb_op, [cb_op] * len(experiments)):
expts_rdx = experiments.change_basis(cb_op)
for expt in expts_rdx:
assert expt.crystal.get_unit_cell().parameters() == pytest.approx(
reindexed_uc, abs=0.1)
assert expt.crystal.get_space_group() == reindexed_sg
experiments.change_basis(cb_op, in_place=True)
for expt in experiments:
assert expt.crystal.get_unit_cell().parameters() == pytest.approx(
reindexed_uc, abs=0.1)
assert expt.crystal.get_space_group() == reindexed_sg
with pytest.raises(AssertionError):
experiments.change_basis([cb_op, cb_op])
def protk_experiments_and_reflections(dials_data):
data_dir = dials_data("multi_crystal_proteinase_k")
# Load experiments
experiments = ExperimentList()
for expt_file in sorted(f.strpath
for f in data_dir.listdir("experiments*.json")):
experiments.extend(load.experiment_list(expt_file, check_format=False))
# Load reflection tables
reflections = [
flex.reflection_table.from_file(refl_file) for refl_file in sorted(
f.strpath for f in data_dir.listdir("reflections*.pickle"))
]
# Setup experiment identifiers
reflections = parse_multiple_datasets(reflections)
experiments, reflections = assign_unique_identifiers(
experiments, reflections)
# Combine into single ExperimentList
reflections_all = flex.reflection_table()
for i, (expt, refl) in enumerate(zip(experiments, reflections)):
reflections_all.extend(refl)
reflections_all.assert_experiment_identifiers_are_consistent(experiments)
return experiments, reflections_all
def test_update_imageset_ids(dials_data):
expts = ExperimentList()
refls = []
for i in [1, 2, 3, 4, 5, 7, 8, 10]:
refls.append(
flex.reflection_table.from_file(
dials_data("multi_crystal_proteinase_k", pathlib=True) /
f"reflections_{i}.pickle"))
expts.extend(
load.experiment_list(
dials_data("multi_crystal_proteinase_k", pathlib=True) /
f"experiments_{i}.json",
check_format=False,
))
# first make sure ids are set up correctly.
experiments, reflections = assign_unique_identifiers(expts, refls)
reflections = update_imageset_ids(experiments, reflections)
joint_reflections = flex.reflection_table()
for refls in reflections:
joint_reflections.extend(refls)
# check that there are 8 unique id and imageset_ids, and that these
# correctly correspond to each experiment
assert len(set(joint_reflections["id"])) == 8
assert len(set(joint_reflections["imageset_id"])) == 8
for id_ in range(8):
sel = joint_reflections["id"] == id_
assert set(joint_reflections["imageset_id"].select(sel)) == {id_}
def from_filenames(
filenames,
verbose=False,
unhandled=None,
compare_beam=None,
compare_detector=None,
compare_goniometer=None,
scan_tolerance=None,
format_kwargs=None,
load_models=True,
):
"""Create a list of data blocks from a list of directory or file names."""
experiments = ExperimentList()
for db in DataBlockFactory.from_filenames(
filenames,
verbose=verbose,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
):
experiments.extend(
ExperimentListFactory.from_datablock_and_crystal(db, None, load_models)
)
return experiments
def from_datablock_and_crystal(datablock, crystal, load_models=True):
"""Load an experiment list from a datablock."""
# Initialise the experiment list
experiments = ExperimentList()
# If we have a list, loop through
if isinstance(datablock, list):
for db in datablock:
experiments.extend(
ExperimentListFactory.from_datablock_and_crystal(
db, crystal, load_models
)
)
return experiments
# Add all the imagesets
for imageset in datablock.extract_imagesets():
experiments.extend(
ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal, load_models
)
)
# Check the list is consistent
assert experiments.is_consistent()
return experiments
def test_multi_sweep_fixed_rotation(dials_regression, run_in_tmpdir):
data_dir = os.path.join(dials_regression, "indexing_test_data",
"multi_sweep")
reflection_files = sorted(
glob.glob(
os.path.join(data_dir, "SWEEP[1,2]", "index", "*_strong.pickle")))
experiment_files = sorted(
glob.glob(
os.path.join(data_dir, "SWEEP[1,2]", "index",
"*_datablock_import.json")))
search_beam_position.run(reflection_files + experiment_files)
assert os.path.exists("optimised.expt")
experiments = ExperimentList()
for path in experiment_files:
experiments.extend(load.experiment_list(path, check_format=False))
optimised_experiments = load.experiment_list("optimised.expt",
check_format=False)
for orig_expt, new_expt in zip(experiments, optimised_experiments):
shift = scitbx.matrix.col(
orig_expt.detector[0].get_origin()) - scitbx.matrix.col(
new_expt.detector[0].get_origin())
print(shift)
assert shift.elems == pytest.approx((2.293, -0.399, 0), abs=1e-2)
def from_args(args, verbose=False, unhandled=None):
''' Try to load experiment from any recognised format. '''
from dxtbx.datablock import DataBlockFactory
# Create a list for unhandled arguments
if unhandled is None:
unhandled = []
experiments = ExperimentList()
## First try as image files
#experiments = ExperimentListFactory.from_datablock(
#DataBlockFactory.from_args(args, verbose, unhandled1))
# Try to load from serialized formats
for filename in args:
try:
experiments.extend(
ExperimentListFactory.from_serialized_format(filename))
if verbose: print('Loaded experiments from %s' % filename)
except Exception as e:
if verbose:
print("Could not load experiments from %s: %s" %
(filename, str(e)))
unhandled.append(filename)
# Return the experiments
return experiments
class ExperimentListTemplateImporter(object):
"""A class to import an experiment list from a template."""
def __init__(self, templates, **kwargs):
importer = DataBlockTemplateImporter(templates, **kwargs)
self.experiments = ExperimentList()
for db in importer.datablocks:
self.experiments.extend(
ExperimentListFactory.from_datablock_and_crystal(db, None))
def flatten_experiments(filename_object_list):
"""
Flatten a list of experiment lists
:param filename_object_list: The parameter item
:return: The flattened experiment lists
"""
result = ExperimentList()
for o in filename_object_list:
result.extend(o.data)
return result
def from_args(args, unhandled=None):
"""Try to load serialised experiments from any recognised format."""
# Create a list for unhandled arguments
if unhandled is None:
unhandled = []
experiments = ExperimentList()
# Try to load from serialized formats
for filename in args:
try:
experiments.extend(
ExperimentListFactory.from_serialized_format(filename))
logger.debug(f"Loaded experiments from {filename}")
except Exception as e:
logger.debug(
f"Could not load experiments from {filename}: {e}")
unhandled.append(filename)
return experiments
def from_args(args, verbose=False, unhandled=None):
"""Try to load serialised experiments from any recognised format."""
# Create a list for unhandled arguments
if unhandled is None:
unhandled = []
experiments = ExperimentList()
# Try to load from serialized formats
for filename in args:
try:
experiments.extend(
ExperimentListFactory.from_serialized_format(filename)
)
if verbose:
print("Loaded experiments from %s" % filename)
except Exception as e:
if verbose:
print("Could not load experiments from %s: %s" % (filename, str(e)))
unhandled.append(filename)
return experiments
def from_dict(obj, check_format=True, directory=None):
"""
Load an experiment list from a dictionary.
Args:
obj (dict):
Dictionary containing either ExperimentList or DataBlock
structure.
check_format (bool):
If True, the file will be read to verify metadata.
directory (str):
Returns:
ExperimentList: The dictionary converted
"""
try:
experiments = ExperimentList()
for db in DataBlockFactory.from_dict(
obj, check_format=check_format, directory=directory
):
experiments.extend(
ExperimentListFactory.from_datablock_and_crystal(db, None)
)
except Exception:
experiments = None
# Decode the experiments from the dictionary
if experiments is None:
experiments = ExperimentListDict(
obj, check_format=check_format, directory=directory
).decode()
# Check the list is consistent
assert experiments.is_consistent()
return experiments
def load_imagesets(
template,
directory,
id_image=None,
image_range=None,
use_cache=True,
reversephi=False,
):
global imageset_cache
from dxtbx.model.experiment_list import ExperimentListFactory
from xia2.Applications.xia2setup import known_hdf5_extensions
from dxtbx.imageset import ImageSequence
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.model.experiment_list import BeamComparison
from dxtbx.model.experiment_list import DetectorComparison
from dxtbx.model.experiment_list import GoniometerComparison
params = PhilIndex.params.xia2.settings
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.
polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.
polarization_fraction,
)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin,
)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.
rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.
fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.
setting_rotation,
)
scan_tolerance = params.input.tolerance.scan.oscillation
# If diamond anvil cell data, always use dynamic shadowing
high_pressure = PhilIndex.params.dials.high_pressure.correction
format_kwargs = {
"dynamic_shadowing": params.input.format.dynamic_shadowing
or high_pressure,
"multi_panel": params.input.format.multi_panel,
}
if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions:
# if we are passed the correct file, use this, else look for a master
# file (i.e. something_master.h5)
if os.path.exists(full_template_path) and os.path.isfile(
full_template_path):
master_file = full_template_path
else:
g = glob.glob(os.path.join(directory, "*_master.h5"))
master_file = None
for p in g:
substr = longest_common_substring(template, p)
if substr:
if master_file is None or (len(substr) > len(
longest_common_substring(
template, master_file))):
master_file = p
if master_file is None:
raise RuntimeError("Can't find master file for %s" %
full_template_path)
unhandled = []
experiments = ExperimentListFactory.from_filenames(
[master_file],
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
assert len(unhandled) == 0, (
"unhandled image files identified: %s" % unhandled)
else:
from dxtbx.sequence_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(
locate_files_matching_template_string(full_template_path))
unhandled = []
experiments = ExperimentListFactory.from_filenames(
paths,
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
assert len(unhandled) == 0, (
"unhandled image files identified: %s" % unhandled)
else:
from xia2.Handlers.CommandLine import CommandLine
experiments = ExperimentList()
start_ends = CommandLine.get_start_ends(full_template_path)
if not start_ends:
start_ends.append(None)
for start_end in start_ends:
importer = ExperimentListTemplateImporter(
[full_template_path],
format_kwargs=format_kwargs,
image_range=start_end,
)
experiments.extend(importer.experiments)
imagesets = [
iset for iset in experiments.imagesets()
if isinstance(iset, ImageSequence)
]
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = collections.OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple(-g for g in goniometer.get_rotation_axis()))
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
# Update the geometry
params = PhilIndex.params.xia2.settings
update_geometry = []
from dials.command_line.dials_import import ManualGeometryUpdater
from dials.util.options import geometry_phil_scope
# Then add manual geometry
work_phil = geometry_phil_scope.format(params.input)
diff_phil = geometry_phil_scope.fetch_diff(source=work_phil)
if diff_phil.as_str() != "":
update_geometry.append(ManualGeometryUpdater(params.input))
imageset_list = []
for imageset in imagesets:
for updater in update_geometry:
imageset = updater(imageset)
imageset_list.append(imageset)
imagesets = imageset_list
for imageset in imagesets:
scan = imageset.get_scan()
exposure_times = scan.get_exposure_times()
epochs = scan.get_epochs()
if exposure_times.all_eq(0) or exposure_times[0] == 0:
exposure_times = flex.double(exposure_times.size(), 1)
scan.set_exposure_times(exposure_times)
elif not exposure_times.all_gt(0):
exposure_times = flex.double(exposure_times.size(),
exposure_times[0])
scan.set_exposure_times(exposure_times)
if epochs.size() > 1 and not epochs.all_gt(0):
if epochs[0] == 0:
epochs[0] = 1
for i in range(1, epochs.size()):
epochs[i] = epochs[i - 1] + exposure_times[i - 1]
scan.set_epochs(epochs)
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (image_range[0] >= scan_image_range[0]
and image_range[1] <= scan_image_range[1]):
b0 = scan.get_batch_offset()
i0 = image_range[0] - scan_image_range[0] + b0
i1 = image_range[1] - scan_image_range[0] + b0
imagesets = [imageset[i0:i1 + 1]]
assert len(
imagesets[0]) == image_range[1] - image_range[0] + 1, len(
imagesets[0])
return imagesets
return list(imageset_cache[full_template_path].values())
def test_experiment_list_extend():
"""Check that the extend method of ExperimentList works."""
# Create a minimal ExperimentList instance.
expts = ExperimentList([Experiment()])
# Try to extend it.
expts.extend(expts)
class Screen19(object):
"""Encapsulates the screening script."""
def __init__(self):
# Throughout the pipeline, retain the state of the processing.
self.expts = ExperimentList([])
self.refls = flex.reflection_table()
# Get some default parameters. These must be extracted from the 'fetched'
# PHIL scope, rather than the 'definition' phil scope returned by
# iotbx.phil.parse. Confused? Blame PHIL.
self.params = phil_scope.fetch(iotbx.phil.parse("")).extract()
def _quick_import(self, files): # type: (List[str]) -> bool
"""
Generate xia2-style templates from file names and attempt a quick import.
From each given filename, generate a filename template by substituting a hash
character (#) for each numeral in the last contiguous group of numerals
before the file extension. For example, the filename `example_01_0001.cbf`
becomes `example_01_####.cbf`.
Contiguous image ranges are recorded by associating the start and end image
number of the range with the relevant filename template.
dials.import is then run with options to extrapolate header information from
the first image file, thereby running more quickly than reading each image
header individually.
Args:
files: List of image filenames.
Returns:
Boolean flag indicating whether the quick import has succeeded.
"""
if len(files) == 1:
# No point in quick-importing a single file
return False
debug("Attempting quick import...")
files.sort()
templates = {} # type: Dict[str, List[Optional[List[int]]]]
for f in files:
template, image = screen19.make_template(f)
if template not in templates:
image_range = [image, image] if image else []
templates.update({template: [image_range]})
elif image == templates[template][-1][-1] + 1:
templates[template][-1][-1] = image
elif image == templates[template][-1][-1]:
# We have a duplicate input file name. Do nothing.
pass
else:
templates[template].append([image, image])
# Return tuple of template and image range for each unique image range
templates = [(t, tuple(r)) for t, ranges in templates.items()
for r in ranges] # type: Templates
return self._quick_import_templates(templates)
def _quick_import_templates(self, templates): # type: (Templates) -> bool
"""
Take image file templates and frame number ranges and try to run dials.import.
dials.import is run with options to extrapolate header information from
the first image file, thereby running more quickly than reading each image
header individually.
Args:
templates: A list of tuples, each tuple containing a xia2-style filename
template and the start and end image numbers of the associated
sweep.
Returns:
Boolean flag indicating whether the quick import has succeeded.
"""
debug("Quick import template summary:\n\t%s", templates)
if len(templates) > 1:
debug("Cannot currently run quick import on multiple templates.")
return False
try:
scan_range = templates[0][1] # type: Tuple[int, int]
if not scan_range:
raise IndexError
except IndexError:
debug(
"Cannot run quick import: could not determine image naming template."
)
return False
info("Running quick import.")
self.params.dials_import.input.template = [templates[0][0]]
self.params.dials_import.geometry.scan.image_range = scan_range
self.params.dials_import.geometry.scan.extrapolate_scan = True
self._run_dials_import()
return True
def _import(self, files): # type: (List[str]) -> None
"""
Try to run a quick call of dials.import. Failing that, run a slow call.
Try initially to construct file name templates contiguous groups of files.
Failing that, pass a full list of the files to the importer (slower).
Args:
files: List of image filenames.
"""
info("\nImporting data...")
if len(files) == 1:
if os.path.isdir(files[0]):
debug("You specified a directory. Importing all CBF files in "
"that directory.")
# TODO Support HDF5.
files = [
os.path.join(files[0], f) for f in os.listdir(files[0])
if f.endswith(".cbf") or f.endswith(".cbf.gz")
or f.endswith(".cbf.bz2")
]
elif len(files[0].split(":")) == 3:
debug("You specified an image range in the xia2 format. "
"Importing all specified files.")
template, start, end = files[0].split(":")
template = screen19.make_template(template)[0]
start, end = int(start), int(end)
if not self._quick_import_templates([(template,
(start, end))]):
warning("Could not import specified image range.")
sys.exit(1)
info("Quick import successful.")
return
elif files[0].endswith(".expt"):
debug("You specified an existing experiment list file. "
"No import necessary.")
try:
self.expts = ExperimentList.from_file(files[0])
except (IOError, PickleError, ValueError):
pass
else:
self.params.dials_import.output.experiments = files[0]
if self.expts:
return
if not files:
warning("No images found matching input.")
sys.exit(1)
# Can the files be quick-imported?
if self._quick_import(files):
info("Quick import successful.")
return
self.params.dials_import.input.experiments = files
self._run_dials_import()
def _run_dials_import(self):
"""
Perform a minimal version of dials.import to get an experiment list.
Use some filleted bits of dials.import and dials.util.options.Importer.
"""
# Get some key data format arguments.
try:
format_kwargs = {
"dynamic_shadowing":
self.params.dials_import.format.dynamic_shadowing,
"multi_panel": self.params.dials_import.format.multi_panel,
}
except AttributeError:
format_kwargs = {}
# If filenames contain wildcards, expand
args = []
for arg in self.params.dials_import.input.experiments:
if "*" in arg:
args.extend(glob(arg))
else:
args.append(arg)
if args:
# Are compare{beam,detector,goniometer} and scan_tolerance necessary?
# They are cargo-culted from the DIALS option parser.
tol_params = self.params.dials_import.input.tolerance
compare_beam = BeamComparison(
wavelength_tolerance=tol_params.beam.wavelength,
direction_tolerance=tol_params.beam.direction,
polarization_normal_tolerance=tol_params.beam.
polarization_normal,
polarization_fraction_tolerance=tol_params.beam.
polarization_fraction,
)
compare_detector = DetectorComparison(
fast_axis_tolerance=tol_params.detector.fast_axis,
slow_axis_tolerance=tol_params.detector.slow_axis,
origin_tolerance=tol_params.detector.origin,
)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=tol_params.goniometer.rotation_axis,
fixed_rotation_tolerance=tol_params.goniometer.fixed_rotation,
setting_rotation_tolerance=tol_params.goniometer.
setting_rotation,
)
scan_tolerance = tol_params.scan.oscillation
# Import an experiment list from image data.
try:
experiments = ExperimentListFactory.from_filenames(
args,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
except IOError as e:
warning("%s '%s'", e.strerror, e.filename)
sys.exit(1)
# Record the imported experiments for use elsewhere.
# Quit if there aren't any.
self.expts.extend(experiments)
if not self.expts:
warning("No images found.")
sys.exit(1)
else:
# Use the template importer.
if len(self.params.dials_import.input.template) > 0:
importer = ExperimentListTemplateImporter(
self.params.dials_import.input.template,
format_kwargs=format_kwargs)
# Record the imported experiments for use elsewhere.
# Quit if there aren't any.
self.expts.extend(importer.experiments)
if not self.expts:
warning("No images found matching template %s" %
self.params.dials_import.input.template[0])
sys.exit(1)
# Setup the metadata updater
metadata_updater = MetaDataUpdater(self.params.dials_import)
# Extract the experiments and loop through
self.expts = metadata_updater(self.expts.imagesets())
def _count_processors(self, nproc=None): # type: (Optional[int]) -> None
"""
Determine the number of processors and save it as an instance variable.
The user may specify the number of processors to use. If no value is
given, the number of available processors is returned.
Args:
nproc (optional): Number of processors.
"""
if nproc and nproc is not Auto:
self.nproc = nproc
return
# if environmental variable NSLOTS is set to a number then use that
try:
self.nproc = int(os.environ.get("NSLOTS"))
return
except (ValueError, TypeError):
pass
self.nproc = number_of_processors(return_value_if_unknown=-1)
if self.nproc <= 0:
warning(
"Could not determine number of available processors. Error code %d",
self.nproc,
)
sys.exit(1)
def _count_images(self): # type: () -> int
"""
Attempt to determine the number of diffraction images.
The number of diffraction images is determined from the imported_experiments
JSON file.
Returns:
Number of images.
"""
# FIXME: This exception handling should be redundant. Empty experiment
# lists should get caught at the import stage. Is this so?
try:
return self.expts[0].imageset.size()
except IndexError:
warning("Could not determine number of images in dataset.")
sys.exit(1)
def _check_intensities(self,
mosaicity_correction=True): # type: (bool) -> None
"""
Run xia2.overload and plot a histogram of pixel intensities.
If `mosaicity_correction` is true, the pixel intensities are approximately
adjusted to take account of a systematic defect in the detector count rate
correction. See https://github.com/xia2/screen19/wiki#mosaicity-correction
Args:
mosaicity_correction (optional): default is `True`.
"""
info("\nTesting pixel intensities...")
command = ["xia2.overload", "nproc=%s" % self.nproc, "indexed.expt"]
debug("running %s", command)
result = procrunner.run(command,
print_stdout=False,
debug=procrunner_debug)
debug("result = %s", screen19.prettyprint_dictionary(result))
info("Successfully completed (%.1f sec)", result["runtime"])
if result["exitcode"] != 0:
warning("Failed with exit code %d", result["exitcode"])
sys.exit(1)
with open("overload.json") as fh:
overload_data = json.load(fh)
info("Pixel intensity distribution:")
count_sum = 0
hist = {}
if "bins" in overload_data:
for b in range(overload_data["bin_count"]):
if overload_data["bins"][b] > 0:
hist[b] = overload_data["bins"][b]
count_sum += b * overload_data["bins"][b]
else:
hist = {
int(k): v
for k, v in overload_data["counts"].items() if int(k) > 0
}
count_sum = sum([k * v for k, v in hist.items()])
average_to_peak = 1
if mosaicity_correction:
# Adjust for the detector count rate correction
if self._sigma_m:
delta_z = self._oscillation / self._sigma_m / math.sqrt(2)
average_to_peak = (
math.sqrt(math.pi) * delta_z * math.erf(delta_z) +
math.exp(-(delta_z**2)) - 1) / delta_z**2
info("Average-to-peak intensity ratio: %f", average_to_peak)
scale = 100 * overload_data["scale_factor"] / average_to_peak
info("Determined scale factor for intensities as %f", scale)
debug(
"intensity histogram: { %s }",
", ".join(["%d:%d" % (k, hist[k]) for k in sorted(hist)]),
)
max_count = max(hist.keys())
hist_max = max_count * scale
hist_granularity, hist_format = 1, "%.0f"
if hist_max < 50:
hist_granularity, hist_format = 2, "%.1f"
if hist_max < 15:
hist_granularity, hist_format = 10, "%.1f"
rescaled_hist = {}
for x in hist.keys():
rescaled = round(x * scale * hist_granularity)
if rescaled > 0:
rescaled_hist[rescaled] = hist[x] + rescaled_hist.get(
rescaled, 0)
hist = rescaled_hist
debug(
"rescaled histogram: { %s }",
", ".join([(hist_format + ":%d") % (k / hist_granularity, hist[k])
for k in sorted(hist)]),
)
screen19.plot_intensities(hist,
1 / hist_granularity,
procrunner_debug=procrunner_debug)
linear_response_limit = 100 * self.params.maximum_flux.trusted_range_correction
marginal_limit = max(70, linear_response_limit)
text = "".join((
"Strongest pixel (%d counts) " % max_count,
"reaches %.1f%% " % hist_max,
"of the detector count rate limit",
))
if hist_max > 100:
warning("Warning: %s!", text)
else:
info(text)
if ("overload_limit" in overload_data
and max_count >= overload_data["overload_limit"]):
warning(
"Warning: THE DATA CONTAIN REGULAR OVERLOADS!\n"
" The photon incidence rate is outside the specified "
"limits of the detector.\n"
" The built-in detector count rate correction cannot "
"adjust for this.\n"
" You should aim for count rates below {:.0%} of the "
"detector limit.".format(
self.params.maximum_flux.trusted_range_correction))
elif hist_max > marginal_limit:
warning(
"Warning: The photon incidence rate is well outside the "
"linear response region of the detector (<{:.0%}).\n"
" The built-in detector count rate correction may not be "
"able to adjust for this.".format(
self.params.maximum_flux.trusted_range_correction))
elif hist_max > linear_response_limit:
info("The photon incidence rate is outside the linear response "
"region of the detector (<{:.0%}).\n"
" The built-in detector count rate correction may be able "
"to adjust for this.".format(
self.params.maximum_flux.trusted_range_correction))
if not mosaicity_correction:
warning(
"Warning: Not enough data for proper profile estimation."
" The spot intensities are not corrected for mosaicity.\n"
" The true photon incidence rate will be higher than the "
"given estimate.")
info("Total sum of counts in dataset: %d", count_sum)
def _find_spots(self, args=None): # type: (Optional[List[str]]) -> None
"""
Call `dials.find_spots` on the imported experiment list.
Args:
args (optional): List of any additional PHIL parameters to be used by
dials.import.
"""
info("\nFinding spots...")
dials_start = timeit.default_timer()
# Use some choice fillets from dials.find_spots
# Ignore `args`, use `self.params`
# Loop through all the imagesets and find the strong spots
self.refls = flex.reflection_table.from_observations(
self.expts, self.params.dials_find_spots)
# Add n_signal column - before deleting shoeboxes
good = MaskCode.Foreground | MaskCode.Valid
self.refls["n_signal"] = self.refls["shoebox"].count_mask_values(good)
# Delete the shoeboxes
if not self.params.dials_find_spots.output.shoeboxes:
del self.refls["shoebox"]
info(
60 * "-" + "\n%s\n" + 60 * "-" +
"\nSuccessfully completed (%.1f sec)",
spot_counts_per_image_plot(self.refls),
timeit.default_timer() - dials_start,
)
def _index(self): # type: () -> bool
"""
Call `dials.index` on the output of spot finding.
Returns:
Boolean value indicating whether indexing was successful.
"""
dials_start = timeit.default_timer()
# Prepare max_cell constraint strategies.
max_cell = self.params.dials_index.indexing.max_cell
# By default, try unconstrained max_cell followed by max_cell=20.
# If the user has already specified a max_cell < 20, do not relax to 20Å.
cell_constraints = [([], max_cell)]
if not max_cell or max_cell is Auto or max_cell > 20:
cell_constraints += [(["max_cell constraint"], 20)]
# Prepare indexing methods, preferring the real_space_grid_search if a
# known unit cell has been specified, otherwise using 3D FFT, then 1D FFT.
methods = ([
(["real space grid search"], "real_space_grid_search")
] if self.params.dials_index.indexing.known_symmetry.unit_cell else [])
methods += [(["3D FFT"], "fft3d"), (["1D FFT"], "fft1d")]
# Cycle through the indexing methods for each of the max_cell constraint
# strategies until an indexing solution is found.
for i, (max_cell_msg, max_cell) in enumerate(cell_constraints):
# Set the max_cell constraint strategy.
self.params.dials_index.indexing.max_cell = max_cell
for j, (method_msg, method) in enumerate(methods):
# Set the indexing method.
self.params.dials_index.indexing.method = method
# Log a handy message to the user.
msg = ("Retrying with " +
" and ".join(method_msg + max_cell_msg) if i +
j else "Indexing")
info("\n%s...", msg)
try:
# If indexing is successful, break out of the inner loop.
self.expts, self.refls = index(self.expts, [self.refls],
self.params.dials_index)
break
except (DialsIndexError, ValueError) as e:
# If indexing is unsuccessful, try again with the next
# strategy.
warning("Failed: %s", str(e))
continue
else:
# When all the indexing methods are unsuccessful, move onto
# the next max_cell constraint strategy and try again.
continue
# We should only get here if successfully indexed. Break out of the loop
break
else:
# Indexing completely unsuccessful.
return False
sg_type = self.expts[0].crystal.get_crystal_symmetry().space_group(
).type()
symb = sg_type.universal_hermann_mauguin_symbol()
unit_cell = self.expts[0].crystal.get_unit_cell()
self.refls.as_file(self.params.dials_index.output.reflections)
self.expts.as_file(self.params.dials_index.output.experiments)
self.refls.as_file(self.params.dials_index.output.reflections)
info(
"Found primitive solution: %s %s using %s reflections\n"
"Indexed experiments and reflections saved as %s, %s\n"
"Successfully completed (%.1f sec)",
symb,
unit_cell,
self.refls["id"].count(0),
self.params.dials_index.output.experiments,
self.params.dials_index.output.reflections,
timeit.default_timer() - dials_start,
)
# Report the indexing successful.
return True
def _wilson_calculation(self): # type: () -> None
"""
Run `screen19.minimum_exposure` on an experiment list and reflection table.
For best results, the reflections and experiment list should contain the
results of integration or scaling. If only strong spots are used, the Wilson
plot fit may be poor.
"""
dials_start = timeit.default_timer()
info("\nEstimating lower exposure bound...")
suggest_minimum_exposure(self.expts, self.refls,
self.params.minimum_exposure)
info("Successfully completed (%.1f sec)",
timeit.default_timer() - dials_start)
def _refine(self): # type: () -> None
"""
Run `dials.refine` on the results of indexing.
"""
dials_start = timeit.default_timer()
info("\nRefining...")
try:
self.expts, self.refls, _, _ = run_dials_refine(
self.expts, self.refls, self.params.dials_refine)
except Sorry as e:
warning("dials.refine failed: %d\nGiving up.\n", e)
sys.exit(1)
info("Successfully refined (%.1f sec)",
timeit.default_timer() - dials_start)
def _create_profile_model(self): # type: () -> bool
"""
Run `dials.create_profile_model` on indexed reflections.
The indexed experiment list will be overwritten with a copy that includes
the profile model but is otherwise identical.
Returns:
Boolean value indicating whether it was possible to determine a profile
model from the data.
"""
info("\nCreating profile model...")
command = [
"dials.create_profile_model",
self.params.dials_index.output.experiments,
self.params.dials_index.output.reflections,
"output = %s" % self.params.dials_index.output.experiments,
]
result = procrunner.run(command,
print_stdout=False,
debug=procrunner_debug)
debug("result = %s", screen19.prettyprint_dictionary(result))
self._sigma_m = None
if result["exitcode"] == 0:
db = ExperimentList.from_file(
self.params.dials_index.output.experiments)[0]
self._oscillation = db.imageset.get_scan().get_oscillation()[1]
self._sigma_m = db.profile.sigma_m()
info(
u"%d images, %s° oscillation, σ_m=%.3f°",
db.imageset.get_scan().get_num_images(),
str(self._oscillation),
self._sigma_m,
)
info("Successfully completed (%.1f sec)", result["runtime"])
return True
warning("Failed with exit code %d", result["exitcode"])
return False
def _integrate(self): # type: () -> None
"""Run `dials.integrate` to integrate reflection intensities."""
dials_start = timeit.default_timer()
info("\nIntegrating...")
# Don't waste time recreating the profile model
self.params.dials_integrate.create_profile_model = False
# Get the dials.integrate PHIL scope, populated with parsed input parameters
integrate_scope = phil_scope.get("dials_integrate").objects[0]
integrate_scope.name = ""
integrate_scope = integrate_scope.format(self.params.dials_integrate)
try:
integrated_experiments, integrated_reflections = _run_integration(
integrate_scope,
self.params.dials_index.output.experiments,
self.params.dials_index.output.reflections,
)
# Save the output to files
integrated_reflections.as_file(
self.params.dials_integrate.output.reflections)
integrated_experiments.as_file(
self.params.dials_integrate.output.experiments)
# ... and also store the output internally
self.expts, self.refls = integrated_experiments, integrated_reflections
info(
"Successfully completed (%.1f sec)",
timeit.default_timer() - dials_start,
)
except SystemExit as e:
if e.code:
warning("dials.integrate failed with exit code %d\nGiving up.",
e.code)
sys.exit(1)
# This is a hacky check but should work for as long as DIALS 2.0 is supported.
if version.dials_version() < "DIALS 2.1":
def _refine_bravais(self, experiments, reflections):
# type: (ExperimentList, flex.reflection_table) -> None
"""
Run `dials.refine_bravais_settings` on an experiments and reflections.
Args:
experiments: An experiment list..
reflections: The corresponding reflection table.
"""
info("\nRefining Bravais settings...")
command = [
"dials.refine_bravais_settings", experiments, reflections
]
result = procrunner.run(command,
print_stdout=False,
debug=procrunner_debug)
debug("result = %s", screen19.prettyprint_dictionary(result))
if result["exitcode"] == 0:
m = re.search(
r"[-+]{3,}\n[^\n]*\n[-+|]{3,}\n(.*\n)*[-+]{3,}",
result["stdout"].decode("utf-8"),
)
if m:
info(m.group(0))
else:
info(
"Could not interpret dials.refine_bravais_settings output, "
"please check dials.refine_bravais_settings.log")
info("Successfully completed (%.1f sec)", result["runtime"])
else:
warning("Failed with exit code %d", result["exitcode"])
sys.exit(1)
else:
def _refine_bravais(self): # type: () -> None
"""Run `dials.refine_bravais_settings` to determine the space group."""
dials_start = timeit.default_timer()
info("\nRefining Bravais settings...")
self.refls = eliminate_sys_absent(self.expts, self.refls)
map_to_primitive(self.expts, self.refls)
try:
refined_settings = refined_settings_from_refined_triclinic(
self.expts, self.refls, self.params.dials_refine_bravais)
except RuntimeError as e:
warning("dials.refine_bravais_settings failed.\nGiving up.")
sys.exit(e)
possible_bravais_settings = {
solution["bravais"]
for solution in refined_settings
}
bravais_lattice_to_space_group_table(possible_bravais_settings)
try:
# Old version of dials with as_str() method
logger.info(refined_settings.as_str())
except AttributeError:
# Newer versions of dials (>= 2.2.2) has proper __str__ method
logger.info(refined_settings)
info(
"Successfully completed (%.1f sec)",
timeit.default_timer() - dials_start,
)
def _report(self, experiments, reflections):
# type: (ExperimentList, flex.reflection_table) -> None
"""
Run `dials.report` on an experiment list and reflection table.
Args:
experiments: An experiment list.
reflections: The corresponding reflection table.
"""
info("\nCreating report...")
command = ["dials.report", experiments, reflections]
result = procrunner.run(command,
print_stdout=False,
debug=procrunner_debug)
debug("result = %s", screen19.prettyprint_dictionary(result))
if result["exitcode"] == 0:
info("Successfully completed (%.1f sec)", result["runtime"])
# if sys.stdout.isatty():
# info("Trying to start browser")
# try:
# import subprocess
# d = dict(os.environ)
# d["LD_LIBRARY_PATH"] = ""
# subprocess.Popen(["xdg-open", "dials-report.html"], env=d)
# except Exception as e:
# debug("Could not open browser\n%s", str(e))
else:
warning("Failed with exit code %d", result["exitcode"])
sys.exit(1)
def run(self, args=None, phil=phil_scope, set_up_logging=False):
# type: (Optional[List[str]], scope, bool) -> None
"""
TODO: Docstring.
Args:
args:
phil:
set_up_logging:
Returns:
"""
usage = "%prog [options] image_directory | image_files.cbf | imported.expt"
parser = OptionParser(usage=usage,
epilog=__doc__,
phil=phil,
check_format=False)
self.params, options, unhandled = parser.parse_args(
args=args,
show_diff_phil=True,
return_unhandled=True,
quick_parse=True)
version_information = "screen19 v%s using %s (%s)" % (
screen19.__version__,
dials.util.version.dials_version(),
time.strftime("%Y-%m-%d %H:%M:%S"),
)
start = timeit.default_timer()
if len(unhandled) == 0:
print(__doc__)
print(version_information)
return
if set_up_logging:
# Configure the logging
log.config(verbosity=self.params.verbosity,
logfile=self.params.output.log)
# Unless verbose output has been requested, suppress generation of
# debug and info log records from any child DIALS command, retaining
# those from screen19 itself.
if not self.params.verbosity:
logging.getLogger("dials").setLevel(logging.WARNING)
logging.getLogger("dials.screen19").setLevel(logging.INFO)
info(version_information)
debug("Run with:\n%s\n%s", " ".join(unhandled),
parser.diff_phil.as_str())
self._count_processors(nproc=self.params.nproc)
debug("Using %s processors", self.nproc)
# Set multiprocessing settings for spot-finding, indexing and
# integration to match the top-level specified number of processors
self.params.dials_find_spots.spotfinder.mp.nproc = self.nproc
self.params.dials_index.indexing.nproc = self.nproc
# Setting self.params.dials_refine.refinement.mp.nproc is not helpful
self.params.dials_integrate.integration.mp.nproc = self.nproc
# Set the input and output parameters for the DIALS components
# TODO: Compare to diff_phil and start from later in the pipeline if
# appropriate
self._import(unhandled)
imported_name = self.params.dials_import.output.experiments
self._find_spots()
if not self._index():
info("\nRetrying for stronger spots only...")
strong_refls = self.refls
self.params.dials_find_spots.spotfinder.threshold.dispersion.sigma_strong = (
15)
self._find_spots()
if not self._index():
warning("Giving up.")
self.expts.as_file(imported_name)
strong_refls.as_file("strong.refl")
self.refls.as_file("stronger.refl")
info(
"Could not find an indexing solution. You may want to "
"have a look at the reciprocal space by running:\n\n"
" dials.reciprocal_lattice_viewer %s %s\n\n"
"or, to only include stronger spots:\n\n"
" dials.reciprocal_lattice_viewer %s %s\n",
imported_name,
"strong.refl",
imported_name,
"stronger.refl",
)
sys.exit(1)
if not self._create_profile_model():
info(
"\nRefining model to attempt to increase number of valid spots..."
)
self._refine()
if not self._create_profile_model():
warning("Giving up.")
info(
"The identified indexing solution may not be correct. "
"You may want to have a look at the reciprocal space by "
"running:\n\n"
" dials.reciprocal_lattice_viewer indexed.expt indexed.refl\n"
)
sys.exit(1)
self._check_intensities()
if self.params.minimum_exposure.data == "integrated":
self._integrate()
self._wilson_calculation()
experiments = self.params.dials_integrate.output.experiments
reflections = self.params.dials_integrate.output.reflections
else:
self._wilson_calculation()
experiments = self.params.dials_create_profile.output
reflections = self.params.dials_index.output.reflections
# This is a hacky check but should work for as long as DIALS 2.0 is supported.
if version.dials_version() < "DIALS 2.1":
self._refine_bravais(experiments, reflections)
else:
self._refine_bravais()
self._report(experiments, reflections)
runtime = timeit.default_timer() - start
debug(
"Finished at %s, total runtime: %.1f",
time.strftime("%Y-%m-%d %H:%M:%S"),
runtime,
)
info("screen19 successfully completed (%.1f sec).", runtime)
class XDSRefiner(Refiner):
def __init__(self):
super(XDSRefiner, self).__init__()
# factory functions
def ExportXDS(self):
from xia2.Wrappers.Dials.ExportXDS import ExportXDS as _ExportXDS
export_xds = _ExportXDS()
export_xds.set_working_directory(self.get_working_directory())
auto_logfiler(export_xds)
return export_xds
def _refine_prepare(self):
for epoch, idxr in self._refinr_indexers.iteritems():
experiments = idxr.get_indexer_experiment_list()
assert len(
experiments) == 1 # currently only handle one lattice/sweep
experiment = experiments[0]
crystal_model = experiment.crystal
lattice = idxr.get_indexer_lattice()
# check if the lattice was user assigned...
user_assigned = idxr.get_indexer_user_input_lattice()
# hack to figure out if we did indexing with Dials - if so then need to
# run XYCORR, INIT, and then dials.export_xds before we are ready to
# integrate with XDS
from xia2.Modules.Indexer.DialsIndexer import DialsIndexer
if isinstance(idxr, DialsIndexer):
sweep = idxr.get_indexer_sweep()
imageset = idxr._indxr_imagesets[0]
scan = imageset.get_scan()
first, last = scan.get_image_range()
phi_width = scan.get_oscillation()[1]
last_background = int(round(5.0 / phi_width)) - 1 + first
last_background = min(last, last_background)
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
xds_idxr = XDSIndexer()
xds_idxr.set_working_directory(self.get_working_directory())
xds_idxr.set_indexer_sweep(sweep)
xds_idxr.add_indexer_imageset(imageset)
# next start to process these - first xycorr
# FIXME run these *afterwards* as then we have a refined detector geometry
# so the parallax correction etc. should be slightly better.
#self._indxr_images = [(first, last)]
xycorr = xds_idxr.Xycorr()
xycorr.set_data_range(first, last)
xycorr.set_background_range(first, last_background)
xycorr.set_working_directory(self.get_working_directory())
xycorr.run()
xds_data_files = {}
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
xds_data_files[file] = xycorr.get_output_data_file(file)
# next start to process these - then init
init = xds_idxr.Init()
for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']:
init.set_input_data_file(file, xds_data_files[file])
init.set_data_range(first, last)
init.set_background_range(first, last_background)
init.set_working_directory(self.get_working_directory())
init.run()
for file in ['BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']:
xds_data_files[file] = init.get_output_data_file(file)
exporter = self.ExportXDS()
exporter.set_experiments_filename(
idxr.get_solution()['experiments_file'])
exporter.run()
for file in ['XPARM.XDS']:
xds_data_files[file] = os.path.join(
self.get_working_directory(), 'xds', file)
for k, v in xds_data_files.items():
idxr.set_indexer_payload(k, v)
# check that the indexer is an XDS indexer - if not then
# create one...
elif not idxr.get_indexer_payload('XPARM.XDS'):
Debug.write('Generating an XDS indexer')
idxr_old = idxr
from xia2.Modules.Indexer.XDSIndexer import XDSIndexer
idxr = XDSIndexer()
idxr.set_indexer_sweep(idxr_old.get_indexer_sweep())
self._refinr_indexers[epoch] = idxr
self.set_refiner_prepare_done(False)
# note to self for the future - this set will reset the
# integrater prepare done flag - this means that we will
# go through this routine all over again. However this
# is not a problem as all that will happen is that the
# results will be re-got, no additional processing will
# be performed...
# set the indexer up as per the frameprocessor interface...
# this would usually happen within the IndexerFactory.
idxr.set_indexer_sweep_name(idxr_old.get_indexer_sweep_name())
idxr.add_indexer_imageset(idxr_old.get_imageset())
idxr.set_working_directory(idxr_old.get_working_directory())
# now copy information from the old indexer to the new
# one - lattice, cell, distance etc.
# bug # 2434 - providing the correct target cell
# may be screwing things up - perhaps it would
# be best to allow XDS just to index with a free
# cell but target lattice??
cell = crystal_model.get_unit_cell().parameters()
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
# FIXME this was changed in #42 but not sure logic is right
if not check:
Debug.write(
'Inputting target cell: %.2f %.2f %.2f %.2f %.2f %.2f' % \
cell)
idxr.set_indexer_input_cell(cell)
input_cell = cell
from cctbx.sgtbx import bravais_types
lattice = str(
bravais_types.bravais_lattice(
group=crystal_model.get_space_group()))
idxr.set_indexer_input_lattice(lattice)
if user_assigned:
Debug.write('Assigning the user given lattice: %s' % \
lattice)
idxr.set_indexer_user_input_lattice(True)
idxr.set_detector(experiment.detector)
idxr.set_beam(experiment.beam)
idxr.set_goniometer(experiment.goniometer)
# re-get the unit cell &c. and check that the indexing
# worked correctly
Debug.write('Rerunning indexing with XDS')
experiments = idxr.get_indexer_experiment_list()
assert len(experiments
) == 1 # currently only handle one lattice/sweep
experiment = experiments[0]
crystal_model = experiment.crystal
# then in here check that the target unit cell corresponds
# to the unit cell I wanted as input...? now for this I
# should probably compute the unit cell volume rather
# than comparing the cell axes as they may have been
# switched around...
# FIXME comparison needed
def _refine(self):
import copy
from dxtbx.model import ExperimentList
self._refinr_refined_experiment_list = ExperimentList()
for epoch, idxr in self._refinr_indexers.iteritems():
self._refinr_payload[epoch] = copy.deepcopy(idxr._indxr_payload)
self._refinr_refined_experiment_list.extend(
idxr.get_indexer_experiment_list())
def _refine_finish(self):
pass
def index_all_concurrent(experiments, reflections, params, method_list):
# first determine n_strong per image:
n_strong_per_image = {}
for i, (expt, table) in enumerate(zip(experiments, reflections)):
img = expt.imageset.get_path(i).split("/")[-1]
n_strong = table.get_flags(table.flags.strong).count(True)
n_strong_per_image[img] = n_strong
with concurrent.futures.ProcessPoolExecutor(
max_workers=params.indexing.nproc) as pool:
sys.stdout = open(os.devnull, "w") # block printing from rstbx
futures = {
pool.submit(_index_one, expt, table, params, method_list, i): i
for i, (table, expt) in enumerate(zip(reflections, experiments))
}
tables_list = [None] * len(reflections)
expts_list = [None] * len(reflections)
for future in concurrent.futures.as_completed(futures):
try:
expts, refls = future.result()
j = futures[future]
except Exception as e:
logger.info(e)
else:
if refls and expts:
tables_list[j] = refls
elist = ExperimentList()
for jexpt in expts:
elist.append(
Experiment(
identifier=jexpt.identifier,
beam=jexpt.beam,
detector=jexpt.detector,
scan=jexpt.scan,
goniometer=jexpt.goniometer,
crystal=jexpt.crystal,
imageset=jexpt.imageset[j:j + 1],
))
expts_list[j] = elist
sys.stdout = sys.__stdout__ # restore printing
# now postprocess - record generic information
results_summary = defaultdict(list)
indexed_experiments = ExperimentList()
indexed_reflections = flex.reflection_table()
n_tot = 0
for idx, (elist, table) in enumerate(zip(expts_list, tables_list)):
if not (elist and table):
img = experiments[idx].imageset.get_path(idx).split("/")[-1]
n_strong = n_strong_per_image[img]
results_summary[idx].append({
"Image": img,
"n_indexed": 0,
"n_strong": n_strong,
})
continue
path = elist[0].imageset.get_path(0)
img = path.split("/")[-1]
n_strong = n_strong_per_image[img]
indexed_experiments.extend(elist)
ids_map = dict(table.experiment_identifiers())
for k in table.experiment_identifiers().keys():
del table.experiment_identifiers()[k]
table["id"] += n_tot
for k, v in ids_map.items():
table.experiment_identifiers()[k + n_tot] = v
n_tot += len(ids_map.keys())
indexed_reflections.extend(table)
# record some things for printing to output log/html
for id_ in table.experiment_identifiers().keys():
selr = table.select(table["id"] == id_)
calx, caly, _ = selr["xyzcal.px"].parts()
obsx, obsy, _ = selr["xyzobs.px.value"].parts()
delpsi = selr["delpsical.rad"]
rmsd_x = flex.mean((calx - obsx)**2)**0.5
rmsd_y = flex.mean((caly - obsy)**2)**0.5
rmsd_z = flex.mean(((delpsi) * RAD2DEG)**2)**0.5
n_id_ = calx.size()
results_summary[idx].append({
"Image": img,
"n_indexed": n_id_,
"n_strong": n_strong,
"RMSD_X": rmsd_x,
"RMSD_Y": rmsd_y,
"RMSD_dPsi": rmsd_z,
})
indexed_reflections.assert_experiment_identifiers_are_consistent(
indexed_experiments)
return indexed_experiments, indexed_reflections, results_summary
def index(self):
experiments = ExperimentList()
had_refinement_error = False
have_similar_crystal_models = False
while True:
if had_refinement_error or have_similar_crystal_models:
break
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = (self.params.multiple_lattice_search.
recycle_unindexed_reflections_cutoff)
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_indexed = flex.min(
d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = cutoff_fraction * len(
self.reflections.select(d_spacings > d_min_indexed))
crystal_ids = self.reflections.select(
d_spacings > d_min_indexed)["id"]
if (crystal_ids
== -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% ((crystal_ids == -1).count(True)))
break
n_lattices_previous_cycle = len(experiments)
if self.d_min is None:
self.d_min = self.params.refinement_protocol.d_min_start
if len(experiments) == 0:
new_expts = self.find_lattices()
generate_experiment_identifiers(new_expts)
experiments.extend(new_expts)
else:
try:
new = self.find_lattices()
generate_experiment_identifiers(new)
experiments.extend(new)
except DialsIndexError:
logger.info("Indexing remaining reflections failed")
if self.params.refinement_protocol.d_min_step is libtbx.Auto:
n_cycles = self.params.refinement_protocol.n_macro_cycles
if self.d_min is None or n_cycles == 1:
self.params.refinement_protocol.d_min_step = 0
else:
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_all = flex.min(d_spacings)
self.params.refinement_protocol.d_min_step = (
self.d_min - d_min_all) / (n_cycles - 1)
logger.info("Using d_min_step %.1f" %
self.params.refinement_protocol.d_min_step)
if len(experiments) == 0:
raise DialsIndexError("No suitable lattice could be found.")
elif len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
for i_cycle in range(
self.params.refinement_protocol.n_macro_cycles):
if (i_cycle > 0 and self.d_min is not None
and self.params.refinement_protocol.d_min_step > 0):
d_min = self.d_min - self.params.refinement_protocol.d_min_step
d_min = max(d_min, 0)
if self.params.refinement_protocol.d_min_final is not None:
d_min = max(
d_min, self.params.refinement_protocol.d_min_final)
if d_min >= 0:
self.d_min = d_min
logger.info("Increasing resolution to %.2f Angstrom" %
d_min)
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections["id"] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if i_cycle == 0 and self.params.known_symmetry.space_group is not None:
self._apply_symmetry_post_indexing(
experiments, self.reflections,
n_lattices_previous_cycle)
logger.info("\nIndexed crystal models:")
self.show_experiments(experiments,
self.reflections,
d_min=self.d_min)
if self._check_have_similar_crystal_models(experiments):
have_similar_crystal_models = True
break
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement (macro-cycle %i)" %
(i_cycle + 1))
logger.info("#" * 80)
logger.info("")
self.indexed_reflections = self.reflections["id"] > -1
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections["rlp"].norms()
if self.d_min is not None:
isel = (lengths <= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections["id"] == -1, True)
self.reflections.unset_flags(sel,
self.reflections.flags.indexed)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections)
if self.params.refinement_protocol.mode == "repredict_only":
refined_experiments, refined_reflections = (
experiments,
reflections_for_refinement,
)
from dials.algorithms.refinement.prediction.managed_predictors import (
ExperimentsPredictorFactory, )
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
spherical_relp=self.all_params.refinement.
parameterisation.spherical_relp_model,
)
ref_predictor(refined_reflections)
else:
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement)
except (DialsRefineConfigError,
DialsRefineRuntimeError) as e:
if len(experiments) == 1:
raise DialsIndexRefineError(str(e))
had_refinement_error = True
logger.info("Refinement failed:")
logger.info(e)
del experiments[-1]
# remove experiment id from the reflections associated
# with this deleted experiment - indexed flag removed
# below
last = len(experiments)
sel = refined_reflections["id"] == last
logger.info("Removing %d reflections with id %d" %
(sel.count(True), last))
refined_reflections["id"].set_selected(sel, -1)
break
self._unit_cell_volume_sanity_check(experiments,
refined_experiments)
self.refined_reflections = refined_reflections
self.refined_reflections.unset_flags(
self.refined_reflections["id"] < 0,
self.refined_reflections.flags.indexed,
)
for i, expt in enumerate(self.experiments):
ref_sel = self.refined_reflections.select(
self.refined_reflections["imageset_id"] == i)
ref_sel = ref_sel.select(ref_sel["id"] >= 0)
for i_expt in set(ref_sel["id"]):
refined_expt = refined_experiments[i_expt]
expt.detector = refined_expt.detector
expt.beam = refined_expt.beam
expt.goniometer = refined_expt.goniometer
expt.scan = refined_expt.scan
refined_expt.imageset = expt.imageset
if not (self.all_params.refinement.parameterisation.beam.fix
== "all" and self.all_params.refinement.
parameterisation.detector.fix == "all"):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
self.reflections.map_centroids_to_reciprocal_space(
self.experiments)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
logger.info("\nRefined crystal models:")
self.show_experiments(self.refined_experiments,
self.reflections,
d_min=self.d_min)
if (i_cycle >= 2 and self.d_min
== self.params.refinement_protocol.d_min_final):
logger.info(
"Target d_min_final reached: finished with refinement")
break
if self.refined_experiments is None:
raise DialsIndexRefineError(
"None of the experiments could refine.")
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices import (
rotation_matrix_differences, )
logger.info(
rotation_matrix_differences(
self.refined_experiments.crystals()))
self._xyzcal_mm_to_px(self.refined_experiments,
self.refined_reflections)
def from_filenames(
filenames,
unhandled=None,
compare_beam=None,
compare_detector=None,
compare_goniometer=None,
scan_tolerance=None,
format_kwargs=None,
load_models=True,
):
"""Create a list of data blocks from a list of directory or file names."""
experiments = ExperimentList()
# Process each file given by this path list
to_process = _openingpathiterator(filenames)
find_format = dxtbx.datablock.FormatChecker()
format_groups = collections.OrderedDict()
if format_kwargs is None:
format_kwargs = {}
for filename in to_process:
# We now have a file, pre-opened by Format.open_file (therefore
# cached). Determine its type, and prepare to put into a group
format_class = find_format.find_format(filename)
# Verify this makes sense
if not format_class:
# No format class found?
logger.debug("Could not determine format for %s", filename)
if unhandled is not None:
unhandled.append(filename)
elif format_class.is_abstract():
logger.debug(
f"Image file {filename} appears to be a '{format_class.__name__}', but this is an abstract Format"
)
# Invalid format class found?
if unhandled is not None:
unhandled.append(filename)
elif issubclass(format_class, FormatMultiImage):
imageset = format_class.get_imageset(
os.path.abspath(filename), format_kwargs=format_kwargs)
format_groups.setdefault(format_class, []).append(imageset)
logger.debug("Loaded file: %s", filename)
else:
format_object = format_class(filename, **format_kwargs)
meta = ImageMetadataRecord.from_format(format_object)
assert meta.filename == filename
# Add this entry to our table of formats
format_groups.setdefault(format_class, []).append(meta)
logger.debug("Loaded metadata of file: %s", filename)
# Now, build experiments from these files. Duplicating the logic of
# the previous implementation:
# - FormatMultiImage files each have their own ImageSet
# - Every set of images forming a scan goes into its own ImageSequence
# - Any consecutive still frames that share any metadata with the
# previous still fram get collected into one ImageSet
# Treat each format as a separate datablock
for format_class, records in format_groups.items():
if issubclass(format_class, FormatMultiImage):
for imageset in records:
experiments.extend(
ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal=None, load_models=load_models))
continue
# Merge any consecutive and identical metadata together
_merge_model_metadata(
records,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
)
records = _merge_scans(records, scan_tolerance=scan_tolerance)
imagesets = _convert_to_imagesets(records, format_class,
format_kwargs)
imagesets = list(imagesets)
# Validate this datablock and store it
assert imagesets, "Datablock got no imagesets?"
for imageset in imagesets:
experiments.extend(
ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal=None, load_models=load_models))
return experiments
def from_templates(templates, **kwargs):
"""Import an experiment list from templates"""
assert "verbose" not in kwargs, "The verbose parameter has been removed"
assert len(templates) > 0
experiments = ExperimentList()
find_format = dxtbx.datablock.FormatChecker()
# For each template do an import
for template in templates:
template = os.path.normpath(template)
filenames = sorted(locate_files_matching_template_string(template))
if len(filenames):
logger.debug(
"The following files matched the template string:\n%s",
"\n".join(f" {p}" for p in filenames),
)
# Check if we've matched any filenames
if len(filenames) == 0:
raise ValueError(
f"Template '{template}' does not match any files")
# Get the format from the first image
format_class = find_format.find_format(filenames[0])
# Verify this makes sense
if format_class is None:
raise ValueError(
f"Image file {filenames[0]} format is unknown")
elif format_class.is_abstract():
raise ValueError(
f"Image file {filenames[0]} appears to be a '{type(format_class).__name__}', but this is an abstract Format"
)
else:
index = slice(*template_string_number_index(template))
image_range = kwargs.get("image_range")
if image_range:
first, last = image_range
else:
first = int(filenames[0][index])
last = int(filenames[-1][index])
# Check all images in range are present - if allowed
if not kwargs.get("allow_incomplete_sequences", False):
all_numbers = {int(f[index]) for f in filenames}
missing = set(range(first, last + 1)) - all_numbers
if missing:
raise ValueError(
"Missing image{} {} from imageset ({}-{})".format(
"s" if len(missing) > 1 else "",
", ".join(str(x) for x in sorted(missing)),
first,
last,
))
# Read the image
fmt = format_class(filenames[0],
**(kwargs.get("format_kwargs", {})))
# Update the image range
image_range = (first, last)
scan = fmt.get_scan()
scan.set_image_range(image_range)
# Create the sequence and experiment
imageset = dxtbx.imageset.ImageSetFactory.make_sequence(
template,
list(range(first, last + 1)),
format_class,
fmt.get_beam(),
fmt.get_detector(),
fmt.get_goniometer(),
scan,
format_kwargs=kwargs.get("format_kwargs"),
)
experiments.extend(
ExperimentListFactory.from_imageset_and_crystal(
imageset,
crystal=None,
load_models=True,
))
return experiments
