def test_cbf_writer(image_file, dials_regression, run_in_tmpdir):
filename = os.path.join(dials_regression, image_file)
datablock = DataBlockFactory.from_filenames([filename])[0]
imageset = datablock.extract_imagesets()[0]
FormatCBFMini.as_file(
imageset.get_detector(),
imageset.get_beam(),
imageset.get_goniometer(),
imageset.get_scan(),
imageset.get_raw_data(0)[0],
"image_0001.cbf",
)
assert datablock.format_class()
datablock2 = DataBlockFactory.from_filenames(["image_0001.cbf"])[0]
imageset2 = datablock2.extract_imagesets()[0]
tolerance = tolerance_phil_scope.extract().tolerance
diff = SweepDiff(tolerance)
print("\n".join(diff(imageset, imageset2)))
assert BeamComparison()(imageset.get_beam(), imageset2.get_beam())
assert DetectorComparison(origin_tolerance=tolerance.detector.origin)(
imageset.get_detector(), imageset2.get_detector())
assert GoniometerComparison()(imageset.get_goniometer(),
imageset2.get_goniometer())
s1 = imageset.get_scan()
s2 = imageset.get_scan()
assert s1.get_exposure_times() == s2.get_exposure_times()
assert s1.get_oscillation() == s2.get_oscillation()
assert s1.get_image_range() == s2.get_image_range()
assert imageset.get_raw_data(0) == imageset2.get_raw_data(0)
def combine(datablock_list, reflections_list, params):
'''
Combine the found spots.
'''
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
from dxtbx.datablock import DataBlock
from dxtbx.imageset import ImageSetFactory
from dials.algorithms.spot_finding import StrongSpotCombiner
from dials.array_family import flex
assert len(datablock_list) == len(reflections_list)
# Get a list of imagesets
imageset_list = []
for db in datablock_list:
iset = db.extract_imagesets()
assert len(iset) == 1
imageset_list.append(iset[0])
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
# The initial models
format_class = imageset_list[0].get_format_class()
beam = imageset_list[0].get_beam()
detector = imageset_list[0].get_detector()
goniometer = imageset_list[0].get_goniometer()
scan = imageset_list[0].get_scan()
template = imageset_list[0].get_template()
# Check all the models
for imageset in imageset_list[1:]:
b = imageset.get_beam()
d = imageset.get_detector()
g = imageset.get_goniometer()
s = imageset.get_scan()
if not imageset.get_format_class() == format_class:
raise RuntimeError('Format classes do not match')
if not imageset.get_template() == template:
raise RuntimeError('Templates do not match')
if not compare_beam(beam, b):
raise RuntimeError('Beam models are too dissimilar')
if not compare_detector(detector, d):
raise RuntimeError('Detector models are too dissimilar')
if not compare_goniometer(goniometer, g):
raise RuntimeError('Goniometer models are too dissimilar')
try:
scan.append(s, scan_tolerance=scan_tolerance)
except Exception:
raise RuntimeError('Scans do not match')
# Get the image range
image_range = scan.get_image_range()
image_range = (image_range[0], image_range[1]+1)
# Create the sweep
imageset = ImageSetFactory.make_sweep(
template, range(*image_range),
format_class,
beam, detector,
goniometer, scan)
# Combine spots
combiner = StrongSpotCombiner()
for index, rlist in enumerate(reflections_list, start=1):
assert rlist['id'].all_eq(0)
logger.info("Combining %d reflections from reflection list %d" % (
len(rlist),
index))
combiner.add(rlist['shoebox'])
shoeboxes = combiner.shoeboxes()
# Calculate the spot centroids and intensities
logger.info('Combined into %d reflections' % len(shoeboxes))
centroid = shoeboxes.centroid_valid()
logger.info('Calculated {0} spot centroids'.format(len(shoeboxes)))
intensity = shoeboxes.summed_intensity()
logger.info('Calculated {0} spot intensities'.format(len(shoeboxes)))
# Construct the reflection table
reflections = flex.reflection_table(
flex.observation(
shoeboxes.panels(),
centroid,
intensity),
shoeboxes)
reflections['id'] = flex.int(len(reflections), 0)
reflections.set_flags(
flex.size_t_range(len(reflections)),
reflections.flags.strong)
# Return the datablock and reflections
return DataBlock([imageset]), reflections
def parse_args(self,
args,
verbose=False,
return_unhandled=False,
quick_parse=False):
'''
Parse the command line arguments.
:param args: The input arguments
:param verbose: Print verbose output
:param return_unhandled: True/False also return unhandled arguments
:param quick_parse: Return as fast as possible and without reading any data,
ignoring class constructor options.
:return: The options and parameters and (optionally) unhandled arguments
'''
import os.path
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
from dials.util.phil import parse
# Parse the command line phil parameters
user_phils = []
unhandled = []
interpretor = self.system_phil.command_line_argument_interpreter()
for arg in args:
if os.path.isfile(arg) and os.path.getsize(arg) > 0:
name, ext = os.path.splitext(arg)
if ext in ['.phil', '.param', '.params', '.eff', '.def']:
try:
user_phils.append(parse(file_name=arg))
except Exception:
if return_unhandled:
unhandled.append(arg)
else:
raise
else:
unhandled.append(arg)
elif arg.find("=") >= 0:
try:
user_phils.append(interpretor.process_arg(arg=arg))
except Exception:
if return_unhandled:
unhandled.append(arg)
else:
raise
else:
unhandled.append(arg)
# Fetch the phil parameters
self._phil, unused = self.system_phil.fetch(
sources=user_phils, track_unused_definitions=True)
# Print if bad definitions
if len(unused) > 0:
msg = [item.object.as_str().strip() for item in unused]
msg = '\n'.join([' %s' % line for line in msg])
raise RuntimeError(
'The following definitions were not recognised\n%s' % msg)
# Extract the parameters
params = self._phil.extract()
# Stop at this point if quick_parse is set. A second pass may be needed.
if quick_parse:
return params, unhandled
# Create some comparison functions
if self._read_datablocks_from_images:
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.
polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.
polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.
rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.
fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.
setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
# FIXME Should probably make this smarter since it requires editing here
# and in dials.import phil scope
try:
format_kwargs = {
'dynamic_shadowing': params.format.dynamic_shadowing,
'multi_panel': params.format.multi_panel,
}
except Exception:
format_kwargs = None
else:
compare_beam = None
compare_detector = None
compare_goniometer = None
scan_tolerance = None
format_kwargs = None
# Try to import everything
importer = Importer(
unhandled,
read_datablocks=self._read_datablocks,
read_experiments=self._read_experiments,
read_reflections=self._read_reflections,
read_datablocks_from_images=self._read_datablocks_from_images,
check_format=self._check_format,
verbose=verbose,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
# Grab a copy of the errors that occured in case the caller wants them
self.handling_errors = importer.handling_errors
# Add the cached arguments
for obj in importer.datablocks:
params.input.datablock.append(obj)
for obj in importer.experiments:
params.input.experiments.append(obj)
for obj in importer.reflections:
params.input.reflections.append(obj)
# Convert to phil
self._phil = self.system_phil.format(python_object=params)
return params, importer.unhandled
def __call__(self, experiment):
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
if self.tolerance:
compare_beam = BeamComparison(
wavelength_tolerance=self.tolerance.beam.wavelength,
direction_tolerance=self.tolerance.beam.direction,
polarization_normal_tolerance=self.tolerance.beam.
polarization_normal,
polarization_fraction_tolerance=self.tolerance.beam.
polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=self.tolerance.detector.fast_axis,
slow_axis_tolerance=self.tolerance.detector.slow_axis,
origin_tolerance=self.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=self.tolerance.goniometer.
rotation_axis,
fixed_rotation_tolerance=self.tolerance.goniometer.
fixed_rotation,
setting_rotation_tolerance=self.tolerance.goniometer.
setting_rotation)
else:
compare_beam = None
compare_detector = None
compare_goniometer = None
if self.ref_beam:
if compare_beam:
assert (compare_beam(self.ref_beam, experiment.beam))
beam = self.ref_beam
else:
beam = experiment.beam
if self.ref_detector and self.average_detector:
detector = self.ref_detector
elif self.ref_detector and not self.average_detector:
if compare_detector:
assert (compare_detector(self.ref_detector,
experiment.detector))
detector = self.ref_detector
else:
detector = experiment.detector
if self.ref_goniometer:
if compare_goniometer:
assert (compare_goniometer(self.ref_goniometer,
experiment.goniometer))
goniometer = self.ref_goniometer
else:
goniometer = experiment.goniometer
if self.ref_scan:
scan = self.ref_scan
else:
scan = experiment.scan
if self.ref_crystal:
crystal = self.ref_crystal
else:
crystal = experiment.crystal
from dxtbx.model.experiment_list import Experiment
return Experiment(beam=beam,
detector=detector,
scan=scan,
goniometer=goniometer,
crystal=crystal,
imageset=experiment.imageset)
def load_imagesets(template,
directory,
id_image=None,
image_range=None,
use_cache=True,
reversephi=False):
global imageset_cache
from dxtbx.datablock import DataBlockFactory
from xia2.Applications.xia2setup import known_hdf5_extensions
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
params = PhilIndex.params.xia2.settings
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.
polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.
polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.
rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.
fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.
setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
format_kwargs = {
'dynamic_shadowing': params.input.format.dynamic_shadowing,
'multi_panel': params.input.format.multi_panel,
}
if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions:
import glob
g = glob.glob(os.path.join(directory, '*_master.h5'))
master_file = None
for p in g:
substr = longest_common_substring(template, p)
if substr:
if (master_file is None or (len(substr) > len(
longest_common_substring(template, master_file)))):
master_file = p
if master_file is None and \
os.path.exists(full_template_path) and \
os.path.isfile(full_template_path):
master_file = full_template_path
if master_file is None:
raise RuntimeError("Can't find master file for %s" %
full_template_path)
unhandled = []
datablocks = DataBlockFactory.from_filenames(
[master_file],
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.sweep_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(
locate_files_matching_template_string(full_template_path))
unhandled = []
datablocks = DataBlockFactory.from_filenames(
paths,
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter(
[full_template_path], format_kwargs=format_kwargs)
datablocks = importer.datablocks
imagesets = datablocks[0].extract_sweeps()
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = collections.OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple((-g for g in goniometer.get_rotation_axis())))
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
# Update the geometry
params = PhilIndex.params.xia2.settings
update_geometry = []
from dials.command_line.dials_import import ManualGeometryUpdater
from dials.util.options import geometry_phil_scope
# Then add manual geometry
work_phil = geometry_phil_scope.format(params.input)
diff_phil = geometry_phil_scope.fetch_diff(source=work_phil)
if diff_phil.as_str() != "":
update_geometry.append(ManualGeometryUpdater(params.input))
imageset_list = []
for imageset in imagesets:
for updater in update_geometry:
imageset = updater(imageset)
imageset_list.append(imageset)
imagesets = imageset_list
from scitbx.array_family import flex
for imageset in imagesets:
scan = imageset.get_scan()
exposure_times = scan.get_exposure_times()
epochs = scan.get_epochs()
if exposure_times.all_eq(0) or exposure_times[0] == 0:
exposure_times = flex.double(exposure_times.size(), 1)
scan.set_exposure_times(exposure_times)
elif not exposure_times.all_gt(0):
exposure_times = flex.double(exposure_times.size(),
exposure_times[0])
scan.set_exposure_times(exposure_times)
if epochs.size() > 1 and not epochs.all_gt(0):
if epochs[0] == 0:
epochs[0] = 1
for i in range(1, epochs.size()):
epochs[i] = epochs[i - 1] + exposure_times[i - 1]
scan.set_epochs(epochs)
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (image_range[0] >= scan_image_range[0]
and image_range[1] <= scan_image_range[1]):
imagesets = [
imageset[image_range[0] -
scan_image_range[0]:image_range[1] + 1 -
scan_image_range[0]]
]
assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, \
len(imagesets[0])
return imagesets
return imageset_cache[full_template_path].values()