def __init__(self, templates, verbose=False):
''' Import the datablocks from the given templates. '''
from dxtbx.sweep_filenames import locate_files_matching_template_string
assert(len(templates) > 0)
self.datablocks = []
# A function to append or create a new datablock
def append_to_datablocks(iset):
try:
self.datablocks[-1].append(iset)
except Exception:
self.datablocks.append(DataBlock([iset]))
if verbose:
print 'Added imageset to datablock %d' % (len(self.datablocks) - 1)
# For each template do an import
for template in templates:
paths = sorted(locate_files_matching_template_string(template))
if verbose:
print 'The following files matched the template string:'
if len(paths) > 0:
for p in paths:
print ' %s' % p
else:
print ' No files found'
# Get the format from the first image
find_format = FormatChecker(verbose=verbose)
fmt = find_format(paths[0])
if fmt is None:
raise RuntimeError('Image file %s format is unknown' % paths[0])
else:
imageset = self._create_imageset(fmt, template, paths)
append_to_datablocks(imageset)
def __init__(self, templates, verbose=False, **kwargs):
''' Import the datablocks from the given templates. '''
from dxtbx.sweep_filenames import locate_files_matching_template_string
from libtbx.utils import Sorry
from os.path import normpath
assert(len(templates) > 0)
self.datablocks = []
# A function to append or create a new datablock
def append_to_datablocks(iset):
try:
self.datablocks[-1].append(iset)
except Exception:
self.datablocks.append(DataBlock([iset]))
if verbose:
print('Added imageset to datablock %d' % (len(self.datablocks) - 1))
# For each template do an import
for template in templates:
template = normpath(template)
paths = sorted(locate_files_matching_template_string(template))
if verbose:
print('The following files matched the template string:')
if len(paths) > 0:
for p in paths:
print(' %s' % p)
else:
print(' No files found')
# Check if we've matched any filenames
if len(paths) == 0:
raise Sorry('Template "%s" does not match any files' % template)
# Get the format from the first image
find_format = FormatChecker(verbose=verbose)
fmt = find_format(paths[0])
if fmt is None:
raise Sorry('Image file %s format is unknown' % paths[0])
elif fmt.ignore():
raise Sorry('Image file %s format will be ignored' % paths[0])
else:
imageset = self._create_imageset(fmt, template, paths, **kwargs)
append_to_datablocks(imageset)
def __init__(self, templates, verbose=False, **kwargs):
''' Import the datablocks from the given templates. '''
from dxtbx.sweep_filenames import locate_files_matching_template_string
from libtbx.utils import Sorry
assert(len(templates) > 0)
self.datablocks = []
# A function to append or create a new datablock
def append_to_datablocks(iset):
try:
self.datablocks[-1].append(iset)
except Exception:
self.datablocks.append(DataBlock([iset]))
if verbose:
print 'Added imageset to datablock %d' % (len(self.datablocks) - 1)
# For each template do an import
for template in templates:
paths = sorted(locate_files_matching_template_string(template))
if verbose:
print 'The following files matched the template string:'
if len(paths) > 0:
for p in paths:
print ' %s' % p
else:
print ' No files found'
# Check if we've matched any filenames
if len(paths) == 0:
raise Sorry('Template "%s" does not match any files' % template)
# Get the format from the first image
find_format = FormatChecker(verbose=verbose)
fmt = find_format(paths[0])
if fmt is None:
raise Sorry('Image file %s format is unknown' % paths[0])
elif fmt.ignore():
raise Sorry('Image file %s format will be ignored' % paths[0])
else:
imageset = self._create_imageset(fmt, template, paths, **kwargs)
append_to_datablocks(imageset)
def load_imagesets(
template,
directory,
id_image=None,
image_range=None,
use_cache=True,
reversephi=False,
):
global imageset_cache
from dxtbx.model.experiment_list import ExperimentListFactory
from xia2.Applications.xia2setup import known_hdf5_extensions
from dxtbx.imageset import ImageSweep
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.model.experiment_list import BeamComparison
from dxtbx.model.experiment_list import DetectorComparison
from dxtbx.model.experiment_list import GoniometerComparison
params = PhilIndex.params.xia2.settings
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction,
)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin,
)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation,
)
scan_tolerance = params.input.tolerance.scan.oscillation
format_kwargs = {
"dynamic_shadowing": params.input.format.dynamic_shadowing,
"multi_panel": params.input.format.multi_panel,
}
if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions:
# if we are passed the correct file, use this, else look for a master
# file (i.e. something_master.h5)
if os.path.exists(full_template_path) and os.path.isfile(
full_template_path
):
master_file = full_template_path
else:
import glob
g = glob.glob(os.path.join(directory, "*_master.h5"))
master_file = None
for p in g:
substr = longest_common_substring(template, p)
if substr:
if master_file is None or (
len(substr)
> len(longest_common_substring(template, master_file))
):
master_file = p
if master_file is None:
raise RuntimeError("Can't find master file for %s" % full_template_path)
unhandled = []
experiments = ExperimentListFactory.from_filenames(
[master_file],
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
assert len(unhandled) == 0, (
"unhandled image files identified: %s" % unhandled
)
else:
from dxtbx.sweep_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(
locate_files_matching_template_string(full_template_path)
)
unhandled = []
experiments = ExperimentListFactory.from_filenames(
paths,
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
assert len(unhandled) == 0, (
"unhandled image files identified: %s" % unhandled
)
else:
from dxtbx.model.experiment_list import ExperimentListTemplateImporter
importer = ExperimentListTemplateImporter(
[full_template_path], format_kwargs=format_kwargs
)
experiments = importer.experiments
imagesets = [
iset for iset in experiments.imagesets() if isinstance(iset, ImageSweep)
]
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = collections.OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple(-g for g in goniometer.get_rotation_axis())
)
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
# Update the geometry
params = PhilIndex.params.xia2.settings
update_geometry = []
from dials.command_line.dials_import import ManualGeometryUpdater
from dials.util.options import geometry_phil_scope
# Then add manual geometry
work_phil = geometry_phil_scope.format(params.input)
diff_phil = geometry_phil_scope.fetch_diff(source=work_phil)
if diff_phil.as_str() != "":
update_geometry.append(ManualGeometryUpdater(params.input))
imageset_list = []
for imageset in imagesets:
for updater in update_geometry:
imageset = updater(imageset)
imageset_list.append(imageset)
imagesets = imageset_list
from scitbx.array_family import flex
for imageset in imagesets:
scan = imageset.get_scan()
exposure_times = scan.get_exposure_times()
epochs = scan.get_epochs()
if exposure_times.all_eq(0) or exposure_times[0] == 0:
exposure_times = flex.double(exposure_times.size(), 1)
scan.set_exposure_times(exposure_times)
elif not exposure_times.all_gt(0):
exposure_times = flex.double(exposure_times.size(), exposure_times[0])
scan.set_exposure_times(exposure_times)
if epochs.size() > 1 and not epochs.all_gt(0):
if epochs[0] == 0:
epochs[0] = 1
for i in range(1, epochs.size()):
epochs[i] = epochs[i - 1] + exposure_times[i - 1]
scan.set_epochs(epochs)
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (
image_range[0] >= scan_image_range[0]
and image_range[1] <= scan_image_range[1]
):
imagesets = [
imageset[
image_range[0]
- scan_image_range[0] : image_range[1]
+ 1
- scan_image_range[0]
]
]
assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, len(
imagesets[0]
)
return imagesets
return imageset_cache[full_template_path].values()
def load_imagesets(template, directory, id_image=None, image_range=None,
use_cache=True, reversephi=False):
global imageset_cache
from dxtbx.datablock import DataBlockFactory
from xia2.Applications.xia2setup import known_hdf5_extensions
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
params = PhilIndex.params.xia2.settings
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
format_kwargs = {
'dynamic_shadowing' : params.input.format.dynamic_shadowing,
'multi_panel' : params.input.format.multi_panel,
}
if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions:
import glob
g = glob.glob(os.path.join(directory, '*_master.h5'))
master_file = None
for p in g:
substr = longest_common_substring(template, p)
if substr:
if (master_file is None or
(len(substr) > len(longest_common_substring(template, master_file)))):
master_file = p
if master_file is None:
raise RuntimeError("Can't find master file for %s" %full_template_path)
unhandled = []
datablocks = DataBlockFactory.from_filenames(
[master_file], verbose=False, unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.sweep_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(locate_files_matching_template_string(full_template_path))
unhandled = []
datablocks = DataBlockFactory.from_filenames(
paths, verbose=False, unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter(
[full_template_path], kwargs=format_kwargs)
datablocks = importer.datablocks
imagesets = datablocks[0].extract_sweeps()
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple((-g for g in goniometer.get_rotation_axis())))
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
from dials.command_line.dials_import import ManualGeometryUpdater
update_geometry = ManualGeometryUpdater(
PhilIndex.params.xia2.settings.input)
imagesets = [update_geometry(imageset) for imageset in imagesets]
from scitbx.array_family import flex
for imageset in imagesets:
scan = imageset.get_scan()
exposure_times = scan.get_exposure_times()
epochs = scan.get_epochs()
if exposure_times.all_eq(0):
exposure_times = flex.double(exposure_times.size(), 1)
scan.set_exposure_times(exposure_times)
elif not exposure_times.all_gt(0):
exposure_times = flex.double(exposure_times.size(), exposure_times[0])
scan.set_exposure_times(exposure_times)
if epochs.size() > 1 and not epochs.all_gt(0):
for i in range(1, epochs.size()):
epochs[i] = epochs[i-1] + exposure_times[i-1]
scan.set_epochs(epochs)
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (image_range[0] >= scan_image_range[0] and
image_range[1] <= scan_image_range[1]):
imagesets = [imageset[
image_range[0] - scan_image_range[0]:
image_range[1] + 1 - scan_image_range[0]]]
assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, \
len(imagesets[0])
return imagesets
return imageset_cache[full_template_path].values()
def load_imagesets(template, directory, id_image=None, image_range=None,
use_cache=True, reversephi=False):
global imageset_cache
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.datablock import DataBlockFactory
from dxtbx.sweep_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(locate_files_matching_template_string(full_template_path))
unhandled = []
datablocks = DataBlockFactory.from_filenames(
paths, verbose=False, unhandled=unhandled)
assert len(unhandled) == 0, "unhandled image files identified: %s" % \
unhandled
assert len(datablocks) == 1, "1 datablock expected, %d found" % \
len(datablocks)
else:
from dxtbx.datablock import DataBlockTemplateImporter
importer = DataBlockTemplateImporter([full_template_path])
datablocks = importer.datablocks
imagesets = datablocks[0].extract_sweeps()
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple((-g for g in goniometer.get_rotation_axis())))
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
for imageset in imagesets:
scan = imageset.get_scan()
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (image_range[0] >= scan_image_range[0] and
image_range[1] <= scan_image_range[1]):
imagesets = [imageset[
image_range[0] - scan_image_range[0]:
image_range[1] + 1 - scan_image_range[0]]]
assert len(imagesets[0]) == image_range[1] - image_range[0] + 1, \
len(imagesets[0])
return imagesets
return imageset_cache[full_template_path].values()
