def test_imagesetfactory(centroid_files, dials_regression):
from dxtbx.imageset import ImageSetFactory, ImageSweep
filenames = centroid_files
sweep = ImageSetFactory.new(filenames)
assert isinstance(sweep[0], ImageSweep)
template = os.path.join(dials_regression, "centroid_test_data",
"centroid_####.cbf")
image_range = (3, 6)
sweep = ImageSetFactory.from_template(template, image_range)
assert isinstance(sweep[0], ImageSweep)
assert len(sweep[0]) == 4
assert sweep[0].paths()[0].endswith("3.cbf")
assert sweep[0].paths()[-1].endswith("6.cbf")
imageset = ImageSetFactory.make_imageset(filenames)
assert len(imageset) == 9
imageset = ImageSetFactory.make_imageset(filenames, check_format=False)
assert len(imageset) == 9
sweep = ImageSetFactory.make_sweep(template, list(range(1, 9 + 1)))
assert len(sweep) == 9
sweep = ImageSetFactory.make_sweep(template, list(range(3, 6 + 1)))
assert len(sweep) == 4
def run(self):
from dxtbx.imageset import ImageSetFactory, ImageSweep
from os.path import join
filenames = self.get_file_list()
sweep = ImageSetFactory.new(filenames)
assert(isinstance(sweep[0], ImageSweep) == True)
print 'OK'
template = join(dials_regression, "centroid_test_data", "centroid_####.cbf")
image_range = (3, 6)
sweep = ImageSetFactory.from_template(template, image_range)
assert(isinstance(sweep[0], ImageSweep) == True)
assert len(sweep[0]) == 4
assert sweep[0].paths()[0].endswith("3.cbf")
assert sweep[0].paths()[-1].endswith("6.cbf")
print 'OK'
imageset = ImageSetFactory.make_imageset(filenames)
assert len(imageset) == 9
print 'OK'
imageset = ImageSetFactory.make_imageset(
filenames,
check_format=False)
assert len(imageset) == 9
print 'OK'
sweep = ImageSetFactory.make_sweep(
template,
list(range(1, 9+1)))
assert len(sweep) == 9
print 'OK'
sweep = ImageSetFactory.make_sweep(
template,
list(range(3, 6+1)))
assert len(sweep) == 4
print 'OK'
def imagesweep_from_dict(d, check_format=True, directory=None):
"""Construct and image sweep from the dictionary."""
# Get the template (required)
template = load_path(str(d["template"]), directory=directory)
# If the scan isn't set, find all available files
scan_dict = d.get("scan")
if scan_dict is None:
image_range = None
else:
image_range = scan_dict.get("image_range")
# Set the models with the exisiting models as templates
beam = BeamFactory.from_dict(d.get("beam"))
goniometer = GoniometerFactory.from_dict(d.get("goniometer"))
detector = DetectorFactory.from_dict(d.get("detector"))
scan = ScanFactory.from_dict(d.get("scan"))
# Construct the sweep
try:
sweep = ImageSetFactory.from_template(
template,
image_range,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format,
)[0]
except Exception:
indices = range(image_range[0], image_range[1] + 1)
sweep = ImageSetFactory.make_sweep(
template,
indices,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format,
)
# Set some external lookups
if "mask" in d and d["mask"] is not None and d["mask"] is not "":
path = load_path(d["mask"], directory=directory)
with open(path) as infile:
sweep.external_lookup.mask.filename = path
sweep.external_lookup.mask.data = ImageBool(pickle.load(infile))
if "gain" in d and d["gain"] is not None and d["gain"] is not "":
path = load_path(d["gain"], directory=directory)
with open(path) as infile:
sweep.external_lookup.gain.filename = path
sweep.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] is not "":
path = load_path(d["pedestal"], directory=directory)
with open(path) as infile:
sweep.external_lookup.pedestal.filename = path
sweep.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
# Return the sweep
return sweep
def convert_stills_to_sweep(self, imageset):
from dxtbx.model import Scan
assert self.params.geometry.scan.oscillation is not None
for i in range(len(imageset)):
self.override_beam(
imageset.get_beam(index=i),
self.params.geometry.beam)
self.override_detector(
imageset.get_detector(index=i),
self.params.geometry.detector)
self.override_goniometer(
imageset.get_goniometer(index=i),
self.params.geometry.goniometer)
beam = imageset.get_beam(index=0)
detector = imageset.get_detector(index=0)
goniometer = imageset.get_goniometer(index=0)
for i in range(1, len(imageset)):
assert beam.is_similar_to(
imageset.get_beam(index=i),
wavelength_tolerance = self.params.input.tolerance.beam.wavelength,
direction_tolerance = self.params.input.tolerance.beam.direction,
polarization_normal_tolerance = self.params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance = self.params.input.tolerance.beam.polarization_fraction)
assert detector.is_similar_to(
imageset.get_detector(index=i),
fast_axis_tolerance = self.params.input.tolerance.detector.fast_axis,
slow_axis_tolerance = self.params.input.tolerance.detector.slow_axis,
origin_tolerance = self.params.input.tolerance.detector.origin)
assert goniometer.is_similar_to(
imageset.get_goniometer(index=i),
rotation_axis_tolerance = self.params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance = self.params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance = self.params.input.tolerance.goniometer.setting_rotation)
assert beam is not None
assert detector is not None
assert goniometer is not None
if self.params.geometry.scan.image_range is not None:
image_range = self.params.geometry.scan.image_range
else:
image_range = (1, len(imageset))
oscillation = self.params.geometry.scan.oscillation
scan = Scan(image_range=image_range, oscillation=oscillation)
from dxtbx.sweep_filenames import template_regex
from dxtbx.imageset import ImageSetFactory
indices = list(range(image_range[0], image_range[1]+1))
template = template_regex(imageset.get_path(0))[0]
if template is None:
paths = [imageset.get_path(i) for i in range(len(imageset))]
assert len(set(paths)) == 1
template = paths[0]
new_sweep = ImageSetFactory.make_sweep(
template = template,
indices = indices,
format_class = imageset.reader().get_format_class(),
beam = beam,
detector = detector,
goniometer = goniometer,
scan = scan)
return new_sweep
def convert_stills_to_sweep(self, imageset):
from dxtbx.model import Scan
assert self.params.geometry.scan.oscillation is not None
for i in range(len(imageset)):
self.override_beam(
imageset.get_beam(index=i),
self.params.geometry.beam)
self.override_detector(
imageset.get_detector(index=i),
self.params.geometry.detector)
self.override_goniometer(
imageset.get_goniometer(index=i),
self.params.geometry.goniometer)
beam = imageset.get_beam(index=0)
detector = imageset.get_detector(index=0)
goniometer = imageset.get_goniometer(index=0)
for i in range(1, len(imageset)):
assert beam.is_similar_to(
imageset.get_beam(index=i),
wavelength_tolerance = self.params.input.tolerance.beam.wavelength,
direction_tolerance = self.params.input.tolerance.beam.direction,
polarization_normal_tolerance = self.params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance = self.params.input.tolerance.beam.polarization_fraction)
assert detector.is_similar_to(
imageset.get_detector(index=i),
fast_axis_tolerance = self.params.input.tolerance.detector.fast_axis,
slow_axis_tolerance = self.params.input.tolerance.detector.slow_axis,
origin_tolerance = self.params.input.tolerance.detector.origin)
assert goniometer.is_similar_to(
imageset.get_goniometer(index=i),
rotation_axis_tolerance = self.params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance = self.params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance = self.params.input.tolerance.goniometer.setting_rotation)
assert beam is not None
assert detector is not None
assert goniometer is not None
if self.params.geometry.scan.image_range is not None:
image_range = self.params.geometry.scan.image_range
else:
image_range = (1, len(imageset))
oscillation = self.params.geometry.scan.oscillation
scan = Scan(image_range=image_range, oscillation=oscillation)
from dxtbx.sweep_filenames import template_regex
from dxtbx.imageset import ImageSetFactory
indices = list(range(image_range[0], image_range[1]+1))
template = template_regex(imageset.get_path(0))[0]
if template is None:
paths = [imageset.get_path(i) for i in range(len(imageset))]
assert len(set(paths)) == 1
template = paths[0]
new_sweep = ImageSetFactory.make_sweep(
template = template,
indices = indices,
format_class = imageset.reader().get_format_class(),
beam = beam,
detector = detector,
goniometer = goniometer,
scan = scan)
return new_sweep
def imagesweep_from_dict(d, check_format=True):
'''Construct and image sweep from the dictionary.'''
from dxtbx.imageset import ImageSetFactory
from dxtbx.model import BeamFactory, DetectorFactory, GoniometerFactory, ScanFactory
from dxtbx.serialize.filename import load_path
# Get the template (required)
template = load_path(str(d['template']))
# If the scan isn't set, find all available files
scan_dict = d.get('scan')
if scan_dict is None:
image_range = None
else:
image_range = scan_dict.get('image_range')
# Set the models with the exisiting models as templates
beam = BeamFactory.from_dict(d.get('beam'))
goniometer = GoniometerFactory.from_dict(d.get('goniometer'))
detector = DetectorFactory.from_dict(d.get('detector'))
scan = ScanFactory.from_dict(d.get('scan'))
# Construct the sweep
try:
sweep = ImageSetFactory.from_template(template,
image_range,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format)[0]
except Exception:
indices = range(image_range[0], image_range[1] + 1)
sweep = ImageSetFactory.make_sweep(template,
indices,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format)
# Set some external lookups
if 'mask' in d and d['mask'] is not None and d['mask'] is not "":
with open(d['mask']) as infile:
sweep.external_lookup.mask.filename = d['mask']
sweep.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in d and d['gain'] is not None and d['gain'] is not "":
with open(d['gain']) as infile:
sweep.external_lookup.gain.filename = d['gain']
sweep.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in d and d['pedestal'] is not None and d[
'pedestal'] is not "":
with open(d['pedestal']) as infile:
sweep.external_lookup.pedestal.filename = d['pedestal']
sweep.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile))
# Return the sweep
return sweep
def imagesweep_from_dict(d, check_format=True):
'''Construct and image sweep from the dictionary.'''
from dxtbx.imageset import ImageSetFactory
from dxtbx.serialize import beam, detector, goniometer, scan
from dxtbx.serialize.filename import load_path
# Get the template (required)
template = load_path(str(d['template']))
# If the scan isn't set, find all available files
scan_dict = d.get('scan')
if scan_dict is None:
image_range = None
else:
image_range = scan_dict.get('image_range')
# Construct the sweep
try:
sweep = ImageSetFactory.from_template(
template, image_range, check_format=check_format)[0]
# Get the existing models as dictionaries
beam_dict = beam.to_dict(sweep.get_beam())
gonio_dict = goniometer.to_dict(sweep.get_goniometer())
detector_dict = detector.to_dict(sweep.get_detector())
scan_dict = scan.to_dict(sweep.get_scan())
except Exception:
indices = range(image_range[0], image_range[1] + 1)
sweep = ImageSetFactory.make_sweep(
template, indices, check_format=False)
beam_dict = None
gonio_dict = None
detector_dict = None
scan_dict = None
# Set some external lookups
if 'mask' in d and d['mask'] is not None:
with open(d['mask']) as infile:
sweep.external_lookup.mask.filename = d['mask']
sweep.external_lookup.mask.data = pickle.load(infile)
if 'gain' in d and d['gain'] is not None:
with open(d['gain']) as infile:
sweep.external_lookup.gain.filename = d['gain']
sweep.external_lookup.gain.data = pickle.load(infile)
if 'pedestal' in d and d['pedestal'] is not None:
with open(d['pedestal']) as infile:
sweep.external_lookup.pedestal.filename = d['pedestal']
sweep.external_lookup.pedestal.data = pickle.load(infile)
# Set the models with the exisiting models as templates
sweep.set_beam(beam.from_dict(d.get('beam'), beam_dict))
sweep.set_goniometer(goniometer.from_dict(d.get('goniometer'), gonio_dict))
sweep.set_detector(detector.from_dict(d.get('detector'), detector_dict))
sweep.set_scan(scan.from_dict(d.get('scan'), scan_dict))
# Return the sweep
return sweep
def convert_stills_to_sweep(self, imageset):
from dxtbx.model import Scan
assert self.params.geometry.scan.oscillation is not None
beam = imageset.get_beam(index=0)
detector = imageset.get_detector(index=0)
goniometer = imageset.get_goniometer(index=0)
for i in range(1, len(imageset)):
b_i = imageset.get_beam(i)
d_i = imageset.get_detector(i)
g_i = imageset.get_goniometer(i)
assert (beam is None and b_i is None) or beam.is_similar_to(
imageset.get_beam(index=i),
wavelength_tolerance=self.params.input.tolerance.beam.
wavelength,
direction_tolerance=self.params.input.tolerance.beam.direction,
polarization_normal_tolerance=self.params.input.tolerance.beam.
polarization_normal,
polarization_fraction_tolerance=self.params.input.tolerance.
beam.polarization_fraction)
assert (detector is None
and d_i is None) or detector.is_similar_to(
imageset.get_detector(index=i),
fast_axis_tolerance=self.params.input.tolerance.
detector.fast_axis,
slow_axis_tolerance=self.params.input.tolerance.
detector.slow_axis,
origin_tolerance=self.params.input.tolerance.detector.
origin)
assert (goniometer is None
and g_i is None) or goniometer.is_similar_to(
imageset.get_goniometer(index=i),
rotation_axis_tolerance=self.params.input.tolerance.
goniometer.rotation_axis,
fixed_rotation_tolerance=self.params.input.tolerance.
goniometer.fixed_rotation,
setting_rotation_tolerance=self.params.input.tolerance.
goniometer.setting_rotation)
oscillation = self.params.geometry.scan.oscillation
from dxtbx.sweep_filenames import template_regex_from_list
from dxtbx.imageset import ImageSetFactory
template, indices = template_regex_from_list(imageset.paths())
image_range = (min(indices), max(indices))
assert (image_range[1] + 1 - image_range[0]) == len(indices)
scan = Scan(image_range=image_range, oscillation=oscillation)
if template is None:
paths = [imageset.get_path(i) for i in range(len(imageset))]
assert len(set(paths)) == 1
template = paths[0]
new_sweep = ImageSetFactory.make_sweep(
template=template,
indices=indices,
format_class=imageset.reader().get_format_class(),
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan)
return new_sweep
def extrapolate_imageset(self,
imageset=None,
beam = None,
detector = None,
goniometer = None,
scan = None):
from dxtbx.imageset import ImageSetFactory
first, last = scan.get_image_range()
sweep = ImageSetFactory.make_sweep(
template = imageset.get_template(),
indices = range(first, last+1),
format_class = imageset.get_format_class(),
beam = beam,
detector = detector,
goniometer = goniometer,
scan = scan,
format_kwargs = imageset.params())
return sweep
def _make_sweep(self, imageset, scan):
''' Make an image sweep. '''
from dxtbx.sweep_filenames import template_image_range
from dxtbx.imageset import ImageSetFactory
from dxtbx.serialize.filename import load_path
# Get the template format
template = load_path(imageset['template'])
# Get the number of images (if no scan is given we'll try
# to find all the images matching the template
if scan is None:
i0, i1 = template_image_range(template)
else:
i0, i1 = scan.get_image_range()
# Make a sweep from the input data
return ImageSetFactory.make_sweep(template,
list(range(i0, i1+1)), None, check_format=self._check_format)
def _create_multi_file_imageset(self, format_class, records,
format_kwargs=None):
''' Create a multi file sweep or imageset. '''
from dxtbx.imageset import ImageSetFactory
from os.path import abspath
# Make either an imageset or sweep
if len(records) == 1 and records[0].template is not None:
# Get the image range
image_range = records[0].scan.get_image_range()
image_range = (image_range[0], image_range[1]+1)
# Create the sweep
imageset = ImageSetFactory.make_sweep(
abspath(records[0].template), range(*image_range),
format_class,
records[0].beam, records[0].detector,
records[0].goniometer, records[0].scan,
format_kwargs=format_kwargs)
else:
# Get the filenames
filenames = []
for r in records:
assert(r.template is None)
filenames.append(r.filename)
# make an imageset
imageset = ImageSetFactory.make_imageset(
map(abspath, filenames),
format_class,
format_kwargs=format_kwargs)
for i, r in enumerate(records):
imageset.set_beam(r.beam, i)
imageset.set_detector(r.detector, i)
imageset.set_goniometer(r.goniometer, i)
imageset.set_scan(r.scan, i)
# Return the imageset
return imageset
def _make_sweep(self,
imageset,
beam=None,
detector=None,
goniometer=None,
scan=None,
format_kwargs=None):
''' Make an image sweep. '''
from dxtbx.sweep_filenames import template_image_range
from dxtbx.imageset import ImageSetFactory
from dxtbx.serialize.filename import load_path
from dxtbx.format.FormatMultiImage import FormatMultiImage
# Get the template format
template = load_path(imageset['template'])
# Get the number of images (if no scan is given we'll try
# to find all the images matching the template
if scan is None:
i0, i1 = template_image_range(template)
else:
i0, i1 = scan.get_image_range()
format_class = None
if self._check_format is False:
if "single_file_indices" in imageset:
format_class = FormatMultiImage
# Make a sweep from the input data
return ImageSetFactory.make_sweep(
template,
list(range(i0, i1+1)),
format_class = format_class,
check_format=self._check_format,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
format_kwargs=format_kwargs)
def _create_imageset(self, format_class, template, filenames):
''' Create a multi file sweep or imageset. '''
from dxtbx.imageset import ImageSetFactory
from dxtbx.sweep_filenames import template_string_number_index
# Get the image range
index = slice(*template_string_number_index(template))
first = int(filenames[0][index])
last = int(filenames[-1][index])
# Check all images in range are present
numbers = [int(f[index]) for f in filenames]
assert(all([x+1==y for x, y in zip(numbers, numbers[1:])]))
# Read the image
fmt = format_class(filenames[0])
# Get the meta data from the format
b = fmt.get_beam()
d = fmt.get_detector()
g = fmt.get_goniometer()
s = fmt.get_scan()
# Update the image range
s.set_image_range((first, last))
# Get the image range
image_range = s.get_image_range()
image_range = (image_range[0], image_range[1]+1)
# Create the sweep
imageset = ImageSetFactory.make_sweep(
template, range(*image_range),
format_class,
b, d, g, s)
# Return the imageset
return imageset
def _make_sweep(
self,
imageset,
beam=None,
detector=None,
goniometer=None,
scan=None,
format_kwargs=None,
):
""" Make an image sweep. """
# Get the template format
template = resolve_path(imageset["template"],
directory=self._directory)
# Get the number of images (if no scan is given we'll try
# to find all the images matching the template
if scan is None:
i0, i1 = template_image_range(template)
else:
i0, i1 = scan.get_image_range()
format_class = None
if self._check_format is False:
if "single_file_indices" in imageset:
format_class = FormatMultiImage
# Make a sweep from the input data
return ImageSetFactory.make_sweep(
template,
list(range(i0, i1 + 1)),
format_class=format_class,
check_format=self._check_format,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
format_kwargs=format_kwargs,
)
def _create_imageset(self, format_class, template, filenames, **kwargs):
''' Create a multi file sweep or imageset. '''
from dxtbx.imageset import ImageSetFactory
from dxtbx.sweep_filenames import template_string_number_index
# Get the image range
index = slice(*template_string_number_index(template))
first = int(filenames[0][index])
last = int(filenames[-1][index])
# Check all images in range are present
numbers = [int(f[index]) for f in filenames]
assert(all(x+1==y for x, y in zip(numbers, numbers[1:])))
# Read the image
fmt = format_class(filenames[0])
# Get the meta data from the format
b = fmt.get_beam()
d = fmt.get_detector()
g = fmt.get_goniometer()
s = fmt.get_scan()
# Update the image range
s.set_image_range((first, last))
# Get the image range
image_range = s.get_image_range()
image_range = (image_range[0], image_range[1]+1)
# Create the sweep
imageset = ImageSetFactory.make_sweep(
template, range(*image_range),
format_class,
b, d, g, s, format_kwargs=kwargs)
# Return the imageset
return imageset
def _make_sweep(self, imageset, scan, format_kwargs=None):
''' Make an image sweep. '''
from dxtbx.sweep_filenames import template_image_range
from dxtbx.imageset import ImageSetFactory
from dxtbx.serialize.filename import load_path
# Get the template format
template = load_path(imageset['template'])
# Get the number of images (if no scan is given we'll try
# to find all the images matching the template
if scan is None:
i0, i1 = template_image_range(template)
else:
i0, i1 = scan.get_image_range()
# Make a sweep from the input data
return ImageSetFactory.make_sweep(
template,
list(range(i0, i1+1)),
None,
check_format=self._check_format,
scan=scan,
format_kwargs=format_kwargs)
def _load_datablocks(self, obj, check_format=True):
''' Create the datablock from a dictionary. '''
from libtbx.containers import OrderedDict
from dxtbx.format.Registry import Registry
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model import HierarchicalDetector
from dxtbx.serialize.filename import load_path
from dxtbx.imageset import ImageSetFactory
import pickle
# If we have a list, extract for each dictionary in the list
if isinstance(obj, list):
return [self._load_datablocks(dd, check_format) for dd in obj]
elif not isinstance(obj, dict):
raise RuntimeError('unknown datablock dictionary type')
assert(obj['__id__'] == 'DataBlock')
# Get the list of models
blist = obj.get('beam', [])
dlist = obj.get('detector', [])
glist = obj.get('goniometer', [])
slist = obj.get('scan', [])
def load_models(obj):
try:
beam = Beam.from_dict(blist[obj['beam']])
except Exception:
beam = None
try:
dobj = dlist[obj['detector']]
if 'hierarchy' in dobj:
detector = HierarchicalDetector.from_dict(dobj)
else:
detector = Detector.from_dict(dobj)
except Exception:
detector = None
try:
gonio = Goniometer.from_dict(glist[obj['goniometer']])
except Exception:
gonio = None
try:
scan = Scan.from_dict(slist[obj['scan']])
except Exception:
scan = None
return beam, detector, gonio, scan
# Loop through all the imagesets
imagesets = []
for imageset in obj['imageset']:
ident = imageset['__id__']
if ident == 'ImageSweep':
beam, detector, gonio, scan = load_models(imageset)
template = load_path(imageset['template'])
i0, i1 = scan.get_image_range()
iset = ImageSetFactory.make_sweep(
template, range(i0, i1+1), None,
beam, detector, gonio, scan, check_format)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'])
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = pickle.load(infile)
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'])
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = pickle.load(infile)
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'])
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = pickle.load(infile)
imagesets.append(iset)
elif ident == 'ImageSet':
filenames = [image['filename'] for image in imageset['images']]
iset = ImageSetFactory.make_imageset(
filenames, None, check_format)
for i, image in enumerate(imageset['images']):
beam, detector, gonio, scan = load_models(image)
iset.set_beam(beam, i)
iset.set_detector(detector, i)
if gonio:
iset.set_goniometer(gonio, i)
if scan:
iset.set_scan(scan, i)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'])
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = pickle.load(infile)
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'])
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = pickle.load(infile)
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'])
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = pickle.load(infile)
imagesets.append(iset)
else:
raise RuntimeError('expected ImageSet/ImageSweep, got %s' % ident)
# Return the datablock
return DataBlock(imagesets)
def _load_datablocks(self, obj, check_format=True):
''' Create the datablock from a dictionary. '''
from libtbx.containers import OrderedDict
from dxtbx.format.Registry import Registry
from dxtbx.model import Beam, Detector, Scan
from dxtbx.model import Detector
from dxtbx.serialize.filename import load_path
from dxtbx.imageset import ImageSetFactory, ImageGrid
# If we have a list, extract for each dictionary in the list
if isinstance(obj, list):
return [self._load_datablocks(dd, check_format) for dd in obj]
elif not isinstance(obj, dict):
raise RuntimeError('unknown datablock dictionary type')
assert(obj['__id__'] == 'DataBlock')
# Get the list of models
blist = obj.get('beam', [])
dlist = obj.get('detector', [])
glist = obj.get('goniometer', [])
slist = obj.get('scan', [])
def load_models(obj):
try:
beam = Beam.from_dict(blist[obj['beam']])
except Exception:
beam = None
try:
dobj = dlist[obj['detector']]
detector = Detector.from_dict(dobj)
except Exception:
detector = None
try:
from dxtbx.serialize import goniometer
gonio = goniometer.from_dict(glist[obj['goniometer']])
except Exception:
gonio = None
try:
scan = Scan.from_dict(slist[obj['scan']])
except Exception:
scan = None
return beam, detector, gonio, scan
# Loop through all the imagesets
imagesets = []
for imageset in obj['imageset']:
ident = imageset['__id__']
if "params" in imageset:
format_kwargs = imageset['params']
else:
format_kwargs = {}
if ident == 'ImageSweep':
beam, detector, gonio, scan = load_models(imageset)
if "template" in imageset:
template = load_path(imageset['template'])
i0, i1 = scan.get_image_range()
iset = ImageSetFactory.make_sweep(
template, range(i0, i1+1), None,
beam, detector, gonio, scan, check_format,
format_kwargs=format_kwargs)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'])
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = pickle.load(infile)
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'])
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = pickle.load(infile)
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'])
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = pickle.load(infile)
elif "master" in imageset:
template = load_path(imageset['master'])
i0, i1 = scan.get_image_range()
indices = imageset['images']
assert min(indices) == i0-1 and max(indices) == i1-1
iset = ImageSetFactory.make_sweep(
template, range(i0, i1+1), None,
beam, detector, gonio, scan, check_format,
format_kwargs=format_kwargs)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'])
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = pickle.load(infile)
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'])
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = pickle.load(infile)
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'])
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = pickle.load(infile)
imagesets.append(iset)
elif ident == 'ImageSet' or ident == "ImageGrid":
filenames = [image['filename'] for image in imageset['images']]
indices = [image['image'] for image in imageset['images'] if 'image' in image]
assert len(indices) == 0 or len(indices) == len(filenames)
iset = ImageSetFactory.make_imageset(
filenames, None, check_format, indices, format_kwargs=format_kwargs)
if ident == "ImageGrid":
grid_size = imageset['grid_size']
iset = ImageGrid.from_imageset(iset, grid_size)
for i, image in enumerate(imageset['images']):
beam, detector, gonio, scan = load_models(image)
iset.set_beam(beam, i)
iset.set_detector(detector, i)
iset.set_goniometer(gonio, i)
iset.set_scan(scan, i)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'])
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = pickle.load(infile)
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'])
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = pickle.load(infile)
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'])
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = pickle.load(infile)
imagesets.append(iset)
else:
raise RuntimeError('expected ImageSet/ImageSweep, got %s' % ident)
# Return the datablock
return DataBlock(imagesets)
def combine(datablock_list, reflections_list, params):
'''
Combine the found spots.
'''
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
from dxtbx.datablock import DataBlock
from dxtbx.imageset import ImageSetFactory
from dials.algorithms.spot_finding import StrongSpotCombiner
from dials.array_family import flex
assert len(datablock_list) == len(reflections_list)
# Get a list of imagesets
imageset_list = []
for db in datablock_list:
iset = db.extract_imagesets()
assert len(iset) == 1
imageset_list.append(iset[0])
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
# The initial models
format_class = imageset_list[0].get_format_class()
beam = imageset_list[0].get_beam()
detector = imageset_list[0].get_detector()
goniometer = imageset_list[0].get_goniometer()
scan = imageset_list[0].get_scan()
template = imageset_list[0].get_template()
# Check all the models
for imageset in imageset_list[1:]:
b = imageset.get_beam()
d = imageset.get_detector()
g = imageset.get_goniometer()
s = imageset.get_scan()
if not imageset.get_format_class() == format_class:
raise RuntimeError('Format classes do not match')
if not imageset.get_template() == template:
raise RuntimeError('Templates do not match')
if not compare_beam(beam, b):
raise RuntimeError('Beam models are too dissimilar')
if not compare_detector(detector, d):
raise RuntimeError('Detector models are too dissimilar')
if not compare_goniometer(goniometer, g):
raise RuntimeError('Goniometer models are too dissimilar')
try:
scan.append(s, scan_tolerance=scan_tolerance)
except Exception:
raise RuntimeError('Scans do not match')
# Get the image range
image_range = scan.get_image_range()
image_range = (image_range[0], image_range[1]+1)
# Create the sweep
imageset = ImageSetFactory.make_sweep(
template, range(*image_range),
format_class,
beam, detector,
goniometer, scan)
# Combine spots
combiner = StrongSpotCombiner()
for index, rlist in enumerate(reflections_list, start=1):
assert rlist['id'].all_eq(0)
logger.info("Combining %d reflections from reflection list %d" % (
len(rlist),
index))
combiner.add(rlist['shoebox'])
shoeboxes = combiner.shoeboxes()
# Calculate the spot centroids and intensities
logger.info('Combined into %d reflections' % len(shoeboxes))
centroid = shoeboxes.centroid_valid()
logger.info('Calculated {0} spot centroids'.format(len(shoeboxes)))
intensity = shoeboxes.summed_intensity()
logger.info('Calculated {0} spot intensities'.format(len(shoeboxes)))
# Construct the reflection table
reflections = flex.reflection_table(
flex.observation(
shoeboxes.panels(),
centroid,
intensity),
shoeboxes)
reflections['id'] = flex.int(len(reflections), 0)
reflections.set_flags(
flex.size_t_range(len(reflections)),
reflections.flags.strong)
# Return the datablock and reflections
return DataBlock([imageset]), reflections
def combine(datablock_list, reflections_list, params):
'''
Combine the found spots.
'''
from dxtbx.datablock import BeamComparison
from dxtbx.datablock import DetectorComparison
from dxtbx.datablock import GoniometerComparison
from dxtbx.datablock import DataBlock
from dxtbx.imageset import ImageSetFactory
from dials.algorithms.spot_finding import StrongSpotCombiner
from logging import info
from dials.array_family import flex
assert len(datablock_list) == len(reflections_list)
# Get a list of imagesets
imageset_list = []
for db in datablock_list:
iset = db.extract_imagesets()
assert len(iset) == 1
imageset_list.append(iset[0])
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation)
scan_tolerance = params.input.tolerance.scan.oscillation
# The initial models
format_class = imageset_list[0].reader().get_format_class()
beam = imageset_list[0].get_beam()
detector = imageset_list[0].get_detector()
goniometer = imageset_list[0].get_goniometer()
scan = imageset_list[0].get_scan()
template = imageset_list[0].get_template()
# Check all the models
for imageset in imageset_list[1:]:
b = imageset.get_beam()
d = imageset.get_detector()
g = imageset.get_goniometer()
s = imageset.get_scan()
if not imageset.reader().get_format_class() == format_class:
raise RuntimeError('Format classes do not match')
if not imageset.get_template() == template:
raise RuntimeError('Templates do not match')
if not compare_beam(beam, b):
raise RuntimeError('Beam models are too dissimilar')
if not compare_detector(detector, d):
raise RuntimeError('Detector models are too dissimilar')
if not compare_goniometer(goniometer, g):
raise RuntimeError('Goniometer models are too dissimilar')
try:
scan.append(s, scan_tolerance=scan_tolerance)
except Exception:
raise RuntimeError('Scans do not match')
# Get the image range
image_range = scan.get_image_range()
image_range = (image_range[0], image_range[1]+1)
# Create the sweep
imageset = ImageSetFactory.make_sweep(
template, range(*image_range),
format_class,
beam, detector,
goniometer, scan)
# Combine spots
combiner = StrongSpotCombiner()
for index, rlist in enumerate(reflections_list, start=1):
assert rlist['id'].all_eq(0)
info("Combining %d reflections from reflection list %d" % (
len(rlist),
index))
combiner.add(rlist['shoebox'])
shoeboxes = combiner.shoeboxes()
# Calculate the spot centroids and intensities
info('Combined into %d reflections' % len(shoeboxes))
centroid = shoeboxes.centroid_valid()
info('Calculated {0} spot centroids'.format(len(shoeboxes)))
intensity = shoeboxes.summed_intensity()
info('Calculated {0} spot intensities'.format(len(shoeboxes)))
# Construct the reflection table
reflections = flex.reflection_table(
flex.observation(
shoeboxes.panels(),
centroid,
intensity),
shoeboxes)
reflections['id'] = flex.int(len(reflections), 0)
reflections.set_flags(
flex.size_t_range(len(reflections)),
reflections.flags.strong)
# Return the datablock and reflections
return DataBlock([imageset]), reflections
