def run(self):
from dxtbx.imageset import ExternalLookup
from dxtbx.format.image import ImageTileBool
from dxtbx.format.image import ImageTileDouble
from dxtbx.format.image import ImageBool
from dxtbx.format.image import ImageDouble
from scitbx.array_family import flex
mask = flex.bool(flex.grid(10, 10), True)
gain = flex.double(flex.grid(10, 10), 1)
pedestal = flex.double(flex.grid(10, 10), 2)
lookup = ExternalLookup()
lookup.mask.data = ImageBool(ImageTileBool(mask))
lookup.gain.data = ImageDouble(ImageTileDouble(gain))
lookup.pedestal.data = ImageDouble(ImageTileDouble(pedestal))
mask2 = lookup.mask.data.tile(0).data()
gain2 = lookup.gain.data.tile(0).data()
pedestal2 = lookup.pedestal.data.tile(0).data()
assert mask2.all_eq(mask)
assert gain2.all_eq(gain)
assert pedestal2.all_eq(pedestal)
print 'OK'
def update_lookup(self, imageset, lookup):
from dxtbx.format.image import ImageBool, ImageDouble
if lookup.size is not None:
d = imageset.get_detector()
assert len(lookup.size) == len(d), "Incompatible size"
for s, p in zip(lookup.size, d):
assert s == p.get_image_size()[::-1], "Incompatible size"
if lookup.mask.filename is not None:
imageset.external_lookup.mask.filename = lookup.mask.filename
imageset.external_lookup.mask.data = ImageBool(
lookup.mask.data)
if lookup.gain.filename is not None:
imageset.external_lookup.gain.filename = lookup.gain.filename
imageset.external_lookup.gain.data = ImageDouble(
lookup.gain.data)
if lookup.dark.filename is not None:
imageset.external_lookup.pedestal.filename = lookup.dark.filename
imageset.external_lookup.pedestal.data = ImageDouble(
lookup.dark.data)
if lookup.dx.filename is not None:
imageset.external_lookup.dx.filename = lookup.dx.filename
imageset.external_lookup.dx.data = ImageDouble(lookup.dx.data)
if lookup.dy.filename is not None:
imageset.external_lookup.dy.filename = lookup.dy.filename
imageset.external_lookup.dy.data = ImageDouble(lookup.dy.data)
return imageset
def basic_imageset_from_dict(d):
''' Construct an ImageSet class from the dictionary.'''
from dxtbx.model import BeamFactory, DetectorFactory
from dxtbx.imageset import ImageSetFactory
from dxtbx.serialize.filename import load_path
# Get the filename list and create the imageset
filenames = map(lambda p: load_path(p), map(str, d['filenames']))
imageset = ImageSetFactory.new(filenames)[0]
# Set some external lookups
if 'mask' in d and d['mask'] is not None and d['mask'] is not "":
with open(d['mask']) as infile:
imageset.external_lookup.mask.filename = d['mask']
imageset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in d and d['gain'] is not None and d['gain'] is not "":
with open(d['gain']) as infile:
imageset.external_lookup.gain.filename = d['gain']
imageset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in d and d['pedestal'] is not None and d['pedestal'] is not "":
with open(d['pedestal']) as infile:
imageset.external_lookup.pedestal.filename = d['pedestal']
imageset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
# Get the existing models as dictionaries
beam_dict = imageset.get_beam(0).to_dict()
detector_dict = imageset.get_detector(0).to_dict()
# Set models
imageset.set_beam(BeamFactory.from_dict(d.get('beam'), beam_dict))
imageset.set_detector(DetectorFactory.from_dict(d.get('detector'), detector_dict))
# Return the imageset
return imageset
def basic_imageset_from_dict(d, directory=None):
""" Construct an ImageSet class from the dictionary."""
# Get the filename list and create the imageset
filenames = map(
lambda p: load_path(p, directory=directory), map(str, d["filenames"])
)
imageset = ImageSetFactory.new(filenames)[0]
# Set some external lookups
if "mask" in d and d["mask"] is not None and d["mask"] is not "":
path = load_path(d["mask"], directory=directory)
with open(path) as infile:
imageset.external_lookup.mask.filename = path
imageset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if "gain" in d and d["gain"] is not None and d["gain"] is not "":
path = load_path(d["gain"], directory=directory)
with open(path) as infile:
imageset.external_lookup.gain.filename = path
imageset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] is not "":
path = load_path(d["pedestal"], directory=directory)
with open(path) as infile:
imageset.external_lookup.pedestal.filename = path
imageset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
# Get the existing models as dictionaries
beam_dict = imageset.get_beam(0).to_dict()
detector_dict = imageset.get_detector(0).to_dict()
# Set models
imageset.set_beam(BeamFactory.from_dict(d.get("beam"), beam_dict))
imageset.set_detector(DetectorFactory.from_dict(d.get("detector"), detector_dict))
# Return the imageset
return imageset
def imagesweep_from_dict(d, check_format=True, directory=None):
"""Construct and image sweep from the dictionary."""
# Get the template (required)
template = load_path(str(d["template"]), directory=directory)
# If the scan isn't set, find all available files
scan_dict = d.get("scan")
if scan_dict is None:
image_range = None
else:
image_range = scan_dict.get("image_range")
# Set the models with the exisiting models as templates
beam = BeamFactory.from_dict(d.get("beam"))
goniometer = GoniometerFactory.from_dict(d.get("goniometer"))
detector = DetectorFactory.from_dict(d.get("detector"))
scan = ScanFactory.from_dict(d.get("scan"))
# Construct the sweep
try:
sweep = ImageSetFactory.from_template(
template,
image_range,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format,
)[0]
except Exception:
indices = range(image_range[0], image_range[1] + 1)
sweep = ImageSetFactory.make_sweep(
template,
indices,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format,
)
# Set some external lookups
if "mask" in d and d["mask"] is not None and d["mask"] is not "":
path = load_path(d["mask"], directory=directory)
with open(path) as infile:
sweep.external_lookup.mask.filename = path
sweep.external_lookup.mask.data = ImageBool(pickle.load(infile))
if "gain" in d and d["gain"] is not None and d["gain"] is not "":
path = load_path(d["gain"], directory=directory)
with open(path) as infile:
sweep.external_lookup.gain.filename = path
sweep.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] is not "":
path = load_path(d["pedestal"], directory=directory)
with open(path) as infile:
sweep.external_lookup.pedestal.filename = path
sweep.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
# Return the sweep
return sweep
def imagesweep_from_dict(d, check_format=True):
'''Construct and image sweep from the dictionary.'''
from dxtbx.imageset import ImageSetFactory
from dxtbx.model import BeamFactory, DetectorFactory, GoniometerFactory, ScanFactory
from dxtbx.serialize.filename import load_path
# Get the template (required)
template = load_path(str(d['template']))
# If the scan isn't set, find all available files
scan_dict = d.get('scan')
if scan_dict is None:
image_range = None
else:
image_range = scan_dict.get('image_range')
# Set the models with the exisiting models as templates
beam = BeamFactory.from_dict(d.get('beam'))
goniometer = GoniometerFactory.from_dict(d.get('goniometer'))
detector = DetectorFactory.from_dict(d.get('detector'))
scan = ScanFactory.from_dict(d.get('scan'))
# Construct the sweep
try:
sweep = ImageSetFactory.from_template(template,
image_range,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format)[0]
except Exception:
indices = range(image_range[0], image_range[1] + 1)
sweep = ImageSetFactory.make_sweep(template,
indices,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format)
# Set some external lookups
if 'mask' in d and d['mask'] is not None and d['mask'] is not "":
with open(d['mask']) as infile:
sweep.external_lookup.mask.filename = d['mask']
sweep.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in d and d['gain'] is not None and d['gain'] is not "":
with open(d['gain']) as infile:
sweep.external_lookup.gain.filename = d['gain']
sweep.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in d and d['pedestal'] is not None and d[
'pedestal'] is not "":
with open(d['pedestal']) as infile:
sweep.external_lookup.pedestal.filename = d['pedestal']
sweep.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile))
# Return the sweep
return sweep
def _imageset_from_imageset_data(self, imageset_data, models):
""" Make an imageset from imageset_data - help with refactor decode. """
assert imageset_data is not None
if "params" in imageset_data:
format_kwargs = imageset_data["params"]
else:
format_kwargs = {}
beam = models["beam"]
detector = models["detector"]
goniometer = models["goniometer"]
scan = models["scan"]
# Load the external lookup data
mask_filename, mask = self._load_pickle_path(imageset_data, "mask")
gain_filename, gain = self._load_pickle_path(imageset_data, "gain")
pedestal_filename, pedestal = self._load_pickle_path(imageset_data, "pedestal")
dx_filename, dx = self._load_pickle_path(imageset_data, "dx")
dy_filename, dy = self._load_pickle_path(imageset_data, "dy")
if imageset_data["__id__"] == "ImageSet":
imageset = self._make_stills(imageset_data, format_kwargs=format_kwargs)
elif imageset_data["__id__"] == "ImageGrid":
imageset = self._make_grid(imageset_data, format_kwargs=format_kwargs)
elif (
imageset_data["__id__"] == "ImageSequence"
or imageset_data["__id__"] == "ImageSweep"
):
imageset = self._make_sequence(
imageset_data,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
format_kwargs=format_kwargs,
)
elif imageset_data["__id__"] == "MemImageSet":
imageset = self._make_mem_imageset(imageset_data)
else:
raise RuntimeError("Unknown imageset type")
if imageset is not None:
# Set the external lookup
if mask is None:
mask = ImageBool()
else:
mask = ImageBool(mask)
if gain is None:
gain = ImageDouble()
else:
gain = ImageDouble(gain)
if pedestal is None:
pedestal = ImageDouble()
else:
pedestal = ImageDouble(pedestal)
if dx is None:
dx = ImageDouble()
else:
dx = ImageDouble(dx)
if dy is None:
dy = ImageDouble()
else:
dy = ImageDouble(dy)
if not imageset.external_lookup.mask.data.empty():
if not mask.empty():
mask = tuple(m.data() for m in mask)
for m1, m2 in zip(mask, imageset.external_lookup.mask.data):
m1 &= m2.data()
imageset.external_lookup.mask.data = ImageBool(mask)
else:
imageset.external_lookup.mask.data = mask
imageset.external_lookup.mask.filename = mask_filename
imageset.external_lookup.gain.data = gain
imageset.external_lookup.gain.filename = gain_filename
imageset.external_lookup.pedestal.data = pedestal
imageset.external_lookup.pedestal.filename = pedestal_filename
imageset.external_lookup.dx.data = dx
imageset.external_lookup.dx.filename = dx_filename
imageset.external_lookup.dy.data = dy
imageset.external_lookup.dy.filename = dy_filename
# Update the imageset models
if isinstance(imageset, ImageSequence):
imageset.set_beam(beam)
imageset.set_detector(detector)
imageset.set_goniometer(goniometer)
imageset.set_scan(scan)
elif isinstance(imageset, (ImageSet, ImageGrid)):
for i in range(len(imageset)):
imageset.set_beam(beam, i)
imageset.set_detector(detector, i)
imageset.set_goniometer(goniometer, i)
imageset.set_scan(scan, i)
imageset.update_detector_px_mm_data()
return imageset
def _extract_experiments(self):
''' Helper function. Extract the experiments. '''
from dxtbx.imageset import ImageSweep, ImageSet, ImageGrid
from dxtbx.serialize.filename import load_path
import six.moves.cPickle as pickle
from dxtbx.format.image import ImageBool, ImageDouble
# Map of imageset/scan pairs
imagesets = {}
# For every experiment, use the given input to create
# a sensible experiment.
el = ExperimentList()
for eobj in self._obj['experiment']:
# Get the models
identifier = eobj.get("identifier", "")
beam = ExperimentListDict.model_or_none(self._blist, eobj, 'beam')
detector = ExperimentListDict.model_or_none(
self._dlist, eobj, 'detector')
goniometer = ExperimentListDict.model_or_none(
self._glist, eobj, 'goniometer')
scan = ExperimentListDict.model_or_none(self._slist, eobj, 'scan')
crystal = ExperimentListDict.model_or_none(self._clist, eobj,
'crystal')
profile = ExperimentListDict.model_or_none(self._plist, eobj,
'profile')
scaling_model = ExperimentListDict.model_or_none(
self._scalelist, eobj, 'scaling_model')
key = (eobj.get('imageset'), eobj.get('scan'))
try:
imageset = imagesets[key]
except Exception:
imageset = ExperimentListDict.model_or_none(
self._ilist, eobj, 'imageset')
# Create the imageset from the input data
if imageset is not None:
if 'params' in imageset:
format_kwargs = imageset['params']
else:
format_kwargs = {}
if 'mask' in imageset and imageset['mask'] is not None:
mask_filename = load_path(imageset['mask'],
directory=self._directory)
if self._check_format and mask_filename is not "":
with open(mask_filename, 'rb') as fh:
mask = pickle.load(fh)
else:
mask = None
else:
mask_filename = None
mask = None
if 'gain' in imageset and imageset['gain'] is not None:
gain_filename = load_path(imageset['gain'],
directory=self._directory)
if self._check_format and gain_filename is not "":
with open(gain_filename, 'rb') as fh:
gain = pickle.load(fh)
else:
gain = None
else:
gain_filename = None
gain = None
if 'pedestal' in imageset and imageset[
'pedestal'] is not None:
pedestal_filename = load_path(
imageset['pedestal'], directory=self._directory)
if self._check_format and pedestal_filename is not "":
with open(pedestal_filename, 'rb') as fh:
pedestal = pickle.load(fh)
else:
pedestal = None
else:
pedestal_filename = None
pedestal = None
if 'dx' in imageset and imageset['dx'] is not None:
dx_filename = load_path(imageset['dx'],
directory=self._directory)
if dx_filename is not "":
with open(dx_filename, 'rb') as fh:
dx = pickle.load(fh)
else:
dx = None
else:
dx_filename = None
dx = None
if 'dy' in imageset and imageset['dy'] is not None:
dy_filename = load_path(imageset['dy'],
directory=self._directory)
if dy_filename is not "":
with open(dy_filename, 'rb') as fh:
dy = pickle.load(fh)
else:
dy = None
else:
dy_filename = None
dy = None
if imageset['__id__'] == 'ImageSet':
imageset = self._make_stills(
imageset, format_kwargs=format_kwargs)
elif imageset['__id__'] == 'ImageGrid':
imageset = self._make_grid(imageset,
format_kwargs=format_kwargs)
elif imageset['__id__'] == 'ImageSweep':
imageset = self._make_sweep(
imageset,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
format_kwargs=format_kwargs)
elif imageset['__id__'] == 'MemImageSet':
imageset = self._make_mem_imageset(imageset)
else:
raise RuntimeError('Unknown imageset type')
# Set the external lookup
if imageset is not None:
if mask_filename is None:
mask_filename = ""
if gain_filename is None:
gain_filename = ""
if pedestal_filename is None:
pedestal_filename = ""
if dx_filename is None:
dx_filename = ""
if dy_filename is None:
dy_filename = ""
if mask is None:
mask = ImageBool()
else:
mask = ImageBool(mask)
if gain is None:
gain = ImageDouble()
else:
gain = ImageDouble(gain)
if pedestal is None:
pedestal = ImageDouble()
else:
pedestal = ImageDouble(pedestal)
if dx is None:
dx = ImageDouble()
else:
dx = ImageDouble(dx)
if dy is None:
dy = ImageDouble()
else:
dy = ImageDouble(dy)
imageset.external_lookup.mask.data = mask
imageset.external_lookup.mask.filename = mask_filename
imageset.external_lookup.gain.data = gain
imageset.external_lookup.gain.filename = gain_filename
imageset.external_lookup.pedestal.data = pedestal
imageset.external_lookup.pedestal.filename = pedestal_filename
imageset.external_lookup.dx.data = dx
imageset.external_lookup.dx.filename = dx_filename
imageset.external_lookup.dy.data = dy
imageset.external_lookup.dy.filename = dy_filename
# Update the imageset models
if isinstance(imageset, ImageSweep):
imageset.set_beam(beam)
imageset.set_detector(detector)
imageset.set_goniometer(goniometer)
imageset.set_scan(scan)
elif isinstance(imageset, ImageSet):
for i in range(len(imageset)):
imageset.set_beam(beam, i)
imageset.set_detector(detector, i)
imageset.set_goniometer(goniometer, i)
imageset.set_scan(scan, i)
elif isinstance(imageset, ImageGrid):
for i in range(len(imageset)):
imageset.set_beam(beam, i)
imageset.set_detector(detector, i)
imageset.set_goniometer(goniometer, i)
imageset.set_scan(scan, i)
else:
pass
if imageset is not None:
imageset.update_detector_px_mm_data()
# Add the imageset to the dict
imagesets[key] = imageset
# Append the experiment
el.append(
Experiment(imageset=imageset,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
profile=profile,
scaling_model=scaling_model,
identifier=identifier))
# Return the experiment list
return el
def run(self):
from dxtbx.imageset import ImageSetData
from dxtbx.format.image import ImageTileBool
from dxtbx.format.image import ImageTileDouble
from dxtbx.format.image import ImageBool
from dxtbx.format.image import ImageDouble
from scitbx.array_family import flex
import os.path
from glob import glob
from dxtbx.format.FormatCBFMiniPilatus import FormatCBFMiniPilatus as FormatClass
from os.path import join
dials_regression = libtbx.env.dist_path('dials_regression')
filenames = sorted(
glob(join(dials_regression, "centroid_test_data", "*.cbf")))
ReaderClass = FormatClass.get_reader()
MaskerClass = FormatClass.get_masker()
reader = ReaderClass(filenames)
masker = MaskerClass(filenames)
handle = ImageSetData(reader, masker)
data = handle.get_data(0).as_int().tile(0).data()
mask = handle.get_mask(0).tile(0).data()
assert handle.has_single_file_reader() == False
path = handle.get_path(0)
assert path == filenames[0]
master_path = handle.get_master_path()
assert master_path == ""
identifier = handle.get_image_identifier(0)
assert identifier == filenames[0]
beam = FormatClass(filenames[0]).get_beam()
detector = FormatClass(filenames[0]).get_detector()
goniometer = FormatClass(filenames[0]).get_goniometer()
scan = FormatClass(filenames[0]).get_scan()
handle.set_beam(beam, 0)
handle.set_detector(detector, 0)
handle.set_goniometer(goniometer, 0)
handle.set_scan(scan, 0)
beam2 = handle.get_beam(0)
detector2 = handle.get_detector(0)
goniometer2 = handle.get_goniometer(0)
scan2 = handle.get_scan(0)
assert beam2 == beam
assert detector2 == detector
assert goniometer2 == goniometer
assert scan2 == scan
mask = flex.bool(flex.grid(10, 10), True)
gain = flex.double(flex.grid(10, 10), 1)
pedestal = flex.double(flex.grid(10, 10), 2)
handle.external_lookup.mask.data = ImageBool(ImageTileBool(mask))
handle.external_lookup.gain.data = ImageDouble(ImageTileDouble(gain))
handle.external_lookup.pedestal.data = ImageDouble(
ImageTileDouble(pedestal))
mask2 = handle.external_lookup.mask.data.tile(0).data()
gain2 = handle.external_lookup.gain.data.tile(0).data()
pedestal2 = handle.external_lookup.pedestal.data.tile(0).data()
assert mask2.all_eq(mask)
assert gain2.all_eq(gain)
assert pedestal2.all_eq(pedestal)
print 'OK'
def _load_datablocks(self, obj, check_format=True, directory=None):
''' Create the datablock from a dictionary. '''
from dxtbx.format.Registry import Registry
from dxtbx.model import BeamFactory, DetectorFactory
from dxtbx.model import GoniometerFactory, ScanFactory
from dxtbx.serialize.filename import load_path
from dxtbx.format.image import ImageBool, ImageDouble
from dxtbx.format.FormatMultiImage import FormatMultiImage
# If we have a list, extract for each dictionary in the list
if isinstance(obj, list):
return [self._load_datablocks(dd, check_format, directory) for dd in obj]
elif not isinstance(obj, dict):
raise InvalidDataBlockError("Unexpected datablock type {} instead of dict".format(type(obj)))
# Make sure the id signature is correct
if not obj.get("__id__") == "DataBlock":
raise InvalidDataBlockError(
"Expected __id__ 'DataBlock', but found {}".format(repr(obj.get("__id__"))))
# Get the list of models
blist = obj.get('beam', [])
dlist = obj.get('detector', [])
glist = obj.get('goniometer', [])
slist = obj.get('scan', [])
def load_models(obj):
try:
beam = BeamFactory.from_dict(blist[obj['beam']])
except Exception:
beam = None
try:
dobj = dlist[obj['detector']]
detector = DetectorFactory.from_dict(dobj)
except Exception:
detector = None
try:
gonio = GoniometerFactory.from_dict(glist[obj['goniometer']])
except Exception:
gonio = None
try:
scan = ScanFactory.from_dict(slist[obj['scan']])
except Exception:
scan = None
return beam, detector, gonio, scan
# Loop through all the imagesets
imagesets = []
for imageset in obj['imageset']:
ident = imageset['__id__']
if "params" in imageset:
format_kwargs = imageset['params']
else:
format_kwargs = {}
if ident == 'ImageSweep':
beam, detector, gonio, scan = load_models(imageset)
if "template" in imageset:
template = load_path(imageset['template'], directory=directory)
i0, i1 = scan.get_image_range()
iset = dxtbx.imageset.ImageSetFactory.make_sweep(
template, range(i0, i1+1), None,
beam, detector, gonio, scan, check_format,
format_kwargs=format_kwargs)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'], directory=directory)
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'], directory=directory)
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'], directory=directory)
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
if 'dx' in imageset and imageset['dx'] is not None:
imageset['dx'] = load_path(imageset['dx'], directory=directory)
iset.external_lookup.dx.filename = imageset['dx']
with open(imageset['dx']) as infile:
iset.external_lookup.dx.data = ImageDouble(pickle.load(infile))
if 'dy' in imageset and imageset['dy'] is not None:
imageset['dy'] = load_path(imageset['dy'], directory=directory)
iset.external_lookup.dy.filename = imageset['dy']
with open(imageset['dy']) as infile:
iset.external_lookup.dy.data = ImageDouble(pickle.load(infile))
iset.update_detector_px_mm_data()
elif "master" in imageset:
template = load_path(imageset['master'], directory=directory)
i0, i1 = scan.get_image_range()
indices = imageset['images']
if check_format == False:
format_class = FormatMultiImage
else:
format_class = None
iset = dxtbx.imageset.ImageSetFactory.make_sweep(
template,
list(range(i0, i1+1)),
format_class = format_class,
beam = beam,
detector = detector,
goniometer = gonio,
scan = scan,
check_format = check_format,
format_kwargs = format_kwargs)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'], directory)
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'], directory)
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'], directory)
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
if 'dx' in imageset and imageset['dx'] is not None:
imageset['dx'] = load_path(imageset['dx'], directory)
iset.external_lookup.dx.filename = imageset['dx']
with open(imageset['dx']) as infile:
iset.external_lookup.dx.data = ImageDouble(pickle.load(infile))
if 'dy' in imageset and imageset['dy'] is not None:
imageset['dy'] = load_path(imageset['dy'], directory)
iset.external_lookup.dy.filename = imageset['dy']
with open(imageset['dy']) as infile:
iset.external_lookup.dy.data = ImageDouble(pickle.load(infile))
iset.update_detector_px_mm_data()
imagesets.append(iset)
elif ident == 'ImageSet' or ident == "ImageGrid":
filenames = [image['filename'] for image in imageset['images']]
indices = [image['image'] for image in imageset['images'] if 'image' in image]
assert len(indices) == 0 or len(indices) == len(filenames)
iset = dxtbx.imageset.ImageSetFactory.make_imageset(
filenames, None, check_format, indices, format_kwargs=format_kwargs)
if ident == "ImageGrid":
grid_size = imageset['grid_size']
iset = dxtbx.imageset.ImageGrid.from_imageset(iset, grid_size)
for i, image in enumerate(imageset['images']):
beam, detector, gonio, scan = load_models(image)
iset.set_beam(beam, i)
iset.set_detector(detector, i)
iset.set_goniometer(gonio, i)
iset.set_scan(scan, i)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'], directory)
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'], directory)
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'], directory)
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
if 'dx' in imageset and imageset['dx'] is not None:
imageset['dx'] = load_path(imageset['dx'], directory)
iset.external_lookup.dx.filename = imageset['dx']
with open(imageset['dx']) as infile:
iset.external_lookup.dx.data = ImageDouble(pickle.load(infile))
if 'dy' in imageset and imageset['dy'] is not None:
imageset['dy'] = load_path(imageset['dy'], directory)
iset.external_lookup.dy.filename = imageset['dy']
with open(imageset['dy']) as infile:
iset.external_lookup.dy.data = ImageDouble(pickle.load(infile))
iset.update_detector_px_mm_data()
imagesets.append(iset)
else:
raise RuntimeError('expected ImageSet/ImageSweep, got %s' % ident)
# Return the datablock
return DataBlock(imagesets)
def decode(self):
""" Decode the dictionary into a list of experiments. """
# Extract all the experiments
# Map of imageset/scan pairs
imagesets = {}
# For every experiment, use the given input to create
# a sensible experiment.
el = ExperimentList()
for eobj in self._obj["experiment"]:
# Get the models
identifier = eobj.get("identifier", "")
beam = self._lookup_model("beam", eobj)
detector = self._lookup_model("detector", eobj)
goniometer = self._lookup_model("goniometer", eobj)
scan = self._lookup_model("scan", eobj)
crystal = self._lookup_model("crystal", eobj)
profile = self._lookup_model("profile", eobj)
scaling_model = self._lookup_model("scaling_model", eobj)
key = (eobj.get("imageset"), eobj.get("scan"))
imageset = None
try:
imageset = imagesets[key] # type: ImageSet
except KeyError:
# This imageset hasn't been loaded yet - create it
imageset_data = self._lookup_model("imageset", eobj)
# Create the imageset from the input data
if imageset_data is not None:
if "params" in imageset_data:
format_kwargs = imageset_data["params"]
else:
format_kwargs = {}
# Load the external lookup data
mask_filename, mask = self._load_pickle_path(
imageset_data, "mask")
gain_filename, gain = self._load_pickle_path(
imageset_data, "gain")
pedestal_filename, pedestal = self._load_pickle_path(
imageset_data, "pedestal")
dx_filename, dx = self._load_pickle_path(
imageset_data, "dx")
dy_filename, dy = self._load_pickle_path(
imageset_data, "dy")
if imageset_data["__id__"] == "ImageSet":
imageset = self._make_stills(
imageset_data, format_kwargs=format_kwargs)
elif imageset_data["__id__"] == "ImageGrid":
imageset = self._make_grid(imageset_data,
format_kwargs=format_kwargs)
elif imageset_data["__id__"] == "ImageSweep":
imageset = self._make_sweep(
imageset_data,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
format_kwargs=format_kwargs,
)
elif imageset_data["__id__"] == "MemImageSet":
imageset = self._make_mem_imageset(imageset_data)
else:
raise RuntimeError("Unknown imageset type")
if imageset is not None:
# Set the external lookup
if mask is None:
mask = ImageBool()
else:
mask = ImageBool(mask)
if gain is None:
gain = ImageDouble()
else:
gain = ImageDouble(gain)
if pedestal is None:
pedestal = ImageDouble()
else:
pedestal = ImageDouble(pedestal)
if dx is None:
dx = ImageDouble()
else:
dx = ImageDouble(dx)
if dy is None:
dy = ImageDouble()
else:
dy = ImageDouble(dy)
imageset.external_lookup.mask.data = mask
imageset.external_lookup.mask.filename = mask_filename
imageset.external_lookup.gain.data = gain
imageset.external_lookup.gain.filename = gain_filename
imageset.external_lookup.pedestal.data = pedestal
imageset.external_lookup.pedestal.filename = pedestal_filename
imageset.external_lookup.dx.data = dx
imageset.external_lookup.dx.filename = dx_filename
imageset.external_lookup.dy.data = dy
imageset.external_lookup.dy.filename = dy_filename
# Update the imageset models
if isinstance(imageset, ImageSweep):
imageset.set_beam(beam)
imageset.set_detector(detector)
imageset.set_goniometer(goniometer)
imageset.set_scan(scan)
elif isinstance(imageset, (ImageSet, ImageGrid)):
for i in range(len(imageset)):
imageset.set_beam(beam, i)
imageset.set_detector(detector, i)
imageset.set_goniometer(goniometer, i)
imageset.set_scan(scan, i)
imageset.update_detector_px_mm_data()
# Add the imageset to the dict - even if empty - as this will
# prevent a duplicated attempt at reconstruction
imagesets[key] = imageset
# Append the experiment
el.append(
Experiment(
imageset=imageset,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
profile=profile,
scaling_model=scaling_model,
identifier=identifier,
))
# Return the experiment list
return el
def datablocks_from_dict(obj, check_format=True, directory=None):
"""Get the datablocks from the dictionary."""
# If we have a list, extract for each dictionary in the list
if isinstance(obj, list):
return [
datablocks_from_dict(dd, check_format, directory) for dd in obj
]
elif not isinstance(obj, dict):
raise InvalidDataBlockError(
"Unexpected datablock type {} instead of dict".format(type(obj)))
# Make sure the id signature is correct
if not obj.get("__id__") == "DataBlock":
raise InvalidDataBlockError(
"Expected __id__ 'DataBlock', but found {}".format(
repr(obj.get("__id__"))))
# Get the list of models
blist = obj.get("beam", [])
dlist = obj.get("detector", [])
glist = obj.get("goniometer", [])
slist = obj.get("scan", [])
def load_models(obj):
try:
beam = dxtbx.model.BeamFactory.from_dict(blist[obj["beam"]])
except Exception:
beam = None
try:
dobj = dlist[obj["detector"]]
detector = dxtbx.model.DetectorFactory.from_dict(dobj)
except Exception:
detector = None
try:
gonio = dxtbx.model.GoniometerFactory.from_dict(
glist[obj["goniometer"]])
except Exception:
gonio = None
try:
scan = dxtbx.model.ScanFactory.from_dict(slist[obj["scan"]])
except Exception:
scan = None
return beam, detector, gonio, scan
# Loop through all the imagesets
imagesets = []
for imageset in obj["imageset"]:
ident = imageset["__id__"]
if "params" in imageset:
format_kwargs = imageset["params"]
else:
format_kwargs = {}
if ident == "ImageSequence" or ident == "ImageSweep":
beam, detector, gonio, scan = load_models(imageset)
if "template" in imageset:
template = resolve_path(imageset["template"],
directory=directory)
i0, i1 = scan.get_image_range()
iset = dxtbx.imageset.ImageSetFactory.make_sequence(
template,
list(range(i0, i1 + 1)),
None,
beam,
detector,
gonio,
scan,
check_format,
format_kwargs=format_kwargs,
)
if "mask" in imageset and imageset["mask"] is not None:
imageset["mask"] = resolve_path(imageset["mask"],
directory=directory)
iset.external_lookup.mask.filename = imageset["mask"]
if check_format:
with open(imageset["mask"], "rb") as infile:
iset.external_lookup.mask.data = ImageBool(
pickle.load(infile, encoding="bytes"))
if "gain" in imageset and imageset["gain"] is not None:
imageset["gain"] = resolve_path(imageset["gain"],
directory=directory)
iset.external_lookup.gain.filename = imageset["gain"]
if check_format:
with open(imageset["gain"], "rb") as infile:
iset.external_lookup.gain.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "pedestal" in imageset and imageset["pedestal"] is not None:
imageset["pedestal"] = resolve_path(imageset["pedestal"],
directory=directory)
iset.external_lookup.pedestal.filename = imageset[
"pedestal"]
if check_format:
with open(imageset["pedestal"], "rb") as infile:
iset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dx" in imageset and imageset["dx"] is not None:
imageset["dx"] = resolve_path(imageset["dx"],
directory=directory)
iset.external_lookup.dx.filename = imageset["dx"]
with open(imageset["dx"], "rb") as infile:
iset.external_lookup.dx.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dy" in imageset and imageset["dy"] is not None:
imageset["dy"] = resolve_path(imageset["dy"],
directory=directory)
iset.external_lookup.dy.filename = imageset["dy"]
with open(imageset["dy"], "rb") as infile:
iset.external_lookup.dy.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
iset.update_detector_px_mm_data()
elif "master" in imageset:
template = resolve_path(imageset["master"],
directory=directory)
i0, i1 = scan.get_image_range()
if not check_format:
format_class = FormatMultiImage
else:
format_class = None
iset = dxtbx.imageset.ImageSetFactory.make_sequence(
template,
list(range(i0, i1 + 1)),
format_class=format_class,
beam=beam,
detector=detector,
goniometer=gonio,
scan=scan,
check_format=check_format,
format_kwargs=format_kwargs,
)
if "mask" in imageset and imageset["mask"] is not None:
imageset["mask"] = resolve_path(imageset["mask"],
directory)
iset.external_lookup.mask.filename = imageset["mask"]
if check_format:
with open(imageset["mask"], "rb") as infile:
iset.external_lookup.mask.data = ImageBool(
pickle.load(infile, encoding="bytes"))
if "gain" in imageset and imageset["gain"] is not None:
imageset["gain"] = resolve_path(imageset["gain"],
directory)
iset.external_lookup.gain.filename = imageset["gain"]
if check_format:
with open(imageset["gain"], "rb") as infile:
iset.external_lookup.gain.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "pedestal" in imageset and imageset["pedestal"] is not None:
imageset["pedestal"] = resolve_path(
imageset["pedestal"], directory)
iset.external_lookup.pedestal.filename = imageset[
"pedestal"]
if check_format:
with open(imageset["pedestal"], "rb") as infile:
iset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dx" in imageset and imageset["dx"] is not None:
imageset["dx"] = resolve_path(imageset["dx"], directory)
iset.external_lookup.dx.filename = imageset["dx"]
with open(imageset["dx"], "rb") as infile:
iset.external_lookup.dx.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dy" in imageset and imageset["dy"] is not None:
imageset["dy"] = resolve_path(imageset["dy"], directory)
iset.external_lookup.dy.filename = imageset["dy"]
with open(imageset["dy"], "rb") as infile:
iset.external_lookup.dy.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
iset.update_detector_px_mm_data()
imagesets.append(iset)
elif ident == "ImageSet" or ident == "ImageGrid":
filenames = [image["filename"] for image in imageset["images"]]
indices = [
image["image"] for image in imageset["images"]
if "image" in image
]
assert len(indices) == 0 or len(indices) == len(filenames)
iset = dxtbx.imageset.ImageSetFactory.make_imageset(
filenames,
None,
check_format,
indices,
format_kwargs=format_kwargs)
if ident == "ImageGrid":
grid_size = imageset["grid_size"]
iset = dxtbx.imageset.ImageGrid.from_imageset(iset, grid_size)
for i, image in enumerate(imageset["images"]):
beam, detector, gonio, scan = load_models(image)
iset.set_beam(beam, i)
iset.set_detector(detector, i)
iset.set_goniometer(gonio, i)
iset.set_scan(scan, i)
if "mask" in imageset and imageset["mask"] is not None:
imageset["mask"] = resolve_path(imageset["mask"], directory)
iset.external_lookup.mask.filename = imageset["mask"]
if check_format:
with open(imageset["mask"], "rb") as infile:
iset.external_lookup.mask.data = ImageBool(
pickle.load(infile, encoding="bytes"))
if "gain" in imageset and imageset["gain"] is not None:
imageset["gain"] = resolve_path(imageset["gain"], directory)
iset.external_lookup.gain.filename = imageset["gain"]
if check_format:
with open(imageset["gain"], "rb") as infile:
iset.external_lookup.gain.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "pedestal" in imageset and imageset["pedestal"] is not None:
imageset["pedestal"] = resolve_path(imageset["pedestal"],
directory)
iset.external_lookup.pedestal.filename = imageset["pedestal"]
if check_format:
with open(imageset["pedestal"], "rb") as infile:
iset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dx" in imageset and imageset["dx"] is not None:
imageset["dx"] = resolve_path(imageset["dx"], directory)
iset.external_lookup.dx.filename = imageset["dx"]
with open(imageset["dx"], "rb") as infile:
iset.external_lookup.dx.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dy" in imageset and imageset["dy"] is not None:
imageset["dy"] = resolve_path(imageset["dy"], directory)
iset.external_lookup.dy.filename = imageset["dy"]
with open(imageset["dy"], "rb") as infile:
iset.external_lookup.dy.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
iset.update_detector_px_mm_data()
imagesets.append(iset)
else:
raise RuntimeError("expected ImageSet/ImageSequence, got %s" %
ident)
return DataBlock(imagesets)