def __call__(self, experiments):
"""
Do the spot finding.
:param experiments: The experiments to process
:return: The observed spots
"""
from dials.array_family import flex
import six.moves.cPickle as pickle
from dxtbx.format.image import ImageBool
# Loop through all the imagesets and find the strong spots
reflections = flex.reflection_table()
for i, experiment in enumerate(experiments):
imageset = experiment.imageset
# Find the strong spots in the sweep
logger.info("-" * 80)
logger.info("Finding strong spots in imageset %d" % i)
logger.info("-" * 80)
logger.info("")
table, hot_mask = self._find_spots_in_imageset(imageset)
table["id"] = flex.int(table.nrows(), i)
reflections.extend(table)
# Write a hot pixel mask
if self.write_hot_mask:
if not imageset.external_lookup.mask.data.empty():
for m1, m2 in zip(hot_mask,
imageset.external_lookup.mask.data):
m1 &= m2.data()
imageset.external_lookup.mask.data = ImageBool(hot_mask)
else:
imageset.external_lookup.mask.data = ImageBool(hot_mask)
imageset.external_lookup.mask.filename = "%s_%d.pickle" % (
self.hot_mask_prefix,
i,
)
# Write the hot mask
with open(imageset.external_lookup.mask.filename,
"wb") as outfile:
pickle.dump(hot_mask,
outfile,
protocol=pickle.HIGHEST_PROTOCOL)
# Set the strong spot flag
reflections.set_flags(flex.size_t_range(len(reflections)),
reflections.flags.strong)
# Check for overloads
reflections.is_overloaded(experiments)
# Return the reflections
return reflections
def _add_static_mask_to_iset(format_instance: Format, iset: ImageSet) -> None:
"""Combine any static mask from a Format with the ImageSet's mask"""
if format_instance is not None:
static_mask = format_instance.get_static_mask()
if static_mask is not None:
if not iset.external_lookup.mask.data.empty():
for m1, m2 in zip(static_mask, iset.external_lookup.mask.data):
m1 &= m2.data()
iset.external_lookup.mask.data = ImageBool(static_mask)
else:
iset.external_lookup.mask.data = ImageBool(static_mask)
def update_lookup(self, imageset, lookup):
from dxtbx.format.image import ImageBool, ImageDouble
if lookup.size is not None:
d = imageset.get_detector()
assert len(lookup.size) == len(d), "Incompatible size"
for s, p in zip(lookup.size, d):
assert s == p.get_image_size()[::-1], "Incompatible size"
if lookup.mask.filename is not None:
imageset.external_lookup.mask.filename = lookup.mask.filename
imageset.external_lookup.mask.data = ImageBool(
lookup.mask.data)
if lookup.gain.filename is not None:
imageset.external_lookup.gain.filename = lookup.gain.filename
imageset.external_lookup.gain.data = ImageDouble(
lookup.gain.data)
if lookup.dark.filename is not None:
imageset.external_lookup.pedestal.filename = lookup.dark.filename
imageset.external_lookup.pedestal.data = ImageDouble(
lookup.dark.data)
if lookup.dx.filename is not None:
imageset.external_lookup.dx.filename = lookup.dx.filename
imageset.external_lookup.dx.data = ImageDouble(lookup.dx.data)
if lookup.dy.filename is not None:
imageset.external_lookup.dy.filename = lookup.dy.filename
imageset.external_lookup.dy.data = ImageDouble(lookup.dy.data)
return imageset
def imagesweep_from_dict(d, check_format=True, directory=None):
"""Construct and image sweep from the dictionary."""
# Get the template (required)
template = load_path(str(d["template"]), directory=directory)
# If the scan isn't set, find all available files
scan_dict = d.get("scan")
if scan_dict is None:
image_range = None
else:
image_range = scan_dict.get("image_range")
# Set the models with the exisiting models as templates
beam = BeamFactory.from_dict(d.get("beam"))
goniometer = GoniometerFactory.from_dict(d.get("goniometer"))
detector = DetectorFactory.from_dict(d.get("detector"))
scan = ScanFactory.from_dict(d.get("scan"))
# Construct the sweep
try:
sweep = ImageSetFactory.from_template(
template,
image_range,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format,
)[0]
except Exception:
indices = range(image_range[0], image_range[1] + 1)
sweep = ImageSetFactory.make_sweep(
template,
indices,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format,
)
# Set some external lookups
if "mask" in d and d["mask"] is not None and d["mask"] is not "":
path = load_path(d["mask"], directory=directory)
with open(path) as infile:
sweep.external_lookup.mask.filename = path
sweep.external_lookup.mask.data = ImageBool(pickle.load(infile))
if "gain" in d and d["gain"] is not None and d["gain"] is not "":
path = load_path(d["gain"], directory=directory)
with open(path) as infile:
sweep.external_lookup.gain.filename = path
sweep.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] is not "":
path = load_path(d["pedestal"], directory=directory)
with open(path) as infile:
sweep.external_lookup.pedestal.filename = path
sweep.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
# Return the sweep
return sweep
def basic_imageset_from_dict(d):
''' Construct an ImageSet class from the dictionary.'''
from dxtbx.model import BeamFactory, DetectorFactory
from dxtbx.imageset import ImageSetFactory
from dxtbx.serialize.filename import load_path
# Get the filename list and create the imageset
filenames = map(lambda p: load_path(p), map(str, d['filenames']))
imageset = ImageSetFactory.new(filenames)[0]
# Set some external lookups
if 'mask' in d and d['mask'] is not None and d['mask'] is not "":
with open(d['mask']) as infile:
imageset.external_lookup.mask.filename = d['mask']
imageset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in d and d['gain'] is not None and d['gain'] is not "":
with open(d['gain']) as infile:
imageset.external_lookup.gain.filename = d['gain']
imageset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in d and d['pedestal'] is not None and d['pedestal'] is not "":
with open(d['pedestal']) as infile:
imageset.external_lookup.pedestal.filename = d['pedestal']
imageset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
# Get the existing models as dictionaries
beam_dict = imageset.get_beam(0).to_dict()
detector_dict = imageset.get_detector(0).to_dict()
# Set models
imageset.set_beam(BeamFactory.from_dict(d.get('beam'), beam_dict))
imageset.set_detector(DetectorFactory.from_dict(d.get('detector'), detector_dict))
# Return the imageset
return imageset
def basic_imageset_from_dict(d, directory=None):
""" Construct an ImageSet class from the dictionary."""
# Get the filename list and create the imageset
filenames = map(
lambda p: load_path(p, directory=directory), map(str, d["filenames"])
)
imageset = ImageSetFactory.new(filenames)[0]
# Set some external lookups
if "mask" in d and d["mask"] is not None and d["mask"] is not "":
path = load_path(d["mask"], directory=directory)
with open(path) as infile:
imageset.external_lookup.mask.filename = path
imageset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if "gain" in d and d["gain"] is not None and d["gain"] is not "":
path = load_path(d["gain"], directory=directory)
with open(path) as infile:
imageset.external_lookup.gain.filename = path
imageset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] is not "":
path = load_path(d["pedestal"], directory=directory)
with open(path) as infile:
imageset.external_lookup.pedestal.filename = path
imageset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
# Get the existing models as dictionaries
beam_dict = imageset.get_beam(0).to_dict()
detector_dict = imageset.get_detector(0).to_dict()
# Set models
imageset.set_beam(BeamFactory.from_dict(d.get("beam"), beam_dict))
imageset.set_detector(DetectorFactory.from_dict(d.get("detector"), detector_dict))
# Return the imageset
return imageset
def run(self):
from dxtbx.imageset import ExternalLookup
from dxtbx.format.image import ImageTileBool
from dxtbx.format.image import ImageTileDouble
from dxtbx.format.image import ImageBool
from dxtbx.format.image import ImageDouble
from scitbx.array_family import flex
mask = flex.bool(flex.grid(10, 10), True)
gain = flex.double(flex.grid(10, 10), 1)
pedestal = flex.double(flex.grid(10, 10), 2)
lookup = ExternalLookup()
lookup.mask.data = ImageBool(ImageTileBool(mask))
lookup.gain.data = ImageDouble(ImageTileDouble(gain))
lookup.pedestal.data = ImageDouble(ImageTileDouble(pedestal))
mask2 = lookup.mask.data.tile(0).data()
gain2 = lookup.gain.data.tile(0).data()
pedestal2 = lookup.pedestal.data.tile(0).data()
assert mask2.all_eq(mask)
assert gain2.all_eq(gain)
assert pedestal2.all_eq(pedestal)
print 'OK'
def imagesweep_from_dict(d, check_format=True):
'''Construct and image sweep from the dictionary.'''
from dxtbx.imageset import ImageSetFactory
from dxtbx.model import BeamFactory, DetectorFactory, GoniometerFactory, ScanFactory
from dxtbx.serialize.filename import load_path
# Get the template (required)
template = load_path(str(d['template']))
# If the scan isn't set, find all available files
scan_dict = d.get('scan')
if scan_dict is None:
image_range = None
else:
image_range = scan_dict.get('image_range')
# Set the models with the exisiting models as templates
beam = BeamFactory.from_dict(d.get('beam'))
goniometer = GoniometerFactory.from_dict(d.get('goniometer'))
detector = DetectorFactory.from_dict(d.get('detector'))
scan = ScanFactory.from_dict(d.get('scan'))
# Construct the sweep
try:
sweep = ImageSetFactory.from_template(template,
image_range,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format)[0]
except Exception:
indices = range(image_range[0], image_range[1] + 1)
sweep = ImageSetFactory.make_sweep(template,
indices,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format)
# Set some external lookups
if 'mask' in d and d['mask'] is not None and d['mask'] is not "":
with open(d['mask']) as infile:
sweep.external_lookup.mask.filename = d['mask']
sweep.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in d and d['gain'] is not None and d['gain'] is not "":
with open(d['gain']) as infile:
sweep.external_lookup.gain.filename = d['gain']
sweep.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in d and d['pedestal'] is not None and d[
'pedestal'] is not "":
with open(d['pedestal']) as infile:
sweep.external_lookup.pedestal.filename = d['pedestal']
sweep.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile))
# Return the sweep
return sweep
def run(self):
''' Run the script. '''
from dials.util.masking import MaskGenerator
from dials.util.options import flatten_datablocks
from libtbx.utils import Sorry
import six.moves.cPickle as pickle
from dials.util import log
from dxtbx.format.image import ImageBool
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
# Configure logging
log.config()
# Check number of args
if len(datablocks) == 0:
self.parser.print_help()
return
if len(datablocks) != 1:
raise Sorry('exactly 1 datablock must be specified')
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
if len(imagesets) != 1:
raise Sorry('datablock must contain exactly 1 imageset')
imageset = imagesets[0]
# Generate the mask
generator = MaskGenerator(params)
mask = generator.generate(imageset)
# Save the mask to file
print("Writing mask to %s" % params.output.mask)
with open(params.output.mask, "wb") as fh:
pickle.dump(mask, fh)
# Save the datablock
if params.output.datablock is not None:
imageset.external_lookup.mask.data = ImageBool(mask)
imageset.external_lookup.mask.filename = params.output.mask
from dxtbx.datablock import DataBlockDumper
print('Saving datablocks to {0}'.format(params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
def run(self, args=None):
"""Run the script."""
from dials.util import Sorry
from dials.util.options import flatten_experiments
# Parse the command line arguments
params, options = self.parser.parse_args(args, show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
# Check that an experiment list and at least one mask file have been provided
if not (experiments and params.input.mask):
self.parser.print_help()
return
# Check number of experiments
n_expts = len(experiments)
n_masks = len(params.input.mask)
if n_expts != n_masks:
raise Sorry(
"The number of masks provided must match the number of imagesets "
"(sequences).\n"
"You have provided an experiment list containing {} imageset(s).\n"
"You have provided {} mask file(s).".format(n_expts, n_masks)
)
# Get the imageset
imagesets = experiments.imagesets()
for i, imageset in enumerate(imagesets):
# Set the lookup
with open(params.input.mask[i], "rb") as f:
if six.PY3:
mask = pickle.load(f, encoding="bytes")
else:
mask = pickle.load(f)
imageset.external_lookup.mask.filename = params.input.mask[i]
imageset.external_lookup.mask.data = ImageBool(mask)
# Dump the experiments
print("Writing experiments to %s" % params.output.experiments)
experiments.as_file(filename=params.output.experiments)
def run(self):
""" Run the script. """
from dials.util.options import flatten_experiments
from dxtbx.model.experiment_list import ExperimentListDumper
from dials.util import Sorry
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
# Check that an experiment list and at least one mask file have been provided
if not (experiments and params.input.mask):
self.parser.print_help()
return
# Check number of experiments
n_expts = len(experiments)
n_masks = len(params.input.mask)
if n_expts != n_masks:
raise Sorry(
"The number of masks provided must match the number of imagesets "
"(sweeps).\n"
"You have provided an experiment list containing {} imageset(s).\n"
"You have provided {} mask file(s).".format(n_expts, n_masks))
# Get the imageset
imagesets = experiments.imagesets()
for i, imageset in enumerate(imagesets):
# Set the lookup
with open(params.input.mask[i]) as f:
mask = pickle.load(f)
imageset.external_lookup.mask.filename = params.input.mask[i]
imageset.external_lookup.mask.data = ImageBool(mask)
# Dump the experiments
print("Writing experiments to %s" % params.output.experiments)
dump = ExperimentListDumper(experiments)
dump.as_json(filename=params.output.experiments)
def test_combine_with_user_static_mask(dials_data, tmpdir):
master_h5 = (dials_data("image_examples").join(
"SACLA-MPCCD-Phase3-21528-5images.h5").strpath)
assert FormatHDF5SaclaMPCCD.understand(master_h5)
expts_from_filename = ExperimentListFactory.from_filenames([master_h5])
for i, expt in enumerate(expts_from_filename):
mask = []
for panel in expt.detector:
m = flex.bool(flex.grid(reversed(panel.get_image_size())), True)
mask_untrusted_resolution_range(m, expt.beam, panel, 0, 2.23)
mask.append(m)
mask = tuple(mask)
# exact number of masked pixels varies with wavelength between experiments
assert [_.count(False) for _ in mask
] == pytest.approx([50643, 0, 0, 48003, 48356, 0, 0, 52191],
abs=1e3)
mask_file = tmpdir.join("pixel_%i.mask" % i)
with mask_file.open("wb") as f:
pickle.dump(mask, f)
expt.imageset.external_lookup.mask.filename = mask_file.strpath
expt.imageset.external_lookup.mask.data = ImageBool(mask)
expts_from_dict = ExperimentListFactory.from_dict(
expts_from_filename.to_dict())
imageset = expts_from_dict[0].imageset
mask = imageset.get_mask(0)
assert len(mask) == 8
assert [_.count(False) for _ in mask] == [
65332,
24296,
24296,
63335,
63660,
24296,
24296,
66691,
]
imageset.reader().nullify_format_instance()
def decode(self):
""" Decode the dictionary into a list of experiments. """
# Extract all the experiments
# Map of imageset/scan pairs
imagesets = {}
# For every experiment, use the given input to create
# a sensible experiment.
el = ExperimentList()
for eobj in self._obj["experiment"]:
# Get the models
identifier = eobj.get("identifier", "")
beam = self._lookup_model("beam", eobj)
detector = self._lookup_model("detector", eobj)
goniometer = self._lookup_model("goniometer", eobj)
scan = self._lookup_model("scan", eobj)
crystal = self._lookup_model("crystal", eobj)
profile = self._lookup_model("profile", eobj)
scaling_model = self._lookup_model("scaling_model", eobj)
key = (eobj.get("imageset"), eobj.get("scan"))
imageset = None
try:
imageset = imagesets[key] # type: ImageSet
except KeyError:
# This imageset hasn't been loaded yet - create it
imageset_data = self._lookup_model("imageset", eobj)
# Create the imageset from the input data
if imageset_data is not None:
if "params" in imageset_data:
format_kwargs = imageset_data["params"]
else:
format_kwargs = {}
# Load the external lookup data
mask_filename, mask = self._load_pickle_path(
imageset_data, "mask")
gain_filename, gain = self._load_pickle_path(
imageset_data, "gain")
pedestal_filename, pedestal = self._load_pickle_path(
imageset_data, "pedestal")
dx_filename, dx = self._load_pickle_path(
imageset_data, "dx")
dy_filename, dy = self._load_pickle_path(
imageset_data, "dy")
if imageset_data["__id__"] == "ImageSet":
imageset = self._make_stills(
imageset_data, format_kwargs=format_kwargs)
elif imageset_data["__id__"] == "ImageGrid":
imageset = self._make_grid(imageset_data,
format_kwargs=format_kwargs)
elif imageset_data["__id__"] == "ImageSweep":
imageset = self._make_sweep(
imageset_data,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
format_kwargs=format_kwargs,
)
elif imageset_data["__id__"] == "MemImageSet":
imageset = self._make_mem_imageset(imageset_data)
else:
raise RuntimeError("Unknown imageset type")
if imageset is not None:
# Set the external lookup
if mask is None:
mask = ImageBool()
else:
mask = ImageBool(mask)
if gain is None:
gain = ImageDouble()
else:
gain = ImageDouble(gain)
if pedestal is None:
pedestal = ImageDouble()
else:
pedestal = ImageDouble(pedestal)
if dx is None:
dx = ImageDouble()
else:
dx = ImageDouble(dx)
if dy is None:
dy = ImageDouble()
else:
dy = ImageDouble(dy)
imageset.external_lookup.mask.data = mask
imageset.external_lookup.mask.filename = mask_filename
imageset.external_lookup.gain.data = gain
imageset.external_lookup.gain.filename = gain_filename
imageset.external_lookup.pedestal.data = pedestal
imageset.external_lookup.pedestal.filename = pedestal_filename
imageset.external_lookup.dx.data = dx
imageset.external_lookup.dx.filename = dx_filename
imageset.external_lookup.dy.data = dy
imageset.external_lookup.dy.filename = dy_filename
# Update the imageset models
if isinstance(imageset, ImageSweep):
imageset.set_beam(beam)
imageset.set_detector(detector)
imageset.set_goniometer(goniometer)
imageset.set_scan(scan)
elif isinstance(imageset, (ImageSet, ImageGrid)):
for i in range(len(imageset)):
imageset.set_beam(beam, i)
imageset.set_detector(detector, i)
imageset.set_goniometer(goniometer, i)
imageset.set_scan(scan, i)
imageset.update_detector_px_mm_data()
# Add the imageset to the dict - even if empty - as this will
# prevent a duplicated attempt at reconstruction
imagesets[key] = imageset
# Append the experiment
el.append(
Experiment(
imageset=imageset,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
profile=profile,
scaling_model=scaling_model,
identifier=identifier,
))
# Return the experiment list
return el
def find_spots(self, experiments: ExperimentList) -> flex.reflection_table:
"""
Do spotfinding for a set of experiments.
Args:
experiments: The experiment list to process
Returns:
A new reflection table of found reflections
"""
# Loop through all the experiments and get the unique imagesets
imagesets = []
for experiment in experiments:
if experiment.imageset not in imagesets:
imagesets.append(experiment.imageset)
# Loop through all the imagesets and find the strong spots
reflections = flex.reflection_table()
for j, imageset in enumerate(imagesets):
# Find the strong spots in the sequence
logger.info(
"-" * 80 + "\nFinding strong spots in imageset %d\n" +
"-" * 80, j)
table, hot_mask = self._find_spots_in_imageset(imageset)
# Fix up the experiment ID's now
table["id"] = flex.int(table.nrows(), -1)
for i, experiment in enumerate(experiments):
if experiment.imageset is not imageset:
continue
if not self.is_stills and experiment.scan:
z0, z1 = experiment.scan.get_array_range()
z = table["xyzobs.px.value"].parts()[2]
table["id"].set_selected((z > z0) & (z < z1), i)
if experiment.identifier:
table.experiment_identifiers(
)[i] = experiment.identifier
else:
table["id"] = flex.int(table.nrows(), j)
if experiment.identifier:
table.experiment_identifiers(
)[j] = experiment.identifier
missed = table["id"] == -1
assert missed.count(
True) == 0, "Failed to remap {} experiment IDs".format(
missed.count(True))
reflections.extend(table)
# Write a hot pixel mask
if self.write_hot_mask:
if not imageset.external_lookup.mask.data.empty():
for m1, m2 in zip(hot_mask,
imageset.external_lookup.mask.data):
m1 &= m2.data()
imageset.external_lookup.mask.data = ImageBool(hot_mask)
else:
imageset.external_lookup.mask.data = ImageBool(hot_mask)
imageset.external_lookup.mask.filename = "%s_%d.pickle" % (
self.hot_mask_prefix,
i,
)
# Write the hot mask
with open(imageset.external_lookup.mask.filename,
"wb") as outfile:
pickle.dump(hot_mask,
outfile,
protocol=pickle.HIGHEST_PROTOCOL)
# Set the strong spot flag
reflections.set_flags(flex.size_t_range(len(reflections)),
reflections.flags.strong)
# Check for overloads
reflections.is_overloaded(experiments)
# Return the reflections
return reflections
def _extract_experiments(self):
''' Helper function. Extract the experiments. '''
from dxtbx.imageset import ImageSweep, ImageSet, ImageGrid
from dxtbx.serialize.filename import load_path
import six.moves.cPickle as pickle
from dxtbx.format.image import ImageBool, ImageDouble
# Map of imageset/scan pairs
imagesets = {}
# For every experiment, use the given input to create
# a sensible experiment.
el = ExperimentList()
for eobj in self._obj['experiment']:
# Get the models
identifier = eobj.get("identifier", "")
beam = ExperimentListDict.model_or_none(self._blist, eobj, 'beam')
detector = ExperimentListDict.model_or_none(
self._dlist, eobj, 'detector')
goniometer = ExperimentListDict.model_or_none(
self._glist, eobj, 'goniometer')
scan = ExperimentListDict.model_or_none(self._slist, eobj, 'scan')
crystal = ExperimentListDict.model_or_none(self._clist, eobj,
'crystal')
profile = ExperimentListDict.model_or_none(self._plist, eobj,
'profile')
scaling_model = ExperimentListDict.model_or_none(
self._scalelist, eobj, 'scaling_model')
key = (eobj.get('imageset'), eobj.get('scan'))
try:
imageset = imagesets[key]
except Exception:
imageset = ExperimentListDict.model_or_none(
self._ilist, eobj, 'imageset')
# Create the imageset from the input data
if imageset is not None:
if 'params' in imageset:
format_kwargs = imageset['params']
else:
format_kwargs = {}
if 'mask' in imageset and imageset['mask'] is not None:
mask_filename = load_path(imageset['mask'],
directory=self._directory)
if self._check_format and mask_filename is not "":
with open(mask_filename, 'rb') as fh:
mask = pickle.load(fh)
else:
mask = None
else:
mask_filename = None
mask = None
if 'gain' in imageset and imageset['gain'] is not None:
gain_filename = load_path(imageset['gain'],
directory=self._directory)
if self._check_format and gain_filename is not "":
with open(gain_filename, 'rb') as fh:
gain = pickle.load(fh)
else:
gain = None
else:
gain_filename = None
gain = None
if 'pedestal' in imageset and imageset[
'pedestal'] is not None:
pedestal_filename = load_path(
imageset['pedestal'], directory=self._directory)
if self._check_format and pedestal_filename is not "":
with open(pedestal_filename, 'rb') as fh:
pedestal = pickle.load(fh)
else:
pedestal = None
else:
pedestal_filename = None
pedestal = None
if 'dx' in imageset and imageset['dx'] is not None:
dx_filename = load_path(imageset['dx'],
directory=self._directory)
if dx_filename is not "":
with open(dx_filename, 'rb') as fh:
dx = pickle.load(fh)
else:
dx = None
else:
dx_filename = None
dx = None
if 'dy' in imageset and imageset['dy'] is not None:
dy_filename = load_path(imageset['dy'],
directory=self._directory)
if dy_filename is not "":
with open(dy_filename, 'rb') as fh:
dy = pickle.load(fh)
else:
dy = None
else:
dy_filename = None
dy = None
if imageset['__id__'] == 'ImageSet':
imageset = self._make_stills(
imageset, format_kwargs=format_kwargs)
elif imageset['__id__'] == 'ImageGrid':
imageset = self._make_grid(imageset,
format_kwargs=format_kwargs)
elif imageset['__id__'] == 'ImageSweep':
imageset = self._make_sweep(
imageset,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
format_kwargs=format_kwargs)
elif imageset['__id__'] == 'MemImageSet':
imageset = self._make_mem_imageset(imageset)
else:
raise RuntimeError('Unknown imageset type')
# Set the external lookup
if imageset is not None:
if mask_filename is None:
mask_filename = ""
if gain_filename is None:
gain_filename = ""
if pedestal_filename is None:
pedestal_filename = ""
if dx_filename is None:
dx_filename = ""
if dy_filename is None:
dy_filename = ""
if mask is None:
mask = ImageBool()
else:
mask = ImageBool(mask)
if gain is None:
gain = ImageDouble()
else:
gain = ImageDouble(gain)
if pedestal is None:
pedestal = ImageDouble()
else:
pedestal = ImageDouble(pedestal)
if dx is None:
dx = ImageDouble()
else:
dx = ImageDouble(dx)
if dy is None:
dy = ImageDouble()
else:
dy = ImageDouble(dy)
imageset.external_lookup.mask.data = mask
imageset.external_lookup.mask.filename = mask_filename
imageset.external_lookup.gain.data = gain
imageset.external_lookup.gain.filename = gain_filename
imageset.external_lookup.pedestal.data = pedestal
imageset.external_lookup.pedestal.filename = pedestal_filename
imageset.external_lookup.dx.data = dx
imageset.external_lookup.dx.filename = dx_filename
imageset.external_lookup.dy.data = dy
imageset.external_lookup.dy.filename = dy_filename
# Update the imageset models
if isinstance(imageset, ImageSweep):
imageset.set_beam(beam)
imageset.set_detector(detector)
imageset.set_goniometer(goniometer)
imageset.set_scan(scan)
elif isinstance(imageset, ImageSet):
for i in range(len(imageset)):
imageset.set_beam(beam, i)
imageset.set_detector(detector, i)
imageset.set_goniometer(goniometer, i)
imageset.set_scan(scan, i)
elif isinstance(imageset, ImageGrid):
for i in range(len(imageset)):
imageset.set_beam(beam, i)
imageset.set_detector(detector, i)
imageset.set_goniometer(goniometer, i)
imageset.set_scan(scan, i)
else:
pass
if imageset is not None:
imageset.update_detector_px_mm_data()
# Add the imageset to the dict
imagesets[key] = imageset
# Append the experiment
el.append(
Experiment(imageset=imageset,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
profile=profile,
scaling_model=scaling_model,
identifier=identifier))
# Return the experiment list
return el
def run(self):
from dxtbx.imageset import ImageSetData
from dxtbx.format.image import ImageTileBool
from dxtbx.format.image import ImageTileDouble
from dxtbx.format.image import ImageBool
from dxtbx.format.image import ImageDouble
from scitbx.array_family import flex
import os.path
from glob import glob
from dxtbx.format.FormatCBFMiniPilatus import FormatCBFMiniPilatus as FormatClass
from os.path import join
dials_regression = libtbx.env.dist_path('dials_regression')
filenames = sorted(
glob(join(dials_regression, "centroid_test_data", "*.cbf")))
ReaderClass = FormatClass.get_reader()
MaskerClass = FormatClass.get_masker()
reader = ReaderClass(filenames)
masker = MaskerClass(filenames)
handle = ImageSetData(reader, masker)
data = handle.get_data(0).as_int().tile(0).data()
mask = handle.get_mask(0).tile(0).data()
assert handle.has_single_file_reader() == False
path = handle.get_path(0)
assert path == filenames[0]
master_path = handle.get_master_path()
assert master_path == ""
identifier = handle.get_image_identifier(0)
assert identifier == filenames[0]
beam = FormatClass(filenames[0]).get_beam()
detector = FormatClass(filenames[0]).get_detector()
goniometer = FormatClass(filenames[0]).get_goniometer()
scan = FormatClass(filenames[0]).get_scan()
handle.set_beam(beam, 0)
handle.set_detector(detector, 0)
handle.set_goniometer(goniometer, 0)
handle.set_scan(scan, 0)
beam2 = handle.get_beam(0)
detector2 = handle.get_detector(0)
goniometer2 = handle.get_goniometer(0)
scan2 = handle.get_scan(0)
assert beam2 == beam
assert detector2 == detector
assert goniometer2 == goniometer
assert scan2 == scan
mask = flex.bool(flex.grid(10, 10), True)
gain = flex.double(flex.grid(10, 10), 1)
pedestal = flex.double(flex.grid(10, 10), 2)
handle.external_lookup.mask.data = ImageBool(ImageTileBool(mask))
handle.external_lookup.gain.data = ImageDouble(ImageTileDouble(gain))
handle.external_lookup.pedestal.data = ImageDouble(
ImageTileDouble(pedestal))
mask2 = handle.external_lookup.mask.data.tile(0).data()
gain2 = handle.external_lookup.gain.data.tile(0).data()
pedestal2 = handle.external_lookup.pedestal.data.tile(0).data()
assert mask2.all_eq(mask)
assert gain2.all_eq(gain)
assert pedestal2.all_eq(pedestal)
print 'OK'
def _load_datablocks(self, obj, check_format=True, directory=None):
''' Create the datablock from a dictionary. '''
from dxtbx.format.Registry import Registry
from dxtbx.model import BeamFactory, DetectorFactory
from dxtbx.model import GoniometerFactory, ScanFactory
from dxtbx.serialize.filename import load_path
from dxtbx.format.image import ImageBool, ImageDouble
from dxtbx.format.FormatMultiImage import FormatMultiImage
# If we have a list, extract for each dictionary in the list
if isinstance(obj, list):
return [self._load_datablocks(dd, check_format, directory) for dd in obj]
elif not isinstance(obj, dict):
raise InvalidDataBlockError("Unexpected datablock type {} instead of dict".format(type(obj)))
# Make sure the id signature is correct
if not obj.get("__id__") == "DataBlock":
raise InvalidDataBlockError(
"Expected __id__ 'DataBlock', but found {}".format(repr(obj.get("__id__"))))
# Get the list of models
blist = obj.get('beam', [])
dlist = obj.get('detector', [])
glist = obj.get('goniometer', [])
slist = obj.get('scan', [])
def load_models(obj):
try:
beam = BeamFactory.from_dict(blist[obj['beam']])
except Exception:
beam = None
try:
dobj = dlist[obj['detector']]
detector = DetectorFactory.from_dict(dobj)
except Exception:
detector = None
try:
gonio = GoniometerFactory.from_dict(glist[obj['goniometer']])
except Exception:
gonio = None
try:
scan = ScanFactory.from_dict(slist[obj['scan']])
except Exception:
scan = None
return beam, detector, gonio, scan
# Loop through all the imagesets
imagesets = []
for imageset in obj['imageset']:
ident = imageset['__id__']
if "params" in imageset:
format_kwargs = imageset['params']
else:
format_kwargs = {}
if ident == 'ImageSweep':
beam, detector, gonio, scan = load_models(imageset)
if "template" in imageset:
template = load_path(imageset['template'], directory=directory)
i0, i1 = scan.get_image_range()
iset = dxtbx.imageset.ImageSetFactory.make_sweep(
template, range(i0, i1+1), None,
beam, detector, gonio, scan, check_format,
format_kwargs=format_kwargs)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'], directory=directory)
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'], directory=directory)
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'], directory=directory)
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
if 'dx' in imageset and imageset['dx'] is not None:
imageset['dx'] = load_path(imageset['dx'], directory=directory)
iset.external_lookup.dx.filename = imageset['dx']
with open(imageset['dx']) as infile:
iset.external_lookup.dx.data = ImageDouble(pickle.load(infile))
if 'dy' in imageset and imageset['dy'] is not None:
imageset['dy'] = load_path(imageset['dy'], directory=directory)
iset.external_lookup.dy.filename = imageset['dy']
with open(imageset['dy']) as infile:
iset.external_lookup.dy.data = ImageDouble(pickle.load(infile))
iset.update_detector_px_mm_data()
elif "master" in imageset:
template = load_path(imageset['master'], directory=directory)
i0, i1 = scan.get_image_range()
indices = imageset['images']
if check_format == False:
format_class = FormatMultiImage
else:
format_class = None
iset = dxtbx.imageset.ImageSetFactory.make_sweep(
template,
list(range(i0, i1+1)),
format_class = format_class,
beam = beam,
detector = detector,
goniometer = gonio,
scan = scan,
check_format = check_format,
format_kwargs = format_kwargs)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'], directory)
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'], directory)
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'], directory)
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
if 'dx' in imageset and imageset['dx'] is not None:
imageset['dx'] = load_path(imageset['dx'], directory)
iset.external_lookup.dx.filename = imageset['dx']
with open(imageset['dx']) as infile:
iset.external_lookup.dx.data = ImageDouble(pickle.load(infile))
if 'dy' in imageset and imageset['dy'] is not None:
imageset['dy'] = load_path(imageset['dy'], directory)
iset.external_lookup.dy.filename = imageset['dy']
with open(imageset['dy']) as infile:
iset.external_lookup.dy.data = ImageDouble(pickle.load(infile))
iset.update_detector_px_mm_data()
imagesets.append(iset)
elif ident == 'ImageSet' or ident == "ImageGrid":
filenames = [image['filename'] for image in imageset['images']]
indices = [image['image'] for image in imageset['images'] if 'image' in image]
assert len(indices) == 0 or len(indices) == len(filenames)
iset = dxtbx.imageset.ImageSetFactory.make_imageset(
filenames, None, check_format, indices, format_kwargs=format_kwargs)
if ident == "ImageGrid":
grid_size = imageset['grid_size']
iset = dxtbx.imageset.ImageGrid.from_imageset(iset, grid_size)
for i, image in enumerate(imageset['images']):
beam, detector, gonio, scan = load_models(image)
iset.set_beam(beam, i)
iset.set_detector(detector, i)
iset.set_goniometer(gonio, i)
iset.set_scan(scan, i)
if 'mask' in imageset and imageset['mask'] is not None:
imageset['mask'] = load_path(imageset['mask'], directory)
iset.external_lookup.mask.filename = imageset['mask']
if check_format:
with open(imageset['mask']) as infile:
iset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if 'gain' in imageset and imageset['gain'] is not None:
imageset['gain'] = load_path(imageset['gain'], directory)
iset.external_lookup.gain.filename = imageset['gain']
if check_format:
with open(imageset['gain']) as infile:
iset.external_lookup.gain.data = ImageDouble(pickle.load(infile))
if 'pedestal' in imageset and imageset['pedestal'] is not None:
imageset['pedestal'] = load_path(imageset['pedestal'], directory)
iset.external_lookup.pedestal.filename = imageset['pedestal']
if check_format:
with open(imageset['pedestal']) as infile:
iset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile))
if 'dx' in imageset and imageset['dx'] is not None:
imageset['dx'] = load_path(imageset['dx'], directory)
iset.external_lookup.dx.filename = imageset['dx']
with open(imageset['dx']) as infile:
iset.external_lookup.dx.data = ImageDouble(pickle.load(infile))
if 'dy' in imageset and imageset['dy'] is not None:
imageset['dy'] = load_path(imageset['dy'], directory)
iset.external_lookup.dy.filename = imageset['dy']
with open(imageset['dy']) as infile:
iset.external_lookup.dy.data = ImageDouble(pickle.load(infile))
iset.update_detector_px_mm_data()
imagesets.append(iset)
else:
raise RuntimeError('expected ImageSet/ImageSweep, got %s' % ident)
# Return the datablock
return DataBlock(imagesets)
def datablocks_from_dict(obj, check_format=True, directory=None):
"""Get the datablocks from the dictionary."""
# If we have a list, extract for each dictionary in the list
if isinstance(obj, list):
return [
datablocks_from_dict(dd, check_format, directory) for dd in obj
]
elif not isinstance(obj, dict):
raise InvalidDataBlockError(
"Unexpected datablock type {} instead of dict".format(type(obj)))
# Make sure the id signature is correct
if not obj.get("__id__") == "DataBlock":
raise InvalidDataBlockError(
"Expected __id__ 'DataBlock', but found {}".format(
repr(obj.get("__id__"))))
# Get the list of models
blist = obj.get("beam", [])
dlist = obj.get("detector", [])
glist = obj.get("goniometer", [])
slist = obj.get("scan", [])
def load_models(obj):
try:
beam = dxtbx.model.BeamFactory.from_dict(blist[obj["beam"]])
except Exception:
beam = None
try:
dobj = dlist[obj["detector"]]
detector = dxtbx.model.DetectorFactory.from_dict(dobj)
except Exception:
detector = None
try:
gonio = dxtbx.model.GoniometerFactory.from_dict(
glist[obj["goniometer"]])
except Exception:
gonio = None
try:
scan = dxtbx.model.ScanFactory.from_dict(slist[obj["scan"]])
except Exception:
scan = None
return beam, detector, gonio, scan
# Loop through all the imagesets
imagesets = []
for imageset in obj["imageset"]:
ident = imageset["__id__"]
if "params" in imageset:
format_kwargs = imageset["params"]
else:
format_kwargs = {}
if ident == "ImageSequence" or ident == "ImageSweep":
beam, detector, gonio, scan = load_models(imageset)
if "template" in imageset:
template = resolve_path(imageset["template"],
directory=directory)
i0, i1 = scan.get_image_range()
iset = dxtbx.imageset.ImageSetFactory.make_sequence(
template,
list(range(i0, i1 + 1)),
None,
beam,
detector,
gonio,
scan,
check_format,
format_kwargs=format_kwargs,
)
if "mask" in imageset and imageset["mask"] is not None:
imageset["mask"] = resolve_path(imageset["mask"],
directory=directory)
iset.external_lookup.mask.filename = imageset["mask"]
if check_format:
with open(imageset["mask"], "rb") as infile:
iset.external_lookup.mask.data = ImageBool(
pickle.load(infile, encoding="bytes"))
if "gain" in imageset and imageset["gain"] is not None:
imageset["gain"] = resolve_path(imageset["gain"],
directory=directory)
iset.external_lookup.gain.filename = imageset["gain"]
if check_format:
with open(imageset["gain"], "rb") as infile:
iset.external_lookup.gain.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "pedestal" in imageset and imageset["pedestal"] is not None:
imageset["pedestal"] = resolve_path(imageset["pedestal"],
directory=directory)
iset.external_lookup.pedestal.filename = imageset[
"pedestal"]
if check_format:
with open(imageset["pedestal"], "rb") as infile:
iset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dx" in imageset and imageset["dx"] is not None:
imageset["dx"] = resolve_path(imageset["dx"],
directory=directory)
iset.external_lookup.dx.filename = imageset["dx"]
with open(imageset["dx"], "rb") as infile:
iset.external_lookup.dx.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dy" in imageset and imageset["dy"] is not None:
imageset["dy"] = resolve_path(imageset["dy"],
directory=directory)
iset.external_lookup.dy.filename = imageset["dy"]
with open(imageset["dy"], "rb") as infile:
iset.external_lookup.dy.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
iset.update_detector_px_mm_data()
elif "master" in imageset:
template = resolve_path(imageset["master"],
directory=directory)
i0, i1 = scan.get_image_range()
if not check_format:
format_class = FormatMultiImage
else:
format_class = None
iset = dxtbx.imageset.ImageSetFactory.make_sequence(
template,
list(range(i0, i1 + 1)),
format_class=format_class,
beam=beam,
detector=detector,
goniometer=gonio,
scan=scan,
check_format=check_format,
format_kwargs=format_kwargs,
)
if "mask" in imageset and imageset["mask"] is not None:
imageset["mask"] = resolve_path(imageset["mask"],
directory)
iset.external_lookup.mask.filename = imageset["mask"]
if check_format:
with open(imageset["mask"], "rb") as infile:
iset.external_lookup.mask.data = ImageBool(
pickle.load(infile, encoding="bytes"))
if "gain" in imageset and imageset["gain"] is not None:
imageset["gain"] = resolve_path(imageset["gain"],
directory)
iset.external_lookup.gain.filename = imageset["gain"]
if check_format:
with open(imageset["gain"], "rb") as infile:
iset.external_lookup.gain.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "pedestal" in imageset and imageset["pedestal"] is not None:
imageset["pedestal"] = resolve_path(
imageset["pedestal"], directory)
iset.external_lookup.pedestal.filename = imageset[
"pedestal"]
if check_format:
with open(imageset["pedestal"], "rb") as infile:
iset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dx" in imageset and imageset["dx"] is not None:
imageset["dx"] = resolve_path(imageset["dx"], directory)
iset.external_lookup.dx.filename = imageset["dx"]
with open(imageset["dx"], "rb") as infile:
iset.external_lookup.dx.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dy" in imageset and imageset["dy"] is not None:
imageset["dy"] = resolve_path(imageset["dy"], directory)
iset.external_lookup.dy.filename = imageset["dy"]
with open(imageset["dy"], "rb") as infile:
iset.external_lookup.dy.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
iset.update_detector_px_mm_data()
imagesets.append(iset)
elif ident == "ImageSet" or ident == "ImageGrid":
filenames = [image["filename"] for image in imageset["images"]]
indices = [
image["image"] for image in imageset["images"]
if "image" in image
]
assert len(indices) == 0 or len(indices) == len(filenames)
iset = dxtbx.imageset.ImageSetFactory.make_imageset(
filenames,
None,
check_format,
indices,
format_kwargs=format_kwargs)
if ident == "ImageGrid":
grid_size = imageset["grid_size"]
iset = dxtbx.imageset.ImageGrid.from_imageset(iset, grid_size)
for i, image in enumerate(imageset["images"]):
beam, detector, gonio, scan = load_models(image)
iset.set_beam(beam, i)
iset.set_detector(detector, i)
iset.set_goniometer(gonio, i)
iset.set_scan(scan, i)
if "mask" in imageset and imageset["mask"] is not None:
imageset["mask"] = resolve_path(imageset["mask"], directory)
iset.external_lookup.mask.filename = imageset["mask"]
if check_format:
with open(imageset["mask"], "rb") as infile:
iset.external_lookup.mask.data = ImageBool(
pickle.load(infile, encoding="bytes"))
if "gain" in imageset and imageset["gain"] is not None:
imageset["gain"] = resolve_path(imageset["gain"], directory)
iset.external_lookup.gain.filename = imageset["gain"]
if check_format:
with open(imageset["gain"], "rb") as infile:
iset.external_lookup.gain.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "pedestal" in imageset and imageset["pedestal"] is not None:
imageset["pedestal"] = resolve_path(imageset["pedestal"],
directory)
iset.external_lookup.pedestal.filename = imageset["pedestal"]
if check_format:
with open(imageset["pedestal"], "rb") as infile:
iset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dx" in imageset and imageset["dx"] is not None:
imageset["dx"] = resolve_path(imageset["dx"], directory)
iset.external_lookup.dx.filename = imageset["dx"]
with open(imageset["dx"], "rb") as infile:
iset.external_lookup.dx.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dy" in imageset and imageset["dy"] is not None:
imageset["dy"] = resolve_path(imageset["dy"], directory)
iset.external_lookup.dy.filename = imageset["dy"]
with open(imageset["dy"], "rb") as infile:
iset.external_lookup.dy.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
iset.update_detector_px_mm_data()
imagesets.append(iset)
else:
raise RuntimeError("expected ImageSet/ImageSequence, got %s" %
ident)
return DataBlock(imagesets)
def __call__(self, experiments):
"""
Do the spot finding.
:param experiments: The experiments to process
:return: The observed spots
"""
import six.moves.cPickle as pickle
from dxtbx.format.image import ImageBool
# Loop through all the experiments and get the unique imagesets
imagesets = []
for experiment in experiments:
if experiment.imageset not in imagesets:
imagesets.append(experiment.imageset)
# Loop through all the imagesets and find the strong spots
reflections = flex.reflection_table()
for j, imageset in enumerate(imagesets):
# Find the strong spots in the sequence
logger.info("-" * 80)
logger.info("Finding strong spots in imageset %d" % j)
logger.info("-" * 80)
logger.info("")
table, hot_mask = self._find_spots_in_imageset(imageset)
# Fix up the experiment ID's now
table["id"] = flex.int(table.nrows(), -1)
for i, experiment in enumerate(experiments):
if experiment.imageset is not imageset:
continue
if experiment.scan:
z0, z1 = experiment.scan.get_array_range()
z = table["xyzobs.px.value"].parts()[2]
table["id"].set_selected((z > z0) & (z < z1), i)
if experiment.identifier:
table.experiment_identifiers()[i] = experiment.identifier
else:
table["id"] = flex.int(table.nrows(), j)
if experiment.identifier:
table.experiment_identifiers()[j] = experiment.identifier
missed = table["id"] == -1
assert missed.count(True) == 0, missed.count(True)
reflections.extend(table)
# Write a hot pixel mask
if self.write_hot_mask:
if not imageset.external_lookup.mask.data.empty():
for m1, m2 in zip(hot_mask, imageset.external_lookup.mask.data):
m1 &= m2.data()
imageset.external_lookup.mask.data = ImageBool(hot_mask)
else:
imageset.external_lookup.mask.data = ImageBool(hot_mask)
imageset.external_lookup.mask.filename = "%s_%d.pickle" % (
self.hot_mask_prefix,
i,
)
# Write the hot mask
with open(imageset.external_lookup.mask.filename, "wb") as outfile:
pickle.dump(hot_mask, outfile, protocol=pickle.HIGHEST_PROTOCOL)
# Set the strong spot flag
reflections.set_flags(
flex.size_t_range(len(reflections)), reflections.flags.strong
)
# Check for overloads
reflections.is_overloaded(experiments)
# Return the reflections
return reflections
def get_imageset(
cls,
filenames,
beam=None,
detector=None,
goniometer=None,
scan=None,
as_sequence=False,
as_imageset=False,
single_file_indices=None,
format_kwargs=None,
template=None,
check_format=True,
lazy=False,
):
"""
Factory method to create an imageset
"""
if isinstance(filenames, str):
filenames = [filenames]
elif len(filenames) > 1:
assert len(set(filenames)) == 1
filenames = filenames[0:1]
# Make filenames absolute
filenames = [os.path.abspath(x) for x in filenames]
# Make it a dictionary
if format_kwargs is None:
format_kwargs = {}
# If get_num_images hasn't been implemented, we need indices for number of images
if cls.get_num_images == FormatMultiImage.get_num_images:
assert single_file_indices is not None
assert min(single_file_indices) >= 0
num_images = max(single_file_indices) + 1
else:
num_images = None
# Get the format instance
assert len(filenames) == 1
if check_format is True:
format_instance = cls(filenames[0], **format_kwargs)
if num_images is None and not lazy:
# As we now have the actual format class we can get the number
# of images from here. This saves having to create another
# format class instance in the Reader() constructor
# NOTE: Having this information breaks internal assumptions in
# *Lazy classes, so they have to figure this out in
# their own time.
num_images = format_instance.get_num_images()
else:
format_instance = None
if not as_sequence:
lazy = True
# Get some information from the format class
reader = cls.get_reader()(filenames, num_images=num_images, **format_kwargs)
# Read the vendor type
if check_format is True:
vendor = format_instance.get_vendortype()
else:
vendor = ""
# Get the format kwargs
params = format_kwargs
# Check if we have a sequence
# Make sure only 1 or none is set
assert [as_imageset, as_sequence].count(True) < 2
if as_imageset:
is_sequence = False
elif as_sequence:
is_sequence = True
else:
if scan is None and format_instance is None:
raise RuntimeError(
"""
One of the following needs to be set
- as_imageset=True
- as_sequence=True
- scan
- check_format=True
"""
)
if scan is None:
test_scan = format_instance.get_scan()
else:
test_scan = scan
if test_scan is not None:
is_sequence = True
else:
is_sequence = False
assert not (as_sequence and lazy), "No lazy support for sequences"
if single_file_indices is not None:
single_file_indices = flex.size_t(single_file_indices)
# Create an imageset or sequence
if not is_sequence:
# Use imagesetlazy
# Setup ImageSetLazy and just return it. No models are set.
if lazy:
iset = ImageSetLazy(
ImageSetData(
reader=reader,
masker=None,
vendor=vendor,
params=params,
format=cls,
),
indices=single_file_indices,
)
return iset
# Create the imageset
iset = ImageSet(
ImageSetData(
reader=reader, masker=None, vendor=vendor, params=params, format=cls
),
indices=single_file_indices,
)
# If any are None then read from format
if [beam, detector, goniometer, scan].count(None) != 0:
# Get list of models
beam = []
detector = []
goniometer = []
scan = []
for i in range(format_instance.get_num_images()):
beam.append(format_instance.get_beam(i))
detector.append(format_instance.get_detector(i))
goniometer.append(format_instance.get_goniometer(i))
scan.append(format_instance.get_scan(i))
if single_file_indices is None:
single_file_indices = list(range(format_instance.get_num_images()))
# Set the list of models
for i in range(len(single_file_indices)):
iset.set_beam(beam[single_file_indices[i]], i)
iset.set_detector(detector[single_file_indices[i]], i)
iset.set_goniometer(goniometer[single_file_indices[i]], i)
iset.set_scan(scan[single_file_indices[i]], i)
else:
# Get the template
template = filenames[0]
# Check indices are sequential
if single_file_indices is not None:
assert all(
i + 1 == j
for i, j in zip(single_file_indices[:-1], single_file_indices[1:])
)
num_images = len(single_file_indices)
else:
num_images = format_instance.get_num_images()
# Check the scan makes sense - we must want to use <= total images
if scan is not None:
assert scan.get_num_images() <= num_images
# If any are None then read from format
if beam is None:
beam = format_instance.get_beam()
if detector is None:
detector = format_instance.get_detector()
if goniometer is None:
goniometer = format_instance.get_goniometer()
if scan is None:
scan = format_instance.get_scan()
if scan is not None:
for f in filenames[1:]:
format_instance = cls(f, **format_kwargs)
scan += format_instance.get_scan()
# Create the masker
if format_instance is not None:
masker = format_instance.get_masker(goniometer=goniometer)
else:
masker = None
isetdata = ImageSetData(
reader=reader,
masker=masker,
vendor=vendor,
params=params,
format=cls,
template=template,
)
# Create the sequence
iset = ImageSequence(
isetdata,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
indices=single_file_indices,
)
if format_instance is not None:
static_mask = format_instance.get_static_mask()
if static_mask is not None:
if not iset.external_lookup.mask.data.empty():
for m1, m2 in zip(static_mask, iset.external_lookup.mask.data):
m1 &= m2.data()
iset.external_lookup.mask.data = ImageBool(static_mask)
else:
iset.external_lookup.mask.data = ImageBool(static_mask)
return iset
def _imageset_from_imageset_data(self, imageset_data, models):
""" Make an imageset from imageset_data - help with refactor decode. """
assert imageset_data is not None
if "params" in imageset_data:
format_kwargs = imageset_data["params"]
else:
format_kwargs = {}
beam = models["beam"]
detector = models["detector"]
goniometer = models["goniometer"]
scan = models["scan"]
# Load the external lookup data
mask_filename, mask = self._load_pickle_path(imageset_data, "mask")
gain_filename, gain = self._load_pickle_path(imageset_data, "gain")
pedestal_filename, pedestal = self._load_pickle_path(imageset_data, "pedestal")
dx_filename, dx = self._load_pickle_path(imageset_data, "dx")
dy_filename, dy = self._load_pickle_path(imageset_data, "dy")
if imageset_data["__id__"] == "ImageSet":
imageset = self._make_stills(imageset_data, format_kwargs=format_kwargs)
elif imageset_data["__id__"] == "ImageGrid":
imageset = self._make_grid(imageset_data, format_kwargs=format_kwargs)
elif (
imageset_data["__id__"] == "ImageSequence"
or imageset_data["__id__"] == "ImageSweep"
):
imageset = self._make_sequence(
imageset_data,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
format_kwargs=format_kwargs,
)
elif imageset_data["__id__"] == "MemImageSet":
imageset = self._make_mem_imageset(imageset_data)
else:
raise RuntimeError("Unknown imageset type")
if imageset is not None:
# Set the external lookup
if mask is None:
mask = ImageBool()
else:
mask = ImageBool(mask)
if gain is None:
gain = ImageDouble()
else:
gain = ImageDouble(gain)
if pedestal is None:
pedestal = ImageDouble()
else:
pedestal = ImageDouble(pedestal)
if dx is None:
dx = ImageDouble()
else:
dx = ImageDouble(dx)
if dy is None:
dy = ImageDouble()
else:
dy = ImageDouble(dy)
if not imageset.external_lookup.mask.data.empty():
if not mask.empty():
mask = tuple(m.data() for m in mask)
for m1, m2 in zip(mask, imageset.external_lookup.mask.data):
m1 &= m2.data()
imageset.external_lookup.mask.data = ImageBool(mask)
else:
imageset.external_lookup.mask.data = mask
imageset.external_lookup.mask.filename = mask_filename
imageset.external_lookup.gain.data = gain
imageset.external_lookup.gain.filename = gain_filename
imageset.external_lookup.pedestal.data = pedestal
imageset.external_lookup.pedestal.filename = pedestal_filename
imageset.external_lookup.dx.data = dx
imageset.external_lookup.dx.filename = dx_filename
imageset.external_lookup.dy.data = dy
imageset.external_lookup.dy.filename = dy_filename
# Update the imageset models
if isinstance(imageset, ImageSequence):
imageset.set_beam(beam)
imageset.set_detector(detector)
imageset.set_goniometer(goniometer)
imageset.set_scan(scan)
elif isinstance(imageset, (ImageSet, ImageGrid)):
for i in range(len(imageset)):
imageset.set_beam(beam, i)
imageset.set_detector(detector, i)
imageset.set_goniometer(goniometer, i)
imageset.set_scan(scan, i)
imageset.update_detector_px_mm_data()
return imageset
def get_imageset(
Class,
input_filenames,
beam=None,
detector=None,
goniometer=None,
scan=None,
as_imageset=False,
as_sequence=False,
single_file_indices=None,
format_kwargs=None,
template=None,
check_format=True,
):
"""
Factory method to create an imageset
"""
# Import here to avoid cyclic imports
from dxtbx.imageset import ImageSequence, ImageSet, ImageSetData
# Get filename absolute paths, for entries that are filenames
filenames = [
os.path.abspath(x) if not get_url_scheme(x) else x
for x in input_filenames
]
# Make it a dict
if format_kwargs is None:
format_kwargs = {}
# Get some information from the format class
reader = Class.get_reader()(filenames, **format_kwargs)
# Get the format instance
if check_format is True:
Class._current_filename_ = None
Class._current_instance_ = None
format_instance = Class.get_instance(filenames[0], **format_kwargs)
else:
format_instance = None
# Read the vendor type
if check_format is True:
vendor = format_instance.get_vendortype()
else:
vendor = ""
# Get the format kwargs
params = format_kwargs
# Make sure only 1 or none is set
assert [as_imageset, as_sequence].count(True) < 2
if as_imageset:
is_sequence = False
elif as_sequence:
is_sequence = True
else:
if scan is None and format_instance is None:
raise RuntimeError("""
One of the following needs to be set
- as_imageset=True
- as_sequence=True
- scan
- check_format=True
""")
if scan is None:
test_scan = format_instance.get_scan()
else:
test_scan = scan
if test_scan is not None and test_scan.get_oscillation()[1] != 0:
is_sequence = True
else:
is_sequence = False
# Create an imageset or sequence
if not is_sequence:
# Create the imageset
iset = ImageSet(
ImageSetData(
reader=reader,
masker=None,
vendor=vendor,
params=params,
format=Class,
))
# If any are None then read from format
if [beam, detector, goniometer, scan].count(None) != 0:
# Get list of models
beam = []
detector = []
goniometer = []
scan = []
for f in filenames:
format_instance = Class(f, **format_kwargs)
beam.append(format_instance.get_beam())
detector.append(format_instance.get_detector())
goniometer.append(format_instance.get_goniometer())
scan.append(format_instance.get_scan())
# Set the list of models
for i in range(len(filenames)):
iset.set_beam(beam[i], i)
iset.set_detector(detector[i], i)
iset.set_goniometer(goniometer[i], i)
iset.set_scan(scan[i], i)
else:
# Get the template
if template is None:
template = template_regex(filenames[0])[0]
else:
template = str(template)
# Check scan makes sense
if scan:
if check_format is True:
assert scan.get_num_images() == len(filenames)
# If any are None then read from format
if beam is None and format_instance is not None:
beam = format_instance.get_beam()
if detector is None and format_instance is not None:
detector = format_instance.get_detector()
if goniometer is None and format_instance is not None:
goniometer = format_instance.get_goniometer()
if scan is None and format_instance is not None:
scan = format_instance.get_scan()
if scan is not None:
for f in filenames[1:]:
format_instance = Class(f, **format_kwargs)
scan += format_instance.get_scan()
assert beam is not None, "Can't create Sequence without beam"
assert detector is not None, "Can't create Sequence without detector"
assert goniometer is not None, "Can't create Sequence without goniometer"
assert scan is not None, "Can't create Sequence without scan"
# Create the masker
if format_instance is not None:
masker = format_instance.get_masker(goniometer=goniometer)
else:
masker = None
# Create the sequence
iset = ImageSequence(
ImageSetData(
reader=reader,
masker=masker,
vendor=vendor,
params=params,
format=Class,
template=template,
),
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
)
if format_instance is not None:
static_mask = format_instance.get_static_mask()
if static_mask is not None:
if not iset.external_lookup.mask.data.empty():
for m1, m2 in zip(static_mask,
iset.external_lookup.mask.data):
m1 &= m2.data()
iset.external_lookup.mask.data = ImageBool(static_mask)
else:
iset.external_lookup.mask.data = ImageBool(static_mask)
return iset
def generate_mask(
experiments: ExperimentList,
params: phil.scope_extract,
) -> Tuple[Masks, Optional[ExperimentList]]:
"""
Generate a pixel mask for each imageset in an experiment list.
Use the masking parameters :param:`params` and the experiments in the experiment
list :param:`experiments` to define a pixel mask for each of the associated
imagesets. The masks are generated using :mod:`dials.util.masking`.
The masks will be saved to disk at the location specified by
:attr:`params.output.mask`. If the experiment list contains more than one
imageset, multiple mask files will be produced, with filenames differentiated by
an appended number. Optionally, if a path :attr:`params.output.experiments` is
set, a modified copy of :param:`experiments` with the masks applied will be saved
to that location.
Args:
experiments: An experiment list containing only one imageset.
params: Masking parameters, having the structure defined in
:data:`phil_scope`.
Returns:
A list of masks, one for each imageset.
A copy of :param:`experiments` with the masks applied (optional,
only returned if :attr:`params.output.experiments` is set).
"""
imagesets = experiments.imagesets()
masks = []
# Create output mask filenames
num_imagesets = len(imagesets)
if num_imagesets == 1:
filenames = [params.output.mask]
else:
# If there is more than one imageset, append a number to each output filename
name, ext = os.path.splitext(params.output.mask)
pad = len(str(num_imagesets))
filenames = [
"{name}_{num:0{pad}}{ext}".format(name=name,
num=i + 1,
pad=pad,
ext=ext)
for i in range(num_imagesets)
]
for imageset, filename in zip(imagesets, filenames):
mask = dials.util.masking.generate_mask(imageset, params)
masks.append(mask)
# Save the mask to file
log.info("Writing mask to %s", filename)
with open(filename, "wb") as fh:
pickle.dump(mask, fh)
if params.output.experiments:
# Apply the mask to the imageset
imageset.external_lookup.mask.data = ImageBool(mask)
imageset.external_lookup.mask.filename = filename
if params.output.experiments:
# Save the experiment list
log.info("Saving experiments to %s", params.output.experiments)
experiments.as_file(params.output.experiments)
else:
experiments = None
return masks, experiments
