def __call__(self, experiments): """ Do the spot finding. :param experiments: The experiments to process :return: The observed spots """ from dials.array_family import flex import six.moves.cPickle as pickle from dxtbx.format.image import ImageBool # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table() for i, experiment in enumerate(experiments): imageset = experiment.imageset # Find the strong spots in the sweep logger.info("-" * 80) logger.info("Finding strong spots in imageset %d" % i) logger.info("-" * 80) logger.info("") table, hot_mask = self._find_spots_in_imageset(imageset) table["id"] = flex.int(table.nrows(), i) reflections.extend(table) # Write a hot pixel mask if self.write_hot_mask: if not imageset.external_lookup.mask.data.empty(): for m1, m2 in zip(hot_mask, imageset.external_lookup.mask.data): m1 &= m2.data() imageset.external_lookup.mask.data = ImageBool(hot_mask) else: imageset.external_lookup.mask.data = ImageBool(hot_mask) imageset.external_lookup.mask.filename = "%s_%d.pickle" % ( self.hot_mask_prefix, i, ) # Write the hot mask with open(imageset.external_lookup.mask.filename, "wb") as outfile: pickle.dump(hot_mask, outfile, protocol=pickle.HIGHEST_PROTOCOL) # Set the strong spot flag reflections.set_flags(flex.size_t_range(len(reflections)), reflections.flags.strong) # Check for overloads reflections.is_overloaded(experiments) # Return the reflections return reflections
def _add_static_mask_to_iset(format_instance: Format, iset: ImageSet) -> None: """Combine any static mask from a Format with the ImageSet's mask""" if format_instance is not None: static_mask = format_instance.get_static_mask() if static_mask is not None: if not iset.external_lookup.mask.data.empty(): for m1, m2 in zip(static_mask, iset.external_lookup.mask.data): m1 &= m2.data() iset.external_lookup.mask.data = ImageBool(static_mask) else: iset.external_lookup.mask.data = ImageBool(static_mask)
def update_lookup(self, imageset, lookup): from dxtbx.format.image import ImageBool, ImageDouble if lookup.size is not None: d = imageset.get_detector() assert len(lookup.size) == len(d), "Incompatible size" for s, p in zip(lookup.size, d): assert s == p.get_image_size()[::-1], "Incompatible size" if lookup.mask.filename is not None: imageset.external_lookup.mask.filename = lookup.mask.filename imageset.external_lookup.mask.data = ImageBool( lookup.mask.data) if lookup.gain.filename is not None: imageset.external_lookup.gain.filename = lookup.gain.filename imageset.external_lookup.gain.data = ImageDouble( lookup.gain.data) if lookup.dark.filename is not None: imageset.external_lookup.pedestal.filename = lookup.dark.filename imageset.external_lookup.pedestal.data = ImageDouble( lookup.dark.data) if lookup.dx.filename is not None: imageset.external_lookup.dx.filename = lookup.dx.filename imageset.external_lookup.dx.data = ImageDouble(lookup.dx.data) if lookup.dy.filename is not None: imageset.external_lookup.dy.filename = lookup.dy.filename imageset.external_lookup.dy.data = ImageDouble(lookup.dy.data) return imageset
def imagesweep_from_dict(d, check_format=True, directory=None): """Construct and image sweep from the dictionary.""" # Get the template (required) template = load_path(str(d["template"]), directory=directory) # If the scan isn't set, find all available files scan_dict = d.get("scan") if scan_dict is None: image_range = None else: image_range = scan_dict.get("image_range") # Set the models with the exisiting models as templates beam = BeamFactory.from_dict(d.get("beam")) goniometer = GoniometerFactory.from_dict(d.get("goniometer")) detector = DetectorFactory.from_dict(d.get("detector")) scan = ScanFactory.from_dict(d.get("scan")) # Construct the sweep try: sweep = ImageSetFactory.from_template( template, image_range, beam=beam, detector=detector, goniometer=goniometer, scan=scan, check_format=check_format, )[0] except Exception: indices = range(image_range[0], image_range[1] + 1) sweep = ImageSetFactory.make_sweep( template, indices, beam=beam, detector=detector, goniometer=goniometer, scan=scan, check_format=check_format, ) # Set some external lookups if "mask" in d and d["mask"] is not None and d["mask"] is not "": path = load_path(d["mask"], directory=directory) with open(path) as infile: sweep.external_lookup.mask.filename = path sweep.external_lookup.mask.data = ImageBool(pickle.load(infile)) if "gain" in d and d["gain"] is not None and d["gain"] is not "": path = load_path(d["gain"], directory=directory) with open(path) as infile: sweep.external_lookup.gain.filename = path sweep.external_lookup.gain.data = ImageDouble(pickle.load(infile)) if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] is not "": path = load_path(d["pedestal"], directory=directory) with open(path) as infile: sweep.external_lookup.pedestal.filename = path sweep.external_lookup.pedestal.data = ImageDouble(pickle.load(infile)) # Return the sweep return sweep
def basic_imageset_from_dict(d): ''' Construct an ImageSet class from the dictionary.''' from dxtbx.model import BeamFactory, DetectorFactory from dxtbx.imageset import ImageSetFactory from dxtbx.serialize.filename import load_path # Get the filename list and create the imageset filenames = map(lambda p: load_path(p), map(str, d['filenames'])) imageset = ImageSetFactory.new(filenames)[0] # Set some external lookups if 'mask' in d and d['mask'] is not None and d['mask'] is not "": with open(d['mask']) as infile: imageset.external_lookup.mask.filename = d['mask'] imageset.external_lookup.mask.data = ImageBool(pickle.load(infile)) if 'gain' in d and d['gain'] is not None and d['gain'] is not "": with open(d['gain']) as infile: imageset.external_lookup.gain.filename = d['gain'] imageset.external_lookup.gain.data = ImageDouble(pickle.load(infile)) if 'pedestal' in d and d['pedestal'] is not None and d['pedestal'] is not "": with open(d['pedestal']) as infile: imageset.external_lookup.pedestal.filename = d['pedestal'] imageset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile)) # Get the existing models as dictionaries beam_dict = imageset.get_beam(0).to_dict() detector_dict = imageset.get_detector(0).to_dict() # Set models imageset.set_beam(BeamFactory.from_dict(d.get('beam'), beam_dict)) imageset.set_detector(DetectorFactory.from_dict(d.get('detector'), detector_dict)) # Return the imageset return imageset
def basic_imageset_from_dict(d, directory=None): """ Construct an ImageSet class from the dictionary.""" # Get the filename list and create the imageset filenames = map( lambda p: load_path(p, directory=directory), map(str, d["filenames"]) ) imageset = ImageSetFactory.new(filenames)[0] # Set some external lookups if "mask" in d and d["mask"] is not None and d["mask"] is not "": path = load_path(d["mask"], directory=directory) with open(path) as infile: imageset.external_lookup.mask.filename = path imageset.external_lookup.mask.data = ImageBool(pickle.load(infile)) if "gain" in d and d["gain"] is not None and d["gain"] is not "": path = load_path(d["gain"], directory=directory) with open(path) as infile: imageset.external_lookup.gain.filename = path imageset.external_lookup.gain.data = ImageDouble(pickle.load(infile)) if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] is not "": path = load_path(d["pedestal"], directory=directory) with open(path) as infile: imageset.external_lookup.pedestal.filename = path imageset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile)) # Get the existing models as dictionaries beam_dict = imageset.get_beam(0).to_dict() detector_dict = imageset.get_detector(0).to_dict() # Set models imageset.set_beam(BeamFactory.from_dict(d.get("beam"), beam_dict)) imageset.set_detector(DetectorFactory.from_dict(d.get("detector"), detector_dict)) # Return the imageset return imageset
def run(self): from dxtbx.imageset import ExternalLookup from dxtbx.format.image import ImageTileBool from dxtbx.format.image import ImageTileDouble from dxtbx.format.image import ImageBool from dxtbx.format.image import ImageDouble from scitbx.array_family import flex mask = flex.bool(flex.grid(10, 10), True) gain = flex.double(flex.grid(10, 10), 1) pedestal = flex.double(flex.grid(10, 10), 2) lookup = ExternalLookup() lookup.mask.data = ImageBool(ImageTileBool(mask)) lookup.gain.data = ImageDouble(ImageTileDouble(gain)) lookup.pedestal.data = ImageDouble(ImageTileDouble(pedestal)) mask2 = lookup.mask.data.tile(0).data() gain2 = lookup.gain.data.tile(0).data() pedestal2 = lookup.pedestal.data.tile(0).data() assert mask2.all_eq(mask) assert gain2.all_eq(gain) assert pedestal2.all_eq(pedestal) print 'OK'
def imagesweep_from_dict(d, check_format=True): '''Construct and image sweep from the dictionary.''' from dxtbx.imageset import ImageSetFactory from dxtbx.model import BeamFactory, DetectorFactory, GoniometerFactory, ScanFactory from dxtbx.serialize.filename import load_path # Get the template (required) template = load_path(str(d['template'])) # If the scan isn't set, find all available files scan_dict = d.get('scan') if scan_dict is None: image_range = None else: image_range = scan_dict.get('image_range') # Set the models with the exisiting models as templates beam = BeamFactory.from_dict(d.get('beam')) goniometer = GoniometerFactory.from_dict(d.get('goniometer')) detector = DetectorFactory.from_dict(d.get('detector')) scan = ScanFactory.from_dict(d.get('scan')) # Construct the sweep try: sweep = ImageSetFactory.from_template(template, image_range, beam=beam, detector=detector, goniometer=goniometer, scan=scan, check_format=check_format)[0] except Exception: indices = range(image_range[0], image_range[1] + 1) sweep = ImageSetFactory.make_sweep(template, indices, beam=beam, detector=detector, goniometer=goniometer, scan=scan, check_format=check_format) # Set some external lookups if 'mask' in d and d['mask'] is not None and d['mask'] is not "": with open(d['mask']) as infile: sweep.external_lookup.mask.filename = d['mask'] sweep.external_lookup.mask.data = ImageBool(pickle.load(infile)) if 'gain' in d and d['gain'] is not None and d['gain'] is not "": with open(d['gain']) as infile: sweep.external_lookup.gain.filename = d['gain'] sweep.external_lookup.gain.data = ImageDouble(pickle.load(infile)) if 'pedestal' in d and d['pedestal'] is not None and d[ 'pedestal'] is not "": with open(d['pedestal']) as infile: sweep.external_lookup.pedestal.filename = d['pedestal'] sweep.external_lookup.pedestal.data = ImageDouble( pickle.load(infile)) # Return the sweep return sweep
def run(self): ''' Run the script. ''' from dials.util.masking import MaskGenerator from dials.util.options import flatten_datablocks from libtbx.utils import Sorry import six.moves.cPickle as pickle from dials.util import log from dxtbx.format.image import ImageBool # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) datablocks = flatten_datablocks(params.input.datablock) # Configure logging log.config() # Check number of args if len(datablocks) == 0: self.parser.print_help() return if len(datablocks) != 1: raise Sorry('exactly 1 datablock must be specified') datablock = datablocks[0] imagesets = datablock.extract_imagesets() if len(imagesets) != 1: raise Sorry('datablock must contain exactly 1 imageset') imageset = imagesets[0] # Generate the mask generator = MaskGenerator(params) mask = generator.generate(imageset) # Save the mask to file print("Writing mask to %s" % params.output.mask) with open(params.output.mask, "wb") as fh: pickle.dump(mask, fh) # Save the datablock if params.output.datablock is not None: imageset.external_lookup.mask.data = ImageBool(mask) imageset.external_lookup.mask.filename = params.output.mask from dxtbx.datablock import DataBlockDumper print('Saving datablocks to {0}'.format(params.output.datablock)) dump = DataBlockDumper(datablocks) dump.as_file(params.output.datablock)
def run(self, args=None): """Run the script.""" from dials.util import Sorry from dials.util.options import flatten_experiments # Parse the command line arguments params, options = self.parser.parse_args(args, show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) # Check that an experiment list and at least one mask file have been provided if not (experiments and params.input.mask): self.parser.print_help() return # Check number of experiments n_expts = len(experiments) n_masks = len(params.input.mask) if n_expts != n_masks: raise Sorry( "The number of masks provided must match the number of imagesets " "(sequences).\n" "You have provided an experiment list containing {} imageset(s).\n" "You have provided {} mask file(s).".format(n_expts, n_masks) ) # Get the imageset imagesets = experiments.imagesets() for i, imageset in enumerate(imagesets): # Set the lookup with open(params.input.mask[i], "rb") as f: if six.PY3: mask = pickle.load(f, encoding="bytes") else: mask = pickle.load(f) imageset.external_lookup.mask.filename = params.input.mask[i] imageset.external_lookup.mask.data = ImageBool(mask) # Dump the experiments print("Writing experiments to %s" % params.output.experiments) experiments.as_file(filename=params.output.experiments)
def run(self): """ Run the script. """ from dials.util.options import flatten_experiments from dxtbx.model.experiment_list import ExperimentListDumper from dials.util import Sorry # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) # Check that an experiment list and at least one mask file have been provided if not (experiments and params.input.mask): self.parser.print_help() return # Check number of experiments n_expts = len(experiments) n_masks = len(params.input.mask) if n_expts != n_masks: raise Sorry( "The number of masks provided must match the number of imagesets " "(sweeps).\n" "You have provided an experiment list containing {} imageset(s).\n" "You have provided {} mask file(s).".format(n_expts, n_masks)) # Get the imageset imagesets = experiments.imagesets() for i, imageset in enumerate(imagesets): # Set the lookup with open(params.input.mask[i]) as f: mask = pickle.load(f) imageset.external_lookup.mask.filename = params.input.mask[i] imageset.external_lookup.mask.data = ImageBool(mask) # Dump the experiments print("Writing experiments to %s" % params.output.experiments) dump = ExperimentListDumper(experiments) dump.as_json(filename=params.output.experiments)
def test_combine_with_user_static_mask(dials_data, tmpdir): master_h5 = (dials_data("image_examples").join( "SACLA-MPCCD-Phase3-21528-5images.h5").strpath) assert FormatHDF5SaclaMPCCD.understand(master_h5) expts_from_filename = ExperimentListFactory.from_filenames([master_h5]) for i, expt in enumerate(expts_from_filename): mask = [] for panel in expt.detector: m = flex.bool(flex.grid(reversed(panel.get_image_size())), True) mask_untrusted_resolution_range(m, expt.beam, panel, 0, 2.23) mask.append(m) mask = tuple(mask) # exact number of masked pixels varies with wavelength between experiments assert [_.count(False) for _ in mask ] == pytest.approx([50643, 0, 0, 48003, 48356, 0, 0, 52191], abs=1e3) mask_file = tmpdir.join("pixel_%i.mask" % i) with mask_file.open("wb") as f: pickle.dump(mask, f) expt.imageset.external_lookup.mask.filename = mask_file.strpath expt.imageset.external_lookup.mask.data = ImageBool(mask) expts_from_dict = ExperimentListFactory.from_dict( expts_from_filename.to_dict()) imageset = expts_from_dict[0].imageset mask = imageset.get_mask(0) assert len(mask) == 8 assert [_.count(False) for _ in mask] == [ 65332, 24296, 24296, 63335, 63660, 24296, 24296, 66691, ] imageset.reader().nullify_format_instance()
def decode(self): """ Decode the dictionary into a list of experiments. """ # Extract all the experiments # Map of imageset/scan pairs imagesets = {} # For every experiment, use the given input to create # a sensible experiment. el = ExperimentList() for eobj in self._obj["experiment"]: # Get the models identifier = eobj.get("identifier", "") beam = self._lookup_model("beam", eobj) detector = self._lookup_model("detector", eobj) goniometer = self._lookup_model("goniometer", eobj) scan = self._lookup_model("scan", eobj) crystal = self._lookup_model("crystal", eobj) profile = self._lookup_model("profile", eobj) scaling_model = self._lookup_model("scaling_model", eobj) key = (eobj.get("imageset"), eobj.get("scan")) imageset = None try: imageset = imagesets[key] # type: ImageSet except KeyError: # This imageset hasn't been loaded yet - create it imageset_data = self._lookup_model("imageset", eobj) # Create the imageset from the input data if imageset_data is not None: if "params" in imageset_data: format_kwargs = imageset_data["params"] else: format_kwargs = {} # Load the external lookup data mask_filename, mask = self._load_pickle_path( imageset_data, "mask") gain_filename, gain = self._load_pickle_path( imageset_data, "gain") pedestal_filename, pedestal = self._load_pickle_path( imageset_data, "pedestal") dx_filename, dx = self._load_pickle_path( imageset_data, "dx") dy_filename, dy = self._load_pickle_path( imageset_data, "dy") if imageset_data["__id__"] == "ImageSet": imageset = self._make_stills( imageset_data, format_kwargs=format_kwargs) elif imageset_data["__id__"] == "ImageGrid": imageset = self._make_grid(imageset_data, format_kwargs=format_kwargs) elif imageset_data["__id__"] == "ImageSweep": imageset = self._make_sweep( imageset_data, beam=beam, detector=detector, goniometer=goniometer, scan=scan, format_kwargs=format_kwargs, ) elif imageset_data["__id__"] == "MemImageSet": imageset = self._make_mem_imageset(imageset_data) else: raise RuntimeError("Unknown imageset type") if imageset is not None: # Set the external lookup if mask is None: mask = ImageBool() else: mask = ImageBool(mask) if gain is None: gain = ImageDouble() else: gain = ImageDouble(gain) if pedestal is None: pedestal = ImageDouble() else: pedestal = ImageDouble(pedestal) if dx is None: dx = ImageDouble() else: dx = ImageDouble(dx) if dy is None: dy = ImageDouble() else: dy = ImageDouble(dy) imageset.external_lookup.mask.data = mask imageset.external_lookup.mask.filename = mask_filename imageset.external_lookup.gain.data = gain imageset.external_lookup.gain.filename = gain_filename imageset.external_lookup.pedestal.data = pedestal imageset.external_lookup.pedestal.filename = pedestal_filename imageset.external_lookup.dx.data = dx imageset.external_lookup.dx.filename = dx_filename imageset.external_lookup.dy.data = dy imageset.external_lookup.dy.filename = dy_filename # Update the imageset models if isinstance(imageset, ImageSweep): imageset.set_beam(beam) imageset.set_detector(detector) imageset.set_goniometer(goniometer) imageset.set_scan(scan) elif isinstance(imageset, (ImageSet, ImageGrid)): for i in range(len(imageset)): imageset.set_beam(beam, i) imageset.set_detector(detector, i) imageset.set_goniometer(goniometer, i) imageset.set_scan(scan, i) imageset.update_detector_px_mm_data() # Add the imageset to the dict - even if empty - as this will # prevent a duplicated attempt at reconstruction imagesets[key] = imageset # Append the experiment el.append( Experiment( imageset=imageset, beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, profile=profile, scaling_model=scaling_model, identifier=identifier, )) # Return the experiment list return el
def find_spots(self, experiments: ExperimentList) -> flex.reflection_table: """ Do spotfinding for a set of experiments. Args: experiments: The experiment list to process Returns: A new reflection table of found reflections """ # Loop through all the experiments and get the unique imagesets imagesets = [] for experiment in experiments: if experiment.imageset not in imagesets: imagesets.append(experiment.imageset) # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table() for j, imageset in enumerate(imagesets): # Find the strong spots in the sequence logger.info( "-" * 80 + "\nFinding strong spots in imageset %d\n" + "-" * 80, j) table, hot_mask = self._find_spots_in_imageset(imageset) # Fix up the experiment ID's now table["id"] = flex.int(table.nrows(), -1) for i, experiment in enumerate(experiments): if experiment.imageset is not imageset: continue if not self.is_stills and experiment.scan: z0, z1 = experiment.scan.get_array_range() z = table["xyzobs.px.value"].parts()[2] table["id"].set_selected((z > z0) & (z < z1), i) if experiment.identifier: table.experiment_identifiers( )[i] = experiment.identifier else: table["id"] = flex.int(table.nrows(), j) if experiment.identifier: table.experiment_identifiers( )[j] = experiment.identifier missed = table["id"] == -1 assert missed.count( True) == 0, "Failed to remap {} experiment IDs".format( missed.count(True)) reflections.extend(table) # Write a hot pixel mask if self.write_hot_mask: if not imageset.external_lookup.mask.data.empty(): for m1, m2 in zip(hot_mask, imageset.external_lookup.mask.data): m1 &= m2.data() imageset.external_lookup.mask.data = ImageBool(hot_mask) else: imageset.external_lookup.mask.data = ImageBool(hot_mask) imageset.external_lookup.mask.filename = "%s_%d.pickle" % ( self.hot_mask_prefix, i, ) # Write the hot mask with open(imageset.external_lookup.mask.filename, "wb") as outfile: pickle.dump(hot_mask, outfile, protocol=pickle.HIGHEST_PROTOCOL) # Set the strong spot flag reflections.set_flags(flex.size_t_range(len(reflections)), reflections.flags.strong) # Check for overloads reflections.is_overloaded(experiments) # Return the reflections return reflections
def _extract_experiments(self): ''' Helper function. Extract the experiments. ''' from dxtbx.imageset import ImageSweep, ImageSet, ImageGrid from dxtbx.serialize.filename import load_path import six.moves.cPickle as pickle from dxtbx.format.image import ImageBool, ImageDouble # Map of imageset/scan pairs imagesets = {} # For every experiment, use the given input to create # a sensible experiment. el = ExperimentList() for eobj in self._obj['experiment']: # Get the models identifier = eobj.get("identifier", "") beam = ExperimentListDict.model_or_none(self._blist, eobj, 'beam') detector = ExperimentListDict.model_or_none( self._dlist, eobj, 'detector') goniometer = ExperimentListDict.model_or_none( self._glist, eobj, 'goniometer') scan = ExperimentListDict.model_or_none(self._slist, eobj, 'scan') crystal = ExperimentListDict.model_or_none(self._clist, eobj, 'crystal') profile = ExperimentListDict.model_or_none(self._plist, eobj, 'profile') scaling_model = ExperimentListDict.model_or_none( self._scalelist, eobj, 'scaling_model') key = (eobj.get('imageset'), eobj.get('scan')) try: imageset = imagesets[key] except Exception: imageset = ExperimentListDict.model_or_none( self._ilist, eobj, 'imageset') # Create the imageset from the input data if imageset is not None: if 'params' in imageset: format_kwargs = imageset['params'] else: format_kwargs = {} if 'mask' in imageset and imageset['mask'] is not None: mask_filename = load_path(imageset['mask'], directory=self._directory) if self._check_format and mask_filename is not "": with open(mask_filename, 'rb') as fh: mask = pickle.load(fh) else: mask = None else: mask_filename = None mask = None if 'gain' in imageset and imageset['gain'] is not None: gain_filename = load_path(imageset['gain'], directory=self._directory) if self._check_format and gain_filename is not "": with open(gain_filename, 'rb') as fh: gain = pickle.load(fh) else: gain = None else: gain_filename = None gain = None if 'pedestal' in imageset and imageset[ 'pedestal'] is not None: pedestal_filename = load_path( imageset['pedestal'], directory=self._directory) if self._check_format and pedestal_filename is not "": with open(pedestal_filename, 'rb') as fh: pedestal = pickle.load(fh) else: pedestal = None else: pedestal_filename = None pedestal = None if 'dx' in imageset and imageset['dx'] is not None: dx_filename = load_path(imageset['dx'], directory=self._directory) if dx_filename is not "": with open(dx_filename, 'rb') as fh: dx = pickle.load(fh) else: dx = None else: dx_filename = None dx = None if 'dy' in imageset and imageset['dy'] is not None: dy_filename = load_path(imageset['dy'], directory=self._directory) if dy_filename is not "": with open(dy_filename, 'rb') as fh: dy = pickle.load(fh) else: dy = None else: dy_filename = None dy = None if imageset['__id__'] == 'ImageSet': imageset = self._make_stills( imageset, format_kwargs=format_kwargs) elif imageset['__id__'] == 'ImageGrid': imageset = self._make_grid(imageset, format_kwargs=format_kwargs) elif imageset['__id__'] == 'ImageSweep': imageset = self._make_sweep( imageset, beam=beam, detector=detector, goniometer=goniometer, scan=scan, format_kwargs=format_kwargs) elif imageset['__id__'] == 'MemImageSet': imageset = self._make_mem_imageset(imageset) else: raise RuntimeError('Unknown imageset type') # Set the external lookup if imageset is not None: if mask_filename is None: mask_filename = "" if gain_filename is None: gain_filename = "" if pedestal_filename is None: pedestal_filename = "" if dx_filename is None: dx_filename = "" if dy_filename is None: dy_filename = "" if mask is None: mask = ImageBool() else: mask = ImageBool(mask) if gain is None: gain = ImageDouble() else: gain = ImageDouble(gain) if pedestal is None: pedestal = ImageDouble() else: pedestal = ImageDouble(pedestal) if dx is None: dx = ImageDouble() else: dx = ImageDouble(dx) if dy is None: dy = ImageDouble() else: dy = ImageDouble(dy) imageset.external_lookup.mask.data = mask imageset.external_lookup.mask.filename = mask_filename imageset.external_lookup.gain.data = gain imageset.external_lookup.gain.filename = gain_filename imageset.external_lookup.pedestal.data = pedestal imageset.external_lookup.pedestal.filename = pedestal_filename imageset.external_lookup.dx.data = dx imageset.external_lookup.dx.filename = dx_filename imageset.external_lookup.dy.data = dy imageset.external_lookup.dy.filename = dy_filename # Update the imageset models if isinstance(imageset, ImageSweep): imageset.set_beam(beam) imageset.set_detector(detector) imageset.set_goniometer(goniometer) imageset.set_scan(scan) elif isinstance(imageset, ImageSet): for i in range(len(imageset)): imageset.set_beam(beam, i) imageset.set_detector(detector, i) imageset.set_goniometer(goniometer, i) imageset.set_scan(scan, i) elif isinstance(imageset, ImageGrid): for i in range(len(imageset)): imageset.set_beam(beam, i) imageset.set_detector(detector, i) imageset.set_goniometer(goniometer, i) imageset.set_scan(scan, i) else: pass if imageset is not None: imageset.update_detector_px_mm_data() # Add the imageset to the dict imagesets[key] = imageset # Append the experiment el.append( Experiment(imageset=imageset, beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, profile=profile, scaling_model=scaling_model, identifier=identifier)) # Return the experiment list return el
def run(self): from dxtbx.imageset import ImageSetData from dxtbx.format.image import ImageTileBool from dxtbx.format.image import ImageTileDouble from dxtbx.format.image import ImageBool from dxtbx.format.image import ImageDouble from scitbx.array_family import flex import os.path from glob import glob from dxtbx.format.FormatCBFMiniPilatus import FormatCBFMiniPilatus as FormatClass from os.path import join dials_regression = libtbx.env.dist_path('dials_regression') filenames = sorted( glob(join(dials_regression, "centroid_test_data", "*.cbf"))) ReaderClass = FormatClass.get_reader() MaskerClass = FormatClass.get_masker() reader = ReaderClass(filenames) masker = MaskerClass(filenames) handle = ImageSetData(reader, masker) data = handle.get_data(0).as_int().tile(0).data() mask = handle.get_mask(0).tile(0).data() assert handle.has_single_file_reader() == False path = handle.get_path(0) assert path == filenames[0] master_path = handle.get_master_path() assert master_path == "" identifier = handle.get_image_identifier(0) assert identifier == filenames[0] beam = FormatClass(filenames[0]).get_beam() detector = FormatClass(filenames[0]).get_detector() goniometer = FormatClass(filenames[0]).get_goniometer() scan = FormatClass(filenames[0]).get_scan() handle.set_beam(beam, 0) handle.set_detector(detector, 0) handle.set_goniometer(goniometer, 0) handle.set_scan(scan, 0) beam2 = handle.get_beam(0) detector2 = handle.get_detector(0) goniometer2 = handle.get_goniometer(0) scan2 = handle.get_scan(0) assert beam2 == beam assert detector2 == detector assert goniometer2 == goniometer assert scan2 == scan mask = flex.bool(flex.grid(10, 10), True) gain = flex.double(flex.grid(10, 10), 1) pedestal = flex.double(flex.grid(10, 10), 2) handle.external_lookup.mask.data = ImageBool(ImageTileBool(mask)) handle.external_lookup.gain.data = ImageDouble(ImageTileDouble(gain)) handle.external_lookup.pedestal.data = ImageDouble( ImageTileDouble(pedestal)) mask2 = handle.external_lookup.mask.data.tile(0).data() gain2 = handle.external_lookup.gain.data.tile(0).data() pedestal2 = handle.external_lookup.pedestal.data.tile(0).data() assert mask2.all_eq(mask) assert gain2.all_eq(gain) assert pedestal2.all_eq(pedestal) print 'OK'
def _load_datablocks(self, obj, check_format=True, directory=None): ''' Create the datablock from a dictionary. ''' from dxtbx.format.Registry import Registry from dxtbx.model import BeamFactory, DetectorFactory from dxtbx.model import GoniometerFactory, ScanFactory from dxtbx.serialize.filename import load_path from dxtbx.format.image import ImageBool, ImageDouble from dxtbx.format.FormatMultiImage import FormatMultiImage # If we have a list, extract for each dictionary in the list if isinstance(obj, list): return [self._load_datablocks(dd, check_format, directory) for dd in obj] elif not isinstance(obj, dict): raise InvalidDataBlockError("Unexpected datablock type {} instead of dict".format(type(obj))) # Make sure the id signature is correct if not obj.get("__id__") == "DataBlock": raise InvalidDataBlockError( "Expected __id__ 'DataBlock', but found {}".format(repr(obj.get("__id__")))) # Get the list of models blist = obj.get('beam', []) dlist = obj.get('detector', []) glist = obj.get('goniometer', []) slist = obj.get('scan', []) def load_models(obj): try: beam = BeamFactory.from_dict(blist[obj['beam']]) except Exception: beam = None try: dobj = dlist[obj['detector']] detector = DetectorFactory.from_dict(dobj) except Exception: detector = None try: gonio = GoniometerFactory.from_dict(glist[obj['goniometer']]) except Exception: gonio = None try: scan = ScanFactory.from_dict(slist[obj['scan']]) except Exception: scan = None return beam, detector, gonio, scan # Loop through all the imagesets imagesets = [] for imageset in obj['imageset']: ident = imageset['__id__'] if "params" in imageset: format_kwargs = imageset['params'] else: format_kwargs = {} if ident == 'ImageSweep': beam, detector, gonio, scan = load_models(imageset) if "template" in imageset: template = load_path(imageset['template'], directory=directory) i0, i1 = scan.get_image_range() iset = dxtbx.imageset.ImageSetFactory.make_sweep( template, range(i0, i1+1), None, beam, detector, gonio, scan, check_format, format_kwargs=format_kwargs) if 'mask' in imageset and imageset['mask'] is not None: imageset['mask'] = load_path(imageset['mask'], directory=directory) iset.external_lookup.mask.filename = imageset['mask'] if check_format: with open(imageset['mask']) as infile: iset.external_lookup.mask.data = ImageBool(pickle.load(infile)) if 'gain' in imageset and imageset['gain'] is not None: imageset['gain'] = load_path(imageset['gain'], directory=directory) iset.external_lookup.gain.filename = imageset['gain'] if check_format: with open(imageset['gain']) as infile: iset.external_lookup.gain.data = ImageDouble(pickle.load(infile)) if 'pedestal' in imageset and imageset['pedestal'] is not None: imageset['pedestal'] = load_path(imageset['pedestal'], directory=directory) iset.external_lookup.pedestal.filename = imageset['pedestal'] if check_format: with open(imageset['pedestal']) as infile: iset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile)) if 'dx' in imageset and imageset['dx'] is not None: imageset['dx'] = load_path(imageset['dx'], directory=directory) iset.external_lookup.dx.filename = imageset['dx'] with open(imageset['dx']) as infile: iset.external_lookup.dx.data = ImageDouble(pickle.load(infile)) if 'dy' in imageset and imageset['dy'] is not None: imageset['dy'] = load_path(imageset['dy'], directory=directory) iset.external_lookup.dy.filename = imageset['dy'] with open(imageset['dy']) as infile: iset.external_lookup.dy.data = ImageDouble(pickle.load(infile)) iset.update_detector_px_mm_data() elif "master" in imageset: template = load_path(imageset['master'], directory=directory) i0, i1 = scan.get_image_range() indices = imageset['images'] if check_format == False: format_class = FormatMultiImage else: format_class = None iset = dxtbx.imageset.ImageSetFactory.make_sweep( template, list(range(i0, i1+1)), format_class = format_class, beam = beam, detector = detector, goniometer = gonio, scan = scan, check_format = check_format, format_kwargs = format_kwargs) if 'mask' in imageset and imageset['mask'] is not None: imageset['mask'] = load_path(imageset['mask'], directory) iset.external_lookup.mask.filename = imageset['mask'] if check_format: with open(imageset['mask']) as infile: iset.external_lookup.mask.data = ImageBool(pickle.load(infile)) if 'gain' in imageset and imageset['gain'] is not None: imageset['gain'] = load_path(imageset['gain'], directory) iset.external_lookup.gain.filename = imageset['gain'] if check_format: with open(imageset['gain']) as infile: iset.external_lookup.gain.data = ImageDouble(pickle.load(infile)) if 'pedestal' in imageset and imageset['pedestal'] is not None: imageset['pedestal'] = load_path(imageset['pedestal'], directory) iset.external_lookup.pedestal.filename = imageset['pedestal'] if check_format: with open(imageset['pedestal']) as infile: iset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile)) if 'dx' in imageset and imageset['dx'] is not None: imageset['dx'] = load_path(imageset['dx'], directory) iset.external_lookup.dx.filename = imageset['dx'] with open(imageset['dx']) as infile: iset.external_lookup.dx.data = ImageDouble(pickle.load(infile)) if 'dy' in imageset and imageset['dy'] is not None: imageset['dy'] = load_path(imageset['dy'], directory) iset.external_lookup.dy.filename = imageset['dy'] with open(imageset['dy']) as infile: iset.external_lookup.dy.data = ImageDouble(pickle.load(infile)) iset.update_detector_px_mm_data() imagesets.append(iset) elif ident == 'ImageSet' or ident == "ImageGrid": filenames = [image['filename'] for image in imageset['images']] indices = [image['image'] for image in imageset['images'] if 'image' in image] assert len(indices) == 0 or len(indices) == len(filenames) iset = dxtbx.imageset.ImageSetFactory.make_imageset( filenames, None, check_format, indices, format_kwargs=format_kwargs) if ident == "ImageGrid": grid_size = imageset['grid_size'] iset = dxtbx.imageset.ImageGrid.from_imageset(iset, grid_size) for i, image in enumerate(imageset['images']): beam, detector, gonio, scan = load_models(image) iset.set_beam(beam, i) iset.set_detector(detector, i) iset.set_goniometer(gonio, i) iset.set_scan(scan, i) if 'mask' in imageset and imageset['mask'] is not None: imageset['mask'] = load_path(imageset['mask'], directory) iset.external_lookup.mask.filename = imageset['mask'] if check_format: with open(imageset['mask']) as infile: iset.external_lookup.mask.data = ImageBool(pickle.load(infile)) if 'gain' in imageset and imageset['gain'] is not None: imageset['gain'] = load_path(imageset['gain'], directory) iset.external_lookup.gain.filename = imageset['gain'] if check_format: with open(imageset['gain']) as infile: iset.external_lookup.gain.data = ImageDouble(pickle.load(infile)) if 'pedestal' in imageset and imageset['pedestal'] is not None: imageset['pedestal'] = load_path(imageset['pedestal'], directory) iset.external_lookup.pedestal.filename = imageset['pedestal'] if check_format: with open(imageset['pedestal']) as infile: iset.external_lookup.pedestal.data = ImageDouble(pickle.load(infile)) if 'dx' in imageset and imageset['dx'] is not None: imageset['dx'] = load_path(imageset['dx'], directory) iset.external_lookup.dx.filename = imageset['dx'] with open(imageset['dx']) as infile: iset.external_lookup.dx.data = ImageDouble(pickle.load(infile)) if 'dy' in imageset and imageset['dy'] is not None: imageset['dy'] = load_path(imageset['dy'], directory) iset.external_lookup.dy.filename = imageset['dy'] with open(imageset['dy']) as infile: iset.external_lookup.dy.data = ImageDouble(pickle.load(infile)) iset.update_detector_px_mm_data() imagesets.append(iset) else: raise RuntimeError('expected ImageSet/ImageSweep, got %s' % ident) # Return the datablock return DataBlock(imagesets)
def datablocks_from_dict(obj, check_format=True, directory=None): """Get the datablocks from the dictionary.""" # If we have a list, extract for each dictionary in the list if isinstance(obj, list): return [ datablocks_from_dict(dd, check_format, directory) for dd in obj ] elif not isinstance(obj, dict): raise InvalidDataBlockError( "Unexpected datablock type {} instead of dict".format(type(obj))) # Make sure the id signature is correct if not obj.get("__id__") == "DataBlock": raise InvalidDataBlockError( "Expected __id__ 'DataBlock', but found {}".format( repr(obj.get("__id__")))) # Get the list of models blist = obj.get("beam", []) dlist = obj.get("detector", []) glist = obj.get("goniometer", []) slist = obj.get("scan", []) def load_models(obj): try: beam = dxtbx.model.BeamFactory.from_dict(blist[obj["beam"]]) except Exception: beam = None try: dobj = dlist[obj["detector"]] detector = dxtbx.model.DetectorFactory.from_dict(dobj) except Exception: detector = None try: gonio = dxtbx.model.GoniometerFactory.from_dict( glist[obj["goniometer"]]) except Exception: gonio = None try: scan = dxtbx.model.ScanFactory.from_dict(slist[obj["scan"]]) except Exception: scan = None return beam, detector, gonio, scan # Loop through all the imagesets imagesets = [] for imageset in obj["imageset"]: ident = imageset["__id__"] if "params" in imageset: format_kwargs = imageset["params"] else: format_kwargs = {} if ident == "ImageSequence" or ident == "ImageSweep": beam, detector, gonio, scan = load_models(imageset) if "template" in imageset: template = resolve_path(imageset["template"], directory=directory) i0, i1 = scan.get_image_range() iset = dxtbx.imageset.ImageSetFactory.make_sequence( template, list(range(i0, i1 + 1)), None, beam, detector, gonio, scan, check_format, format_kwargs=format_kwargs, ) if "mask" in imageset and imageset["mask"] is not None: imageset["mask"] = resolve_path(imageset["mask"], directory=directory) iset.external_lookup.mask.filename = imageset["mask"] if check_format: with open(imageset["mask"], "rb") as infile: iset.external_lookup.mask.data = ImageBool( pickle.load(infile, encoding="bytes")) if "gain" in imageset and imageset["gain"] is not None: imageset["gain"] = resolve_path(imageset["gain"], directory=directory) iset.external_lookup.gain.filename = imageset["gain"] if check_format: with open(imageset["gain"], "rb") as infile: iset.external_lookup.gain.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "pedestal" in imageset and imageset["pedestal"] is not None: imageset["pedestal"] = resolve_path(imageset["pedestal"], directory=directory) iset.external_lookup.pedestal.filename = imageset[ "pedestal"] if check_format: with open(imageset["pedestal"], "rb") as infile: iset.external_lookup.pedestal.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dx" in imageset and imageset["dx"] is not None: imageset["dx"] = resolve_path(imageset["dx"], directory=directory) iset.external_lookup.dx.filename = imageset["dx"] with open(imageset["dx"], "rb") as infile: iset.external_lookup.dx.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dy" in imageset and imageset["dy"] is not None: imageset["dy"] = resolve_path(imageset["dy"], directory=directory) iset.external_lookup.dy.filename = imageset["dy"] with open(imageset["dy"], "rb") as infile: iset.external_lookup.dy.data = ImageDouble( pickle.load(infile, encoding="bytes")) iset.update_detector_px_mm_data() elif "master" in imageset: template = resolve_path(imageset["master"], directory=directory) i0, i1 = scan.get_image_range() if not check_format: format_class = FormatMultiImage else: format_class = None iset = dxtbx.imageset.ImageSetFactory.make_sequence( template, list(range(i0, i1 + 1)), format_class=format_class, beam=beam, detector=detector, goniometer=gonio, scan=scan, check_format=check_format, format_kwargs=format_kwargs, ) if "mask" in imageset and imageset["mask"] is not None: imageset["mask"] = resolve_path(imageset["mask"], directory) iset.external_lookup.mask.filename = imageset["mask"] if check_format: with open(imageset["mask"], "rb") as infile: iset.external_lookup.mask.data = ImageBool( pickle.load(infile, encoding="bytes")) if "gain" in imageset and imageset["gain"] is not None: imageset["gain"] = resolve_path(imageset["gain"], directory) iset.external_lookup.gain.filename = imageset["gain"] if check_format: with open(imageset["gain"], "rb") as infile: iset.external_lookup.gain.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "pedestal" in imageset and imageset["pedestal"] is not None: imageset["pedestal"] = resolve_path( imageset["pedestal"], directory) iset.external_lookup.pedestal.filename = imageset[ "pedestal"] if check_format: with open(imageset["pedestal"], "rb") as infile: iset.external_lookup.pedestal.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dx" in imageset and imageset["dx"] is not None: imageset["dx"] = resolve_path(imageset["dx"], directory) iset.external_lookup.dx.filename = imageset["dx"] with open(imageset["dx"], "rb") as infile: iset.external_lookup.dx.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dy" in imageset and imageset["dy"] is not None: imageset["dy"] = resolve_path(imageset["dy"], directory) iset.external_lookup.dy.filename = imageset["dy"] with open(imageset["dy"], "rb") as infile: iset.external_lookup.dy.data = ImageDouble( pickle.load(infile, encoding="bytes")) iset.update_detector_px_mm_data() imagesets.append(iset) elif ident == "ImageSet" or ident == "ImageGrid": filenames = [image["filename"] for image in imageset["images"]] indices = [ image["image"] for image in imageset["images"] if "image" in image ] assert len(indices) == 0 or len(indices) == len(filenames) iset = dxtbx.imageset.ImageSetFactory.make_imageset( filenames, None, check_format, indices, format_kwargs=format_kwargs) if ident == "ImageGrid": grid_size = imageset["grid_size"] iset = dxtbx.imageset.ImageGrid.from_imageset(iset, grid_size) for i, image in enumerate(imageset["images"]): beam, detector, gonio, scan = load_models(image) iset.set_beam(beam, i) iset.set_detector(detector, i) iset.set_goniometer(gonio, i) iset.set_scan(scan, i) if "mask" in imageset and imageset["mask"] is not None: imageset["mask"] = resolve_path(imageset["mask"], directory) iset.external_lookup.mask.filename = imageset["mask"] if check_format: with open(imageset["mask"], "rb") as infile: iset.external_lookup.mask.data = ImageBool( pickle.load(infile, encoding="bytes")) if "gain" in imageset and imageset["gain"] is not None: imageset["gain"] = resolve_path(imageset["gain"], directory) iset.external_lookup.gain.filename = imageset["gain"] if check_format: with open(imageset["gain"], "rb") as infile: iset.external_lookup.gain.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "pedestal" in imageset and imageset["pedestal"] is not None: imageset["pedestal"] = resolve_path(imageset["pedestal"], directory) iset.external_lookup.pedestal.filename = imageset["pedestal"] if check_format: with open(imageset["pedestal"], "rb") as infile: iset.external_lookup.pedestal.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dx" in imageset and imageset["dx"] is not None: imageset["dx"] = resolve_path(imageset["dx"], directory) iset.external_lookup.dx.filename = imageset["dx"] with open(imageset["dx"], "rb") as infile: iset.external_lookup.dx.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dy" in imageset and imageset["dy"] is not None: imageset["dy"] = resolve_path(imageset["dy"], directory) iset.external_lookup.dy.filename = imageset["dy"] with open(imageset["dy"], "rb") as infile: iset.external_lookup.dy.data = ImageDouble( pickle.load(infile, encoding="bytes")) iset.update_detector_px_mm_data() imagesets.append(iset) else: raise RuntimeError("expected ImageSet/ImageSequence, got %s" % ident) return DataBlock(imagesets)
def __call__(self, experiments): """ Do the spot finding. :param experiments: The experiments to process :return: The observed spots """ import six.moves.cPickle as pickle from dxtbx.format.image import ImageBool # Loop through all the experiments and get the unique imagesets imagesets = [] for experiment in experiments: if experiment.imageset not in imagesets: imagesets.append(experiment.imageset) # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table() for j, imageset in enumerate(imagesets): # Find the strong spots in the sequence logger.info("-" * 80) logger.info("Finding strong spots in imageset %d" % j) logger.info("-" * 80) logger.info("") table, hot_mask = self._find_spots_in_imageset(imageset) # Fix up the experiment ID's now table["id"] = flex.int(table.nrows(), -1) for i, experiment in enumerate(experiments): if experiment.imageset is not imageset: continue if experiment.scan: z0, z1 = experiment.scan.get_array_range() z = table["xyzobs.px.value"].parts()[2] table["id"].set_selected((z > z0) & (z < z1), i) if experiment.identifier: table.experiment_identifiers()[i] = experiment.identifier else: table["id"] = flex.int(table.nrows(), j) if experiment.identifier: table.experiment_identifiers()[j] = experiment.identifier missed = table["id"] == -1 assert missed.count(True) == 0, missed.count(True) reflections.extend(table) # Write a hot pixel mask if self.write_hot_mask: if not imageset.external_lookup.mask.data.empty(): for m1, m2 in zip(hot_mask, imageset.external_lookup.mask.data): m1 &= m2.data() imageset.external_lookup.mask.data = ImageBool(hot_mask) else: imageset.external_lookup.mask.data = ImageBool(hot_mask) imageset.external_lookup.mask.filename = "%s_%d.pickle" % ( self.hot_mask_prefix, i, ) # Write the hot mask with open(imageset.external_lookup.mask.filename, "wb") as outfile: pickle.dump(hot_mask, outfile, protocol=pickle.HIGHEST_PROTOCOL) # Set the strong spot flag reflections.set_flags( flex.size_t_range(len(reflections)), reflections.flags.strong ) # Check for overloads reflections.is_overloaded(experiments) # Return the reflections return reflections
def get_imageset( cls, filenames, beam=None, detector=None, goniometer=None, scan=None, as_sequence=False, as_imageset=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, lazy=False, ): """ Factory method to create an imageset """ if isinstance(filenames, str): filenames = [filenames] elif len(filenames) > 1: assert len(set(filenames)) == 1 filenames = filenames[0:1] # Make filenames absolute filenames = [os.path.abspath(x) for x in filenames] # Make it a dictionary if format_kwargs is None: format_kwargs = {} # If get_num_images hasn't been implemented, we need indices for number of images if cls.get_num_images == FormatMultiImage.get_num_images: assert single_file_indices is not None assert min(single_file_indices) >= 0 num_images = max(single_file_indices) + 1 else: num_images = None # Get the format instance assert len(filenames) == 1 if check_format is True: format_instance = cls(filenames[0], **format_kwargs) if num_images is None and not lazy: # As we now have the actual format class we can get the number # of images from here. This saves having to create another # format class instance in the Reader() constructor # NOTE: Having this information breaks internal assumptions in # *Lazy classes, so they have to figure this out in # their own time. num_images = format_instance.get_num_images() else: format_instance = None if not as_sequence: lazy = True # Get some information from the format class reader = cls.get_reader()(filenames, num_images=num_images, **format_kwargs) # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Check if we have a sequence # Make sure only 1 or none is set assert [as_imageset, as_sequence].count(True) < 2 if as_imageset: is_sequence = False elif as_sequence: is_sequence = True else: if scan is None and format_instance is None: raise RuntimeError( """ One of the following needs to be set - as_imageset=True - as_sequence=True - scan - check_format=True """ ) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None: is_sequence = True else: is_sequence = False assert not (as_sequence and lazy), "No lazy support for sequences" if single_file_indices is not None: single_file_indices = flex.size_t(single_file_indices) # Create an imageset or sequence if not is_sequence: # Use imagesetlazy # Setup ImageSetLazy and just return it. No models are set. if lazy: iset = ImageSetLazy( ImageSetData( reader=reader, masker=None, vendor=vendor, params=params, format=cls, ), indices=single_file_indices, ) return iset # Create the imageset iset = ImageSet( ImageSetData( reader=reader, masker=None, vendor=vendor, params=params, format=cls ), indices=single_file_indices, ) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for i in range(format_instance.get_num_images()): beam.append(format_instance.get_beam(i)) detector.append(format_instance.get_detector(i)) goniometer.append(format_instance.get_goniometer(i)) scan.append(format_instance.get_scan(i)) if single_file_indices is None: single_file_indices = list(range(format_instance.get_num_images())) # Set the list of models for i in range(len(single_file_indices)): iset.set_beam(beam[single_file_indices[i]], i) iset.set_detector(detector[single_file_indices[i]], i) iset.set_goniometer(goniometer[single_file_indices[i]], i) iset.set_scan(scan[single_file_indices[i]], i) else: # Get the template template = filenames[0] # Check indices are sequential if single_file_indices is not None: assert all( i + 1 == j for i, j in zip(single_file_indices[:-1], single_file_indices[1:]) ) num_images = len(single_file_indices) else: num_images = format_instance.get_num_images() # Check the scan makes sense - we must want to use <= total images if scan is not None: assert scan.get_num_images() <= num_images # If any are None then read from format if beam is None: beam = format_instance.get_beam() if detector is None: detector = format_instance.get_detector() if goniometer is None: goniometer = format_instance.get_goniometer() if scan is None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = cls(f, **format_kwargs) scan += format_instance.get_scan() # Create the masker if format_instance is not None: masker = format_instance.get_masker(goniometer=goniometer) else: masker = None isetdata = ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=cls, template=template, ) # Create the sequence iset = ImageSequence( isetdata, beam=beam, detector=detector, goniometer=goniometer, scan=scan, indices=single_file_indices, ) if format_instance is not None: static_mask = format_instance.get_static_mask() if static_mask is not None: if not iset.external_lookup.mask.data.empty(): for m1, m2 in zip(static_mask, iset.external_lookup.mask.data): m1 &= m2.data() iset.external_lookup.mask.data = ImageBool(static_mask) else: iset.external_lookup.mask.data = ImageBool(static_mask) return iset
def _imageset_from_imageset_data(self, imageset_data, models): """ Make an imageset from imageset_data - help with refactor decode. """ assert imageset_data is not None if "params" in imageset_data: format_kwargs = imageset_data["params"] else: format_kwargs = {} beam = models["beam"] detector = models["detector"] goniometer = models["goniometer"] scan = models["scan"] # Load the external lookup data mask_filename, mask = self._load_pickle_path(imageset_data, "mask") gain_filename, gain = self._load_pickle_path(imageset_data, "gain") pedestal_filename, pedestal = self._load_pickle_path(imageset_data, "pedestal") dx_filename, dx = self._load_pickle_path(imageset_data, "dx") dy_filename, dy = self._load_pickle_path(imageset_data, "dy") if imageset_data["__id__"] == "ImageSet": imageset = self._make_stills(imageset_data, format_kwargs=format_kwargs) elif imageset_data["__id__"] == "ImageGrid": imageset = self._make_grid(imageset_data, format_kwargs=format_kwargs) elif ( imageset_data["__id__"] == "ImageSequence" or imageset_data["__id__"] == "ImageSweep" ): imageset = self._make_sequence( imageset_data, beam=beam, detector=detector, goniometer=goniometer, scan=scan, format_kwargs=format_kwargs, ) elif imageset_data["__id__"] == "MemImageSet": imageset = self._make_mem_imageset(imageset_data) else: raise RuntimeError("Unknown imageset type") if imageset is not None: # Set the external lookup if mask is None: mask = ImageBool() else: mask = ImageBool(mask) if gain is None: gain = ImageDouble() else: gain = ImageDouble(gain) if pedestal is None: pedestal = ImageDouble() else: pedestal = ImageDouble(pedestal) if dx is None: dx = ImageDouble() else: dx = ImageDouble(dx) if dy is None: dy = ImageDouble() else: dy = ImageDouble(dy) if not imageset.external_lookup.mask.data.empty(): if not mask.empty(): mask = tuple(m.data() for m in mask) for m1, m2 in zip(mask, imageset.external_lookup.mask.data): m1 &= m2.data() imageset.external_lookup.mask.data = ImageBool(mask) else: imageset.external_lookup.mask.data = mask imageset.external_lookup.mask.filename = mask_filename imageset.external_lookup.gain.data = gain imageset.external_lookup.gain.filename = gain_filename imageset.external_lookup.pedestal.data = pedestal imageset.external_lookup.pedestal.filename = pedestal_filename imageset.external_lookup.dx.data = dx imageset.external_lookup.dx.filename = dx_filename imageset.external_lookup.dy.data = dy imageset.external_lookup.dy.filename = dy_filename # Update the imageset models if isinstance(imageset, ImageSequence): imageset.set_beam(beam) imageset.set_detector(detector) imageset.set_goniometer(goniometer) imageset.set_scan(scan) elif isinstance(imageset, (ImageSet, ImageGrid)): for i in range(len(imageset)): imageset.set_beam(beam, i) imageset.set_detector(detector, i) imageset.set_goniometer(goniometer, i) imageset.set_scan(scan, i) imageset.update_detector_px_mm_data() return imageset
def get_imageset( Class, input_filenames, beam=None, detector=None, goniometer=None, scan=None, as_imageset=False, as_sequence=False, single_file_indices=None, format_kwargs=None, template=None, check_format=True, ): """ Factory method to create an imageset """ # Import here to avoid cyclic imports from dxtbx.imageset import ImageSequence, ImageSet, ImageSetData # Get filename absolute paths, for entries that are filenames filenames = [ os.path.abspath(x) if not get_url_scheme(x) else x for x in input_filenames ] # Make it a dict if format_kwargs is None: format_kwargs = {} # Get some information from the format class reader = Class.get_reader()(filenames, **format_kwargs) # Get the format instance if check_format is True: Class._current_filename_ = None Class._current_instance_ = None format_instance = Class.get_instance(filenames[0], **format_kwargs) else: format_instance = None # Read the vendor type if check_format is True: vendor = format_instance.get_vendortype() else: vendor = "" # Get the format kwargs params = format_kwargs # Make sure only 1 or none is set assert [as_imageset, as_sequence].count(True) < 2 if as_imageset: is_sequence = False elif as_sequence: is_sequence = True else: if scan is None and format_instance is None: raise RuntimeError(""" One of the following needs to be set - as_imageset=True - as_sequence=True - scan - check_format=True """) if scan is None: test_scan = format_instance.get_scan() else: test_scan = scan if test_scan is not None and test_scan.get_oscillation()[1] != 0: is_sequence = True else: is_sequence = False # Create an imageset or sequence if not is_sequence: # Create the imageset iset = ImageSet( ImageSetData( reader=reader, masker=None, vendor=vendor, params=params, format=Class, )) # If any are None then read from format if [beam, detector, goniometer, scan].count(None) != 0: # Get list of models beam = [] detector = [] goniometer = [] scan = [] for f in filenames: format_instance = Class(f, **format_kwargs) beam.append(format_instance.get_beam()) detector.append(format_instance.get_detector()) goniometer.append(format_instance.get_goniometer()) scan.append(format_instance.get_scan()) # Set the list of models for i in range(len(filenames)): iset.set_beam(beam[i], i) iset.set_detector(detector[i], i) iset.set_goniometer(goniometer[i], i) iset.set_scan(scan[i], i) else: # Get the template if template is None: template = template_regex(filenames[0])[0] else: template = str(template) # Check scan makes sense if scan: if check_format is True: assert scan.get_num_images() == len(filenames) # If any are None then read from format if beam is None and format_instance is not None: beam = format_instance.get_beam() if detector is None and format_instance is not None: detector = format_instance.get_detector() if goniometer is None and format_instance is not None: goniometer = format_instance.get_goniometer() if scan is None and format_instance is not None: scan = format_instance.get_scan() if scan is not None: for f in filenames[1:]: format_instance = Class(f, **format_kwargs) scan += format_instance.get_scan() assert beam is not None, "Can't create Sequence without beam" assert detector is not None, "Can't create Sequence without detector" assert goniometer is not None, "Can't create Sequence without goniometer" assert scan is not None, "Can't create Sequence without scan" # Create the masker if format_instance is not None: masker = format_instance.get_masker(goniometer=goniometer) else: masker = None # Create the sequence iset = ImageSequence( ImageSetData( reader=reader, masker=masker, vendor=vendor, params=params, format=Class, template=template, ), beam=beam, detector=detector, goniometer=goniometer, scan=scan, ) if format_instance is not None: static_mask = format_instance.get_static_mask() if static_mask is not None: if not iset.external_lookup.mask.data.empty(): for m1, m2 in zip(static_mask, iset.external_lookup.mask.data): m1 &= m2.data() iset.external_lookup.mask.data = ImageBool(static_mask) else: iset.external_lookup.mask.data = ImageBool(static_mask) return iset
def generate_mask( experiments: ExperimentList, params: phil.scope_extract, ) -> Tuple[Masks, Optional[ExperimentList]]: """ Generate a pixel mask for each imageset in an experiment list. Use the masking parameters :param:`params` and the experiments in the experiment list :param:`experiments` to define a pixel mask for each of the associated imagesets. The masks are generated using :mod:`dials.util.masking`. The masks will be saved to disk at the location specified by :attr:`params.output.mask`. If the experiment list contains more than one imageset, multiple mask files will be produced, with filenames differentiated by an appended number. Optionally, if a path :attr:`params.output.experiments` is set, a modified copy of :param:`experiments` with the masks applied will be saved to that location. Args: experiments: An experiment list containing only one imageset. params: Masking parameters, having the structure defined in :data:`phil_scope`. Returns: A list of masks, one for each imageset. A copy of :param:`experiments` with the masks applied (optional, only returned if :attr:`params.output.experiments` is set). """ imagesets = experiments.imagesets() masks = [] # Create output mask filenames num_imagesets = len(imagesets) if num_imagesets == 1: filenames = [params.output.mask] else: # If there is more than one imageset, append a number to each output filename name, ext = os.path.splitext(params.output.mask) pad = len(str(num_imagesets)) filenames = [ "{name}_{num:0{pad}}{ext}".format(name=name, num=i + 1, pad=pad, ext=ext) for i in range(num_imagesets) ] for imageset, filename in zip(imagesets, filenames): mask = dials.util.masking.generate_mask(imageset, params) masks.append(mask) # Save the mask to file log.info("Writing mask to %s", filename) with open(filename, "wb") as fh: pickle.dump(mask, fh) if params.output.experiments: # Apply the mask to the imageset imageset.external_lookup.mask.data = ImageBool(mask) imageset.external_lookup.mask.filename = filename if params.output.experiments: # Save the experiment list log.info("Saving experiments to %s", params.output.experiments) experiments.as_file(params.output.experiments) else: experiments = None return masks, experiments