def run(self): ''' Run the script. ''' from dials.util.masking import MaskGenerator from dials.util.options import flatten_datablocks from libtbx.utils import Sorry import cPickle as pickle from dials.util import log # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) datablocks = flatten_datablocks(params.input.datablock) # COnfigu logging log.config() # Check number of args if len(datablocks) == 0: self.parser.print_help() return if len(datablocks) != 1: raise Sorry('exactly 1 datablock must be specified') datablock = datablocks[0] imagesets = datablock.extract_imagesets() if len(imagesets) != 1: raise Sorry('datablock must contain exactly 1 imageset') imageset = imagesets[0] # Generate the mask generator = MaskGenerator(params) mask = generator.generate(imageset) # Save the mask to file print "Writing mask to %s" % params.output.mask pickle.dump(mask, open(params.output.mask, "w"))
def UpdateMask(self): image_viewer_frame = self.GetParent().GetParent() # Generate the mask from dials.util.masking import MaskGenerator generator = MaskGenerator(self.params.masking) imageset = image_viewer_frame.imagesets[0] # XXX mask = generator.generate(imageset) image_viewer_frame.mask_image_viewer = mask image_viewer_frame.update_settings(layout=False)
def run(self): ''' Run the script. ''' from dials.util.masking import MaskGenerator from dials.util.options import flatten_datablocks from libtbx.utils import Sorry import six.moves.cPickle as pickle from dials.util import log from dxtbx.format.image import ImageBool # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) datablocks = flatten_datablocks(params.input.datablock) # Configure logging log.config() # Check number of args if len(datablocks) == 0: self.parser.print_help() return if len(datablocks) != 1: raise Sorry('exactly 1 datablock must be specified') datablock = datablocks[0] imagesets = datablock.extract_imagesets() if len(imagesets) != 1: raise Sorry('datablock must contain exactly 1 imageset') imageset = imagesets[0] # Generate the mask generator = MaskGenerator(params) mask = generator.generate(imageset) # Save the mask to file print("Writing mask to %s" % params.output.mask) with open(params.output.mask, "wb") as fh: pickle.dump(mask, fh) # Save the datablock if params.output.datablock is not None: imageset.external_lookup.mask.data = ImageBool(mask) imageset.external_lookup.mask.filename = params.output.mask from dxtbx.datablock import DataBlockDumper print('Saving datablocks to {0}'.format(params.output.datablock)) dump = DataBlockDumper(datablocks) dump.as_file(params.output.datablock)
def background(imageset, indx, n_bins, mask_params=None): from dials.array_family import flex from libtbx.phil import parse from scitbx import matrix if mask_params is None: # Default mask params for trusted range mask_params = phil_scope.fetch(parse("")).extract().masking from dials.util.masking import MaskGenerator mask_generator = MaskGenerator(mask_params) mask = mask_generator.generate(imageset) detector = imageset.get_detector() beam = imageset.get_beam() # Only working with single panel detector for now assert len(detector) == 1 panel = detector[0] mask = mask[0] n = matrix.col(panel.get_normal()).normalize() b = matrix.col(beam.get_s0()).normalize() wavelength = beam.get_wavelength() if math.fabs(b.dot(n)) < 0.95: raise Sorry("Detector not perpendicular to beam") data = imageset.get_raw_data(indx) assert len(data) == 1 data = data[0] data = data.as_double() spot_params = spot_phil.fetch(source=parse("")).extract() threshold_function = SpotFinderFactory.configure_threshold(spot_params) peak_pixels = threshold_function.compute_threshold(data, mask) signal = data.select(peak_pixels.iselection()) background_pixels = mask & ~peak_pixels background = data.select(background_pixels.iselection()) # print some summary information print("Mean background: %.3f" % (flex.sum(background) / background.size())) if len(signal) > 0: print("Max/total signal pixels: %.0f / %.0f" % (flex.max(signal), flex.sum(signal))) else: print("No signal pixels on this image") print("Peak/background/masked pixels: %d / %d / %d" % (peak_pixels.count(True), background.size(), mask.count(False))) # compute histogram of two-theta values, then same weighted # by pixel values, finally divide latter by former to get # the radial profile out, need to set the number of bins # sensibly; inspired by method in PyFAI two_theta_array = panel.get_two_theta_array(beam.get_s0()) two_theta_array = two_theta_array.as_1d().select( background_pixels.iselection()) # Use flex.weighted_histogram h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins) h1 = flex.weighted_histogram(two_theta_array, background, n_slots=n_bins) h2 = flex.weighted_histogram(two_theta_array, background * background, n_slots=n_bins) d0 = h0.slots() d1 = h1.slots() d2 = h2.slots() I = d1 / d0 I2 = d2 / d0 sig = flex.sqrt(I2 - flex.pow2(I)) tt = h0.slot_centers() d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt)) return d_spacings, I, sig
def from_parameters(params=None, experiments=None): """ Given a set of parameters, construct the spot finder :param params: The input parameters :returns: The spot finder instance """ from dials.util.masking import MaskGenerator from dials.algorithms.spot_finding.finder import SpotFinder from libtbx.phil import parse from dxtbx.imageset import ImageSequence if params is None: params = phil_scope.fetch(source=parse("")).extract() if params.spotfinder.force_2d and params.output.shoeboxes is False: no_shoeboxes_2d = True elif experiments is not None and params.output.shoeboxes is False: no_shoeboxes_2d = False all_stills = True for experiment in experiments: if isinstance(experiment.imageset, ImageSequence): all_stills = False break if all_stills: no_shoeboxes_2d = True else: no_shoeboxes_2d = False # Read in the lookup files mask = SpotFinderFactory.load_image(params.spotfinder.lookup.mask) params.spotfinder.lookup.mask = mask # Configure the filter options filter_spots = SpotFinderFactory.configure_filter(params) # Create the threshold strategy threshold_function = SpotFinderFactory.configure_threshold( params, experiments) # Configure the mask generator mask_generator = MaskGenerator(params.spotfinder.filter) # Make sure 'none' is interpreted as None if params.spotfinder.mp.method == "none": params.spotfinder.mp.method = None # Setup the spot finder return SpotFinder( threshold_function=threshold_function, mask=params.spotfinder.lookup.mask, filter_spots=filter_spots, scan_range=params.spotfinder.scan_range, write_hot_mask=params.spotfinder.write_hot_mask, hot_mask_prefix=params.spotfinder.hot_mask_prefix, mp_method=params.spotfinder.mp.method, mp_nproc=params.spotfinder.mp.nproc, mp_njobs=params.spotfinder.mp.njobs, mp_chunksize=params.spotfinder.mp.chunksize, max_strong_pixel_fraction=params.spotfinder.filter. max_strong_pixel_fraction, compute_mean_background=params.spotfinder.compute_mean_background, region_of_interest=params.spotfinder.region_of_interest, mask_generator=mask_generator, min_spot_size=params.spotfinder.filter.min_spot_size, max_spot_size=params.spotfinder.filter.max_spot_size, no_shoeboxes_2d=no_shoeboxes_2d, min_chunksize=params.spotfinder.mp.min_chunksize, )
def process_event(self, run, timestamp): """ Process a single event from a run @param run psana run object @param timestamp psana timestamp object """ ts = cspad_tbx.evt_timestamp((timestamp.seconds(),timestamp.nanoseconds()/1e6)) if ts is None: print "No timestamp, skipping shot" return if len(self.params_cache.debug.event_timestamp) > 0 and ts not in self.params_cache.debug.event_timestamp: return if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events: if ts in self.known_events: if self.known_events[ts] not in ["stop", "done", "fail"]: if self.params_cache.debug.skip_bad_events: print "Skipping event %s: possibly caused an unknown exception previously"%ts return elif self.params_cache.debug.skip_processed_events: print "Skipping event %s: processed successfully previously"%ts return else: if self.params_cache.debug.skip_unprocessed_events: print "Skipping event %s: not processed previously"%ts return self.debug_start(ts) evt = run.event(timestamp) if evt.get("skip_event") or "skip_event" in [key.key() for key in evt.keys()]: print "Skipping event",ts self.debug_write("psana_skip", "skip") return print "Accepted", ts self.params = copy.deepcopy(self.params_cache) # the data needs to have already been processed and put into the event by psana if self.params.format.file_format == 'cbf': # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data(self.psana_det, evt, use_default=False, dark=True, common_mode=self.common_mode, apply_gain_mask=self.params.format.cbf.gain_mask_value is not None, gain_mask_value=self.params.format.cbf.gain_mask_value, per_pixel_gain=False) if data is None: print "No data" self.debug_write("no_data", "skip") return if self.params.format.cbf.override_distance is None: distance = cspad_tbx.env_distance(self.params.input.address, run.env(), self.params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" self.debug_write("no_distance", "skip") return else: distance = self.params.format.cbf.override_distance if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" self.debug_write("no_wavelength", "skip") return else: wavelength = 12398.4187/self.params.format.cbf.override_energy if self.params.format.file_format == 'pickle': image_dict = evt.get(self.params.format.pickle.out_key) data = image_dict['DATA'] timestamp = t = ts s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23] print "Processing shot", s if self.params.format.file_format == 'cbf': # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data(self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address) if self.params.input.reference_geometry is not None: from dxtbx.model import Detector # copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead. cspad_img._detector_instance = Detector.from_dict(self.reference_detector.to_dict()) cspad_img.sync_detector_to_cbf() elif self.params.format.file_format == 'pickle': from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory cspad_img = FormatPYunspecifiedStillInMemory(image_dict) cspad_img.timestamp = s if self.params.dispatch.dump_all: self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s)) self.cache_ranges(cspad_img, self.params) imgset = MemImageSet([cspad_img]) if self.params.dispatch.estimate_gain_only: from dials.command_line.estimate_gain import estimate_gain estimate_gain(imgset) return if not self.params.dispatch.find_spots: self.debug_write("data_loaded", "done") return datablock = DataBlockFactory.from_imageset(imgset)[0] # before calling DIALS for processing, set output paths according to the templates if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template: self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.indexed_filename_template%("idx-" + s)) if "%s" in self.refined_experiments_filename_template: self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.refined_experiments_filename_template%("idx-" + s)) if "%s" in self.integrated_filename_template: self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.integrated_filename_template%("idx-" + s)) if "%s" in self.reindexedstrong_filename_template: self.params.output.reindexedstrong_filename = os.path.join(self.params.output.output_dir, self.reindexedstrong_filename_template%("idx-" + s)) # Load a dials mask from the trusted range and psana mask from dials.util.masking import MaskGenerator generator = MaskGenerator(self.params.border_mask) mask = generator.generate(imgset) if self.params.format.file_format == "cbf": mask = tuple([a&b for a, b in zip(mask,self.dials_mask)]) if self.spotfinder_mask is None: self.params.spotfinder.lookup.mask = mask else: self.params.spotfinder.lookup.mask = tuple([a&b for a, b in zip(mask,self.spotfinder_mask)]) if self.integration_mask is None: self.params.integration.lookup.mask = mask else: self.params.integration.lookup.mask = tuple([a&b for a, b in zip(mask,self.integration_mask)]) self.debug_write("spotfind_start") try: observed = self.find_spots(datablock) except Exception, e: import traceback; traceback.print_exc() print str(e), "event", timestamp self.debug_write("spotfinding_exception", "fail") return
def process_event(self, run, timestamp): """ Process a single event from a run @param run psana run object @param timestamp psana timestamp object """ ts = cspad_tbx.evt_timestamp( (timestamp.seconds(), timestamp.nanoseconds() / 1e6)) if ts is None: print "No timestamp, skipping shot" return if len(self.params_cache.debug.event_timestamp ) > 0 and ts not in self.params_cache.debug.event_timestamp: return if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events: if ts in self.known_events: if self.known_events[ts] not in ["stop", "done", "fail"]: if self.params_cache.debug.skip_bad_events: print "Skipping event %s: possibly caused an unknown exception previously" % ts return elif self.params_cache.debug.skip_processed_events: print "Skipping event %s: processed successfully previously" % ts return else: if self.params_cache.debug.skip_unprocessed_events: print "Skipping event %s: not processed previously" % ts return self.debug_start(ts) evt = run.event(timestamp) if evt.get("skip_event") or "skip_event" in [ key.key() for key in evt.keys() ]: print "Skipping event", ts self.debug_write("psana_skip", "skip") return print "Accepted", ts self.params = copy.deepcopy(self.params_cache) # the data needs to have already been processed and put into the event by psana if self.params.format.file_format == 'cbf': # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data( self.psana_det, evt, use_default=False, dark=True, common_mode=self.common_mode, apply_gain_mask=self.params.format.cbf.gain_mask_value is not None, gain_mask_value=self.params.format.cbf.gain_mask_value, per_pixel_gain=False) if data is None: print "No data" self.debug_write("no_data", "skip") return if self.params.format.cbf.override_distance is None: distance = cspad_tbx.env_distance( self.params.input.address, run.env(), self.params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" self.debug_write("no_distance", "skip") return else: distance = self.params.format.cbf.override_distance if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" self.debug_write("no_wavelength", "skip") return else: wavelength = 12398.4187 / self.params.format.cbf.override_energy if self.params.format.file_format == 'pickle': image_dict = evt.get(self.params.format.pickle.out_key) data = image_dict['DATA'] timestamp = t = ts s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[ 20:23] print "Processing shot", s if self.params.format.file_format == 'cbf': # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data( self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address) if self.params.input.reference_geometry is not None: from dxtbx.model import Detector # copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead. cspad_img._detector_instance = Detector.from_dict( self.reference_detector.to_dict()) cspad_img.sync_detector_to_cbf() elif self.params.format.file_format == 'pickle': from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory cspad_img = FormatPYunspecifiedStillInMemory(image_dict) cspad_img.timestamp = s if self.params.dispatch.dump_all: self.save_image( cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s)) self.cache_ranges(cspad_img, self.params) imgset = MemImageSet([cspad_img]) if self.params.dispatch.estimate_gain_only: from dials.command_line.estimate_gain import estimate_gain estimate_gain(imgset) return if not self.params.dispatch.find_spots: self.debug_write("data_loaded", "done") return datablock = DataBlockFactory.from_imageset(imgset)[0] # before calling DIALS for processing, set output paths according to the templates if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template: self.params.output.indexed_filename = os.path.join( self.params.output.output_dir, self.indexed_filename_template % ("idx-" + s)) if "%s" in self.refined_experiments_filename_template: self.params.output.refined_experiments_filename = os.path.join( self.params.output.output_dir, self.refined_experiments_filename_template % ("idx-" + s)) if "%s" in self.integrated_filename_template: self.params.output.integrated_filename = os.path.join( self.params.output.output_dir, self.integrated_filename_template % ("idx-" + s)) if "%s" in self.reindexedstrong_filename_template: self.params.output.reindexedstrong_filename = os.path.join( self.params.output.output_dir, self.reindexedstrong_filename_template % ("idx-" + s)) # Load a dials mask from the trusted range and psana mask from dials.util.masking import MaskGenerator generator = MaskGenerator(self.params.border_mask) mask = generator.generate(imgset) if self.params.format.file_format == "cbf": mask = tuple([a & b for a, b in zip(mask, self.dials_mask)]) if self.spotfinder_mask is None: self.params.spotfinder.lookup.mask = mask else: self.params.spotfinder.lookup.mask = tuple( [a & b for a, b in zip(mask, self.spotfinder_mask)]) if self.integration_mask is None: self.params.integration.lookup.mask = mask else: self.params.integration.lookup.mask = tuple( [a & b for a, b in zip(mask, self.integration_mask)]) self.debug_write("spotfind_start") try: observed = self.find_spots(datablock) except Exception, e: import traceback traceback.print_exc() print str(e), "event", timestamp self.debug_write("spotfinding_exception", "fail") return
def generate_mask( experiments, # type: ExperimentList params, # type: phil.scope_extract ): # type: (...) -> Tuple[Masks, Optional[ExperimentList]] """ Generate a pixel mask for each imageset in an experiment list. Use the masking parameters :param:`params` and the experiments in the experiment list :param:`experiments` to define a pixel mask for each of the associated imagesets. The masks are generated using :mod:`dials.util.masking`. The masks will be saved to disk at the location specified by :attr:`params.output.mask`. If the experiment list contains more than one imageset, multiple mask files will be produced, with filenames differentiated by an appended number. Optionally, if a path :attr:`params.output.experiments` is set, a modified copy of :param:`experiments` with the masks applied will be saved to that location. Args: experiments: An experiment list containing only one imageset. params: Masking parameters, having the structure defined in :data:`phil_scope`. Returns: A list of masks, one for each imageset. A copy of :param:`experiments` with the masks applied (optional, only returned if :attr:`params.output.experiments` is set). """ imagesets = experiments.imagesets() masks = [] # Create output mask filenames num_imagesets = len(imagesets) if num_imagesets == 1: filenames = [params.output.mask] else: # If there is more than one imageset, append a number to each output filename name, ext = os.path.splitext(params.output.mask) pad = len(str(num_imagesets)) filenames = [ "{name}_{num:0{pad}}{ext}".format(name=name, num=i + 1, pad=pad, ext=ext) for i in range(num_imagesets) ] # Generate the mask generator = MaskGenerator(params) for imageset, filename in zip(imagesets, filenames): mask = generator.generate(imageset) masks.append(mask) # Save the mask to file log.info("Writing mask to %s", filename) with open(filename, "wb") as fh: pickle.dump(mask, fh) if params.output.experiments: # Apply the mask to the imageset imageset.external_lookup.mask.data = ImageBool(mask) imageset.external_lookup.mask.filename = filename if params.output.experiments: # Save the experiment list log.info("Saving experiments to %s", params.output.experiments) experiments.as_file(params.output.experiments) else: experiments = None return masks, experiments