def calculate_parameters(self, experiments=None): """Image modification for current cctbx.xfel.""" if not experiments: # If data are given, apply modifications as specified below error = "IOTA IMPORT ERROR: Experiment list not found!" return None, error else: error = [] # Calculate auto-threshold # TODO: Revisit this; I REALLY don't like it. if self.auto_threshold: beamX = self.img_object.final["beamX"] beamY = self.img_object.final["beamY"] px_size = self.img_object.final["pixel"] try: img = dxtbx.load(self.img_object.img_path) raw_data = img.get_raw_data() beam_x_px = int(beamX / px_size) beam_y_px = int(beamY / px_size) data_array = raw_data.as_numpy_array().astype(float) self.center_int = np.nanmax( data_array[ beam_y_px - 20 : beam_y_px + 20, beam_x_px - 20 : beam_x_px + 20, ] ) except Exception as e: error.append( "IOTA IMPORT ERROR: Auto-threshold failed! {}".format(e) ) # Estimate gain (or set gain to 1.00 if cannot calculate) if self.estimate_gain: with util.Capturing() as junk_output: try: assert self.img_object.experiments # Must have experiments here imageset = self.img_object.experiments.extract_imagesets()[0] self.img_object.gain = estimate_gain(imageset) except Exception as e: error.append( "IOTA IMPORT ERROR: Estimate gain failed! ".format(e) ) # Collect error messages for logging if error: error_message = "\n".join(error) else: error_message = None return experiments, error_message
def calculate_parameters(self, datablock=None): ''' Image modification for current cctbx.xfel ''' if not datablock: # If data are given, apply modifications as specified below error = 'IOTA IMPORT ERROR: Datablock not found!' return None, error else: error = [] # Calculate auto-threshold # TODO: Revisit this; I REALLY don't like it. if self.iparams.cctbx_xfel.auto_threshold: beamX = self.img_object.final['beamX'] beamY = self.img_object.final['beamY'] px_size = self.img_object.final['pixel'] try: img = dxtbx.load(self.img_object.img_path) raw_data = img.get_raw_data() beam_x_px = int(beamX / px_size) beam_y_px = int(beamY / px_size) data_array = raw_data.as_numpy_array().astype(float) self.center_int = np.nanmax( data_array[beam_y_px - 20:beam_y_px + 20, beam_x_px - 20:beam_x_px + 20]) except Exception as e: error.append( 'IOTA IMPORT ERROR: Auto-threshold failed! {}'.format( e)) # Estimate gain (or set gain to 1.00 if cannot calculate) if self.iparams.advanced.estimate_gain: with util.Capturing() as junk_output: try: assert self.img_object.datablock # Must have datablock here imageset = self.img_object.datablock.extract_imagesets( )[0] self.img_object.gain = estimate_gain(imageset) except Exception as e: error.append( 'IOTA IMPORT ERROR: Estimate gain failed! '.format( e)) # Collect error messages for logging if error: error_message = '\n'.join(error) else: error_message = None return datablock, error_message
class SingleImage(object): def __init__(self, img, init, verbose=True, imported_grid=None): """ Constructor for the SingleImage object using a raw image file or pickle """ # Initialize parameters self.params = init.params self.args = init.args self.user_id = init.user_id self.raw_img = img[2] self.conv_img = img[2] self.img_index = img[0] self.center_int = 0 self.status = None self.fail = None self.final = None self.log_info = [] self.gs_results = [] self.main_log = init.logfile self.verbose = verbose self.hmed = self.params.cctbx.grid_search.height_median self.amed = self.params.cctbx.grid_search.area_median self.input_base = init.input_base self.conv_base = init.conv_base self.int_base = init.int_base self.obj_base = init.obj_base self.fin_base = init.fin_base self.viz_base = init.viz_base self.log_base = init.log_base self.tmp_base = init.tmp_base self.abort_file = os.path.join(self.int_base, '.abort.tmp') self.obj_path = None self.obj_file = None self.fin_path = None self.fin_file = None self.viz_path = None # ============================== SELECTION-ONLY FUNCTIONS ============================== # def import_int_file(self, init): """ Replaces path settings in imported image object with new settings NEED TO RE-DO LATER """ if os.path.isfile(self.abort_file): self.fail = 'aborted' return self # Generate paths to output files self.params = init.params self.main_log = init.logfile self.input_base = init.input_base self.conv_base = init.conv_base self.int_base = init.int_base self.obj_base = init.obj_base self.fin_base = init.fin_base self.log_base = init.log_base self.viz_base = init.viz_base self.obj_path = misc.make_image_path(self.conv_img, self.input_base, self.obj_base) self.obj_file = os.path.abspath( os.path.join( self.obj_path, os.path.basename(self.conv_img).split('.')[0] + ".int")) self.fin_path = misc.make_image_path(self.conv_img, self.input_base, self.fin_base) self.log_path = misc.make_image_path(self.conv_img, self.input_base, self.log_base) self.fin_file = os.path.abspath( os.path.join( self.fin_path, os.path.basename(self.conv_img).split('.')[0] + "_int.pickle")) self.final['final'] = self.fin_file self.final['img'] = self.conv_img self.viz_path = misc.make_image_path(self.conv_img, self.input_base, self.viz_base) self.viz_file = os.path.join( self.viz_path, os.path.basename(self.conv_img).split('.')[0] + "_int.png") # Create actual folders (if necessary) try: if not os.path.isdir(self.obj_path): os.makedirs(self.obj_path) if not os.path.isdir(self.fin_path): os.makedirs(self.fin_path) if not os.path.isdir(self.viz_path): os.makedirs(self.viz_path) except OSError: pass # Grid search / integration log file self.int_log = os.path.join( self.log_path, os.path.basename(self.conv_img).split('.')[0] + '.tmp') # Reset status to 'grid search' to pick up at selection (if no fail) if self.fail == None: self.status = 'bypass grid search' return self def determine_gs_result_file(self): """ For 'selection-only' cctbx.xfel runs, determine where the image objects are """ if self.params.cctbx.selection.select_only.grid_search_path != None: obj_path = os.path.abspath( self.params.cctbx.selection.select_only.grid_search_path) else: run_number = int(os.path.basename(self.int_base)) - 1 obj_path = "{}/integration/{:03d}/image_objects"\ "".format(os.path.abspath(os.curdir), run_number) gs_result_file = os.path.join(obj_path, os.path.basename(self.obj_file)) return gs_result_file # =============================== IMAGE IMPORT FUNCTIONS =============================== # def load_image(self): """ Reads raw image file and extracts data for conversion into pickle format. Also estimates gain if turned on.""" # Load raw image or image pickle try: with misc.Capturing() as junk_output: loaded_img = dxtbx.load(self.raw_img) except Exception, e: print 'IOTA IMPORT ERROR:', e loaded_img = None pass # Extract image information if loaded_img is not None: raw_data = loaded_img.get_raw_data() detector = loaded_img.get_detector()[0] beam = loaded_img.get_beam() scan = loaded_img.get_scan() distance = detector.get_distance() pixel_size = detector.get_pixel_size()[0] overload = detector.get_trusted_range()[1] wavelength = beam.get_wavelength() beam_x = detector.get_beam_centre(beam.get_s0())[0] beam_y = detector.get_beam_centre(beam.get_s0())[1] if scan is None: timestamp = None img_type = 'pickle' else: img_type = 'raw' msec, sec = math.modf(scan.get_epochs()[0]) timestamp = evt_timestamp((sec, msec)) # Assemble datapack data = dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data['OSC_START'] = 0 #osc_start data['OSC_RANGE'] = 0 #osc_start data['TIME'] = scan.get_exposure_times()[0] else: data = None img_type = 'not imported' # Estimate gain (or set gain to 1.00 if cannot calculate) # Cribbed from estimate_gain.py by Richard Gildea if self.params.advanced.estimate_gain: from dxtbx.datablock import DataBlockFactory from dials.command_line.estimate_gain import estimate_gain with misc.Capturing() as junk_output: try: datablock = DataBlockFactory.from_filenames([self.raw_img ])[0] imageset = datablock.extract_imagesets()[0] self.gain = estimate_gain(imageset) except Exception, e: self.gain = 1.0
def process_event(self, run, timestamp): """ Process a single event from a run @param run psana run object @param timestamp psana timestamp object """ ts = cspad_tbx.evt_timestamp( (timestamp.seconds(), timestamp.nanoseconds() / 1e6)) if ts is None: print "No timestamp, skipping shot" return if len(self.params_cache.debug.event_timestamp ) > 0 and ts not in self.params_cache.debug.event_timestamp: return if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events: if ts in self.known_events: if self.known_events[ts] not in ["stop", "done", "fail"]: if self.params_cache.debug.skip_bad_events: print "Skipping event %s: possibly caused an unknown exception previously" % ts return elif self.params_cache.debug.skip_processed_events: print "Skipping event %s: processed successfully previously" % ts return else: if self.params_cache.debug.skip_unprocessed_events: print "Skipping event %s: not processed previously" % ts return self.debug_start(ts) evt = run.event(timestamp) if evt.get("skip_event") or "skip_event" in [ key.key() for key in evt.keys() ]: print "Skipping event", ts self.debug_write("psana_skip", "skip") return print "Accepted", ts self.params = copy.deepcopy(self.params_cache) # the data needs to have already been processed and put into the event by psana if self.params.format.file_format == 'cbf': # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data( self.psana_det, evt, use_default=False, dark=True, common_mode=self.common_mode, apply_gain_mask=self.params.format.cbf.gain_mask_value is not None, gain_mask_value=self.params.format.cbf.gain_mask_value, per_pixel_gain=False) if data is None: print "No data" self.debug_write("no_data", "skip") return if self.params.format.cbf.override_distance is None: distance = cspad_tbx.env_distance( self.params.input.address, run.env(), self.params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" self.debug_write("no_distance", "skip") return else: distance = self.params.format.cbf.override_distance if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" self.debug_write("no_wavelength", "skip") return else: wavelength = 12398.4187 / self.params.format.cbf.override_energy if self.params.format.file_format == 'pickle': image_dict = evt.get(self.params.format.pickle.out_key) data = image_dict['DATA'] timestamp = t = ts s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[ 20:23] print "Processing shot", s if self.params.format.file_format == 'cbf': # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data( self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address) if self.params.input.reference_geometry is not None: from dxtbx.model import Detector # copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead. cspad_img._detector_instance = Detector.from_dict( self.reference_detector.to_dict()) cspad_img.sync_detector_to_cbf() elif self.params.format.file_format == 'pickle': from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory cspad_img = FormatPYunspecifiedStillInMemory(image_dict) cspad_img.timestamp = s if self.params.dispatch.dump_all: self.save_image( cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s)) self.cache_ranges(cspad_img, self.params) imgset = MemImageSet([cspad_img]) if self.params.dispatch.estimate_gain_only: from dials.command_line.estimate_gain import estimate_gain estimate_gain(imgset) return if not self.params.dispatch.find_spots: self.debug_write("data_loaded", "done") return datablock = DataBlockFactory.from_imageset(imgset)[0] # before calling DIALS for processing, set output paths according to the templates if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template: self.params.output.indexed_filename = os.path.join( self.params.output.output_dir, self.indexed_filename_template % ("idx-" + s)) if "%s" in self.refined_experiments_filename_template: self.params.output.refined_experiments_filename = os.path.join( self.params.output.output_dir, self.refined_experiments_filename_template % ("idx-" + s)) if "%s" in self.integrated_filename_template: self.params.output.integrated_filename = os.path.join( self.params.output.output_dir, self.integrated_filename_template % ("idx-" + s)) if "%s" in self.reindexedstrong_filename_template: self.params.output.reindexedstrong_filename = os.path.join( self.params.output.output_dir, self.reindexedstrong_filename_template % ("idx-" + s)) # Load a dials mask from the trusted range and psana mask from dials.util.masking import MaskGenerator generator = MaskGenerator(self.params.border_mask) mask = generator.generate(imgset) if self.params.format.file_format == "cbf": mask = tuple([a & b for a, b in zip(mask, self.dials_mask)]) if self.spotfinder_mask is None: self.params.spotfinder.lookup.mask = mask else: self.params.spotfinder.lookup.mask = tuple( [a & b for a, b in zip(mask, self.spotfinder_mask)]) if self.integration_mask is None: self.params.integration.lookup.mask = mask else: self.params.integration.lookup.mask = tuple( [a & b for a, b in zip(mask, self.integration_mask)]) self.debug_write("spotfind_start") try: observed = self.find_spots(datablock) except Exception, e: import traceback traceback.print_exc() print str(e), "event", timestamp self.debug_write("spotfinding_exception", "fail") return
def load_image(self): """ Reads raw image file and extracts data for conversion into pickle format. Also estimates gain if turned on.""" # Load raw image or image pickle try: with misc.Capturing() as junk_output: loaded_img = dxtbx.load(self.raw_img) except Exception as e: print 'IOTA IMPORT ERROR:', e loaded_img = None pass # Extract image information if loaded_img is not None: raw_data = loaded_img.get_raw_data() detector = loaded_img.get_detector()[0] beam = loaded_img.get_beam() scan = loaded_img.get_scan() distance = detector.get_distance() pixel_size = detector.get_pixel_size()[0] overload = detector.get_trusted_range()[1] wavelength = beam.get_wavelength() beam_x = detector.get_beam_centre(beam.get_s0())[0] beam_y = detector.get_beam_centre(beam.get_s0())[1] if scan is None: timestamp = None img_type = 'pickle' else: img_type = 'raw' msec, sec = math.modf(scan.get_epochs()[0]) timestamp = evt_timestamp((sec,msec)) # Assemble datapack data = dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp ) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data['OSC_START'] = 0 #osc_start data['OSC_RANGE'] = 0 #osc_start data['TIME'] = scan.get_exposure_times()[0] else: data = None img_type = 'not imported' # Estimate gain (or set gain to 1.00 if cannot calculate) # Cribbed from estimate_gain.py by Richard Gildea if self.params.advanced.estimate_gain: from dxtbx.datablock import DataBlockFactory from dials.command_line.estimate_gain import estimate_gain with misc.Capturing() as junk_output: try: datablock = DataBlockFactory.from_filenames([self.raw_img])[0] imageset = datablock.extract_imagesets()[0] self.gain = estimate_gain(imageset) except Exception as e: self.gain = 1.0 else: self.gain = 1.0 return data, img_type