def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ super(mod_radial_average, self).event(evt, env) if (evt.get("skip_event")): return # This module only applies to detectors for which a distance is # available. distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: self.nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), "skip_event") return # See r17537 of mod_average.py. device = cspad_tbx.address_split(self.address)[2] if device == 'Cspad': pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value elif device == 'marccd': pixel_size = 0.079346 saturated_value = 2**16 - 1 d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=self.cspad_img.iround(), # XXX ouch! distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=self.timestamp, wavelength=self.wavelength, xtal_target=self.m_xtal_target) from xfel.command_line.radial_average import run args = [ "file_path=XTC stream", "xfel_target=%s"%self.m_xtal_target, "verbose=False" ] t = self.timestamp s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23] if self._dirname is not None: dest_path = os.path.join(self._dirname, self._basename + s + ".txt") args.append("output_file=%s"%dest_path) self.logger.info("Calculating radial average for image %s"%s) xvals, results = run(args, d) evt.put(xvals, "cctbx.xfel.radial_average.xvals") evt.put(results, "cctbx.xfel.radial_average.results")
def do_work(img_no): n_fails = 0 while True: try: raw_data = data.get_raw_data(img_no) break except (KeyError, ValueError): n_fails += 1 print "Fail to read, attempt number", n_fails if n_fails > 100: raise Exception("Couldn't read the data") import time time.sleep(n_fails * 0.1) imgdict = cspad_tbx.dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, address="Sacla.MPCCD.8tile", active_areas=active_areas) imgdict = crop_image_pickle( imgdict, preserve_active_areas_even_though_cropping_would_invalidate_them=True) dest_path = os.path.join(dest_dir, dest_base + "_%06d.pickle" % img_no) print "Saving image", img_no, "to", dest_path easy_pickle.dump(dest_path, imgdict)
def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ super(mod_image_dict, self).event(evt, env) if (evt.get("skip_event")): return # This module only applies to detectors for which a distance is # available. distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: self.nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), "skip_event") return device = cspad_tbx.address_split(self.address)[2] self.logger.info("Subprocess %02d: process image #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) # See r17537 of mod_average.py. if device == 'Cspad': pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value elif device == 'Rayonix': pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) saturated_value = rayonix_tbx.rayonix_saturated_value elif device == 'marccd': pixel_size = evt.get("marccd_pixel_size") saturated_value = evt.get("marccd_saturated_value") if distance == 0: distance = evt.get("marccd_distance") d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=self.cspad_img.iround(), # XXX ouch! distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=self.timestamp, wavelength=self.wavelength) evt.put(d, self.m_out_key) # Diagnostic message emitted only when all the processing is done. if (env.subprocess() >= 0): self.logger.info("Subprocess %02d: accepted #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) else: self.logger.info("Accepted #%05d @ %s" % (self.nshots, self.timestamp))
def save_image( command_line, imgpath, scan, raw_data, distance, pixel_size, wavelength, beam_x, beam_y, overload, timestamp, image_number=None, ): if image_number is None: destpath = os.path.join( os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + ".pickle", ) else: destpath = os.path.join( os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + "%05d.pickle" % image_number, ) if command_line.options.skip_converted and os.path.isfile(destpath): if command_line.options.verbose: print("Skipping %s, file exists" % imgpath) return data = dpack( data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp, ) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data["OSC_START"] = osc_start data["OSC_RANGE"] = osc_range data["TIME"] = scan.get_exposure_times()[0] if command_line.options.crop: data = crop_image_pickle(data) if command_line.options.verbose: print("Writing", destpath) easy_pickle.dump(destpath, data)
def event(self, evt, env): """The event() function is called for every L1Accept transition. It outputs the detector image associated with the event @p evt to the file system. @param evt Event data object, a configure object @param env Environment object """ super(mod_dump, self).event(evt, env) if (evt.get('skip_event')): return if self.cspad_img is None: print "No image to save for %s"%self.timestamp return # Where the sample-detector distance is not available, set it to # zero. distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: distance = 0 # See r17537 of mod_average.py. device = cspad_tbx.address_split(self.address)[2] if device == 'Cspad': pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value output_filename = self._basename elif device == 'Rayonix': pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) saturated_value = rayonix_tbx.rayonix_saturated_value output_filename = self._basename elif device == 'marccd': if distance == 0: distance = evt.get('marccd_distance') pixel_size = 0.079346 saturated_value = 2**16 - 1 output_filename = self._basename + evt.get(str, 'mccd_name') + "_" d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=self.cspad_img.iround(), # XXX ouch! distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=self.timestamp, wavelength=self.wavelength) if self._format == "pickle": cspad_tbx.dwritef(d, self._dirname, output_filename) elif self._format == "tiff": cspad_tbx.write_tiff(d, self._dirname, output_filename) output_filename = None
def event(self, evt, env): """The event() function is called for every L1Accept transition. It outputs the detector image associated with the event @p evt to the file system. @param evt Event data object, a configure object @param env Environment object """ super(mod_dump, self).event(evt, env) if (evt.get('skip_event')): return if self.cspad_img is None: print("No image to save for %s"%self.timestamp) return # Where the sample-detector distance is not available, set it to # zero. distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: distance = 0 # See r17537 of mod_average.py. device = cspad_tbx.address_split(self.address)[2] if device == 'Cspad': pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value output_filename = self._basename elif device == 'Rayonix': pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) saturated_value = rayonix_tbx.rayonix_saturated_value output_filename = self._basename elif device == 'marccd': if distance == 0: distance = evt.get('marccd_distance') pixel_size = 0.079346 saturated_value = 2**16 - 1 output_filename = self._basename + evt.get(str, 'mccd_name') + "_" d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=self.cspad_img.iround(), # XXX ouch! distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=self.timestamp, wavelength=self.wavelength) if self._format == "pickle": cspad_tbx.dwritef(d, self._dirname, output_filename) elif self._format == "tiff": cspad_tbx.write_tiff(d, self._dirname, output_filename) output_filename = None
def set_up_hitfinder(self): # See r17537 of mod_average.py. device = cspad_tbx.address_split(self.address)[2] if device == 'Cspad': img_dim = (1765, 1765) pixel_size = cspad_tbx.pixel_size elif device == 'marccd': img_dim = (4500, 4500) pixel_size = 0.079346 elif device == 'Rayonix': img_dim = rayonix_tbx.get_rayonix_detector_dimensions( self.bin_size) pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) else: raise RuntimeError("Unsupported device %s" % self.address) if self.beam_center is None: self.beam_center = [0, 0] self.hitfinder_d = cspad_tbx.dpack( active_areas=self.active_areas, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0), xtal_target=self.m_xtal_target) if device == 'Cspad': # Figure out which ASIC:s are on the central four sensors. This # only applies to the CSPAD. assert len(self.active_areas) % 4 == 0 distances = flex.double() for i in range(0, len(self.active_areas), 4): cenasic = ( (self.active_areas[i + 0] + self.active_areas[i + 2]) / 2, (self.active_areas[i + 1] + self.active_areas[i + 3]) / 2) distances.append( math.hypot(cenasic[0] - self.beam_center[0], cenasic[1] - self.beam_center[1])) orders = flex.sort_permutation(distances) # Use the central 8 ASIC:s (central 4 sensors). flags = flex.int(len(self.active_areas) // 4, 0) for i in range(8): flags[orders[i]] = 1 self.asic_filter = "distl.tile_flags=" + ",".join( ["%1d" % b for b in flags]) elif device == 'marccd': # There is only one active area for the MAR CCD, so use it. self.asic_filter = "distl.tile_flags=1" elif device == 'Rayonix': # There is only one active area for the Rayonix, so use it. self.asic_filter = "distl.tile_flags=1"
def set_up_hitfinder(self): # See r17537 of mod_average.py. device = cspad_tbx.address_split(self.address)[2] if device == 'Cspad': img_dim = (1765, 1765) pixel_size = cspad_tbx.pixel_size elif device == 'marccd': img_dim = (4500, 4500) pixel_size = 0.079346 elif device == 'Rayonix': img_dim = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size) pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) else: raise RuntimeError("Unsupported device %s" % self.address) if self.beam_center is None: self.beam_center = [0,0] self.hitfinder_d = cspad_tbx.dpack( active_areas=self.active_areas, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0), xtal_target=self.m_xtal_target) if device == 'Cspad': # Figure out which ASIC:s are on the central four sensors. This # only applies to the CSPAD. assert len(self.active_areas) % 4 == 0 distances = flex.double() for i in range(0, len(self.active_areas), 4): cenasic = ((self.active_areas[i + 0] + self.active_areas[i + 2]) / 2, (self.active_areas[i + 1] + self.active_areas[i + 3]) / 2) distances.append(math.hypot(cenasic[0] - self.beam_center[0], cenasic[1] - self.beam_center[1])) orders = flex.sort_permutation(distances) # Use the central 8 ASIC:s (central 4 sensors). flags = flex.int(len(self.active_areas) // 4, 0) for i in range(8): flags[orders[i]] = 1 self.asic_filter = "distl.tile_flags=" + ",".join( ["%1d" % b for b in flags]) elif device == 'marccd': # There is only one active area for the MAR CCD, so use it. self.asic_filter = "distl.tile_flags=1" elif device == 'Rayonix': # There is only one active area for the Rayonix, so use it. self.asic_filter = "distl.tile_flags=1"
def run(args): assert len(args) == 3 d1 = easy_pickle.load(args[0]) d2 = easy_pickle.load(args[1]) image_1 = d1["DATA"] image_2 = d2["DATA"] assert image_1.all() == image_2.all() diff_image = image_1 - image_2 d = cspad_tbx.dpack( data=diff_image, timestamp=cspad_tbx.evt_timestamp(), distance=1, ) easy_pickle.dump(args[2], d)
def save_image(command_line, imgpath, scan, raw_data, distance, pixel_size, wavelength, beam_x, beam_y, overload, timestamp, image_number = None): if image_number is None: destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + ".pickle") else: destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + "%05d.pickle"%image_number) if command_line.options.skip_converted and os.path.isfile(destpath): if command_line.options.verbose: print "Skipping %s, file exists"%imgpath return data = dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp ) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data['OSC_START'] = osc_start data['OSC_RANGE'] = osc_range data['TIME'] = scan.get_exposure_times()[0] if command_line.options.crop: data = crop_image_pickle(data) if command_line.options.verbose: print "Writing", destpath easy_pickle.dump(destpath, data)
def __init__(self, runs, output_dirname=".", roi=None): avg_basename="avg_" stddev_basename="stddev" self.sum_img = None self.sumsq_img = None self.nmemb = 0 self.roi = cspad_tbx.getOptROI(roi) self.unbound_pixel_mask = cspad_unbound_pixel_mask() for i_run, run in enumerate(runs): run_scratch_dir = run result = finalise_one_run(run_scratch_dir) if result.sum_img is None: continue if self.sum_img is None: self.sum_img = result.sum_img self.sumsq_img = result.sumsq_img else: self.sum_img += result.sum_img self.sumsq_img += result.sumsq_img self.nmemb += result.nmemb self.avg_img = self.sum_img.as_double() / self.nmemb self.stddev_img = flex.sqrt((self.sumsq_img.as_double() - self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1)) self.mask = flex.int(self.sum_img.accessor(), 0) self.mask.set_selected(self.sum_img == 0, 1) self.mask.set_selected(self.unbound_pixel_mask > 0, 1) if (output_dirname is not None and avg_basename is not None): if (not os.path.isdir(output_dirname)): os.makedirs(output_dirname) d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=self.avg_img, distance=1, ) cspad_tbx.dwritef(d, output_dirname, avg_basename) d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=self.sum_img, distance=1, ) cspad_tbx.dwritef(d, output_dirname, "sum_") if 1: output_image(self.avg_img, "%s/avg.png" %output_dirname) output_image(self.avg_img, "%s/avg_inv.png" %output_dirname, invert=True) if 1: output_matlab_form(self.sum_img, "%s/sum.m" %output_dirname) output_matlab_form(self.avg_img, "%s/avg.m" %output_dirname) output_matlab_form(self.stddev_img, "%s/stddev.m" %output_dirname) if (stddev_basename is not None): d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=self.stddev_img, distance=1, ) cspad_tbx.dwritef(d, output_dirname, stddev_basename) # XXX we should really figure out automatically the area where the spectrum is #write an integrated spectrum from lines 186-227 #spectrum_focus = self.sum_img.as_numpy_array()[186:228,:] img = self.sum_img if self.roi is None: spectrum_focus = img mask_focus = self.mask else: slices = (slice(self.roi[2],self.roi[3]), slice(self.roi[0],self.roi[1])) spectrum_focus = img[slices] mask_focus = self.mask[slices] if False: from matplotlib import pylab pylab.imshow(spectrum_focus.as_numpy_array()) pylab.show() output_spectrum(spectrum_focus, mask_focus=mask_focus, output_dirname=output_dirname) print "Total number of images used from %i runs: %i" %(i_run+1, self.nmemb)
def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ super(mod_radial_average, self).event(evt, env) if (evt.get("skip_event")): return # This module only applies to detectors for which a distance is # available. distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: self.nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), "skip_event") return # See r17537 of mod_average.py. device = cspad_tbx.address_split(self.address)[2] if device == 'Cspad': pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value elif device == 'marccd': pixel_size = 0.079346 saturated_value = 2**16 - 1 elif device == 'Rayonix': pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) saturated_value = rayonix_tbx.rayonix_saturated_value d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=self.cspad_img.iround(), # XXX ouch! distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=self.timestamp, wavelength=self.wavelength, xtal_target=self.m_xtal_target) from xfel.command_line.radial_average import run args = [ "file_path=XTC stream", "xfel_target=%s" % self.m_xtal_target, "verbose=False" ] t = self.timestamp s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[ 20:23] if self._dirname is not None: dest_path = os.path.join(self._dirname, self._basename + s + ".txt") args.append("output_file=%s" % dest_path) self.logger.info("Calculating radial average for image %s" % s) xvals, results = run(args, d) evt.put(xvals, "cctbx.xfel.radial_average.xvals") evt.put(results, "cctbx.xfel.radial_average.results") def get_closest_idx(data, val): from scitbx.array_family import flex deltas = flex.abs(data - val) return flex.first_index(deltas, flex.min(deltas)) if self._two_theta_low is not None: i_low = results[get_closest_idx(xvals, self._two_theta_low)] evt.put(i_low, "cctbx.xfel.radial_average.two_theta_low") if self._two_theta_high is not None: i_high = results[get_closest_idx(xvals, self._two_theta_high)] evt.put(i_high, "cctbx.xfel.radial_average.two_theta_high")
def run(args): command_line = (option_parser() .option("-o", "--output_filename", action="store", type="string", help="Filename for the output pickle file", default="gain_map.pickle") .option("-f", "--detector_format_version", action="store", type="string", help="Detector format version to use for generating active areas and laying out tiles", default=None) .option("-m", "--optical_metrology_path", action="store", type="string", help="Path to slac optical metrology file. If not set, use Run 4 metrology", default=None) .option("-d", "--distance", action="store", type="int", help="Detector distance put into the gain pickle file. Not needed for processing.", default="0") .option("-w", "--wavelength", action="store", type="float", help="Incident beam wavelength put into the gain pickle file. Not needed for processing.", default="0") ).process(args=args) output_filename = command_line.options.output_filename detector_format_version = command_line.options.detector_format_version if detector_format_version is None or 'XPP' not in detector_format_version: beam_center_x = None beam_center_y = None else: beam_center_x = 1765 // 2 * 0.11 beam_center_y = 1765 // 2 * 0.11 address, timestamp = address_and_timestamp_from_detector_format_version(detector_format_version) # if no detector format version is provided, make sure to write no address to the image pickle # but CsPadDetector (called later), needs an address, so give it a fake one save_address = address is not None if not save_address: address = "CxiDs1-0|Cspad-0" # time stamp will still be None timestamp = evt_timestamp((timestamp,0)) args = command_line.args assert len(args) == 1 if args[0].endswith('.npy'): data = numpy.load(args[0]) det, active_areas = convert_2x2(data, detector_format_version, address) elif args[0].endswith('.txt') or args[0].endswith('.gain'): raw_data = numpy.loadtxt(args[0]) assert raw_data.shape in [(5920, 388), (11840, 194)] det, active_areas = convert_detector(raw_data, detector_format_version, address, command_line.options.optical_metrology_path) img_diff = det img_sel = (img_diff > 0).as_1d() gain_map = flex.double(img_diff.accessor(), 0) gain_map.as_1d().set_selected(img_sel.iselection(), 1/img_diff.as_1d().select(img_sel)) gain_map /= flex.mean(gain_map.as_1d().select(img_sel)) if not save_address: address = None d = cspad_tbx.dpack(data=gain_map, address=address, active_areas=active_areas, timestamp=timestamp, distance=command_line.options.distance,wavelength=command_line.options.wavelength, beam_center_x = beam_center_x, beam_center_y = beam_center_y) easy_pickle.dump(output_filename, d)
def load_image(self): """ Reads raw image file and extracts data for conversion into pickle format. Also estimates gain if turned on.""" # Load raw image or image pickle try: with misc.Capturing() as junk_output: loaded_img = dxtbx.load(self.raw_img) except Exception as e: print 'IOTA IMPORT ERROR:', e loaded_img = None pass # Extract image information if loaded_img is not None: raw_data = loaded_img.get_raw_data() detector = loaded_img.get_detector()[0] beam = loaded_img.get_beam() scan = loaded_img.get_scan() distance = detector.get_distance() pixel_size = detector.get_pixel_size()[0] overload = detector.get_trusted_range()[1] wavelength = beam.get_wavelength() beam_x = detector.get_beam_centre(beam.get_s0())[0] beam_y = detector.get_beam_centre(beam.get_s0())[1] if scan is None: timestamp = None img_type = 'pickle' else: img_type = 'raw' msec, sec = math.modf(scan.get_epochs()[0]) timestamp = evt_timestamp((sec,msec)) # Assemble datapack data = dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp ) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data['OSC_START'] = 0 #osc_start data['OSC_RANGE'] = 0 #osc_start data['TIME'] = scan.get_exposure_times()[0] else: data = None img_type = 'not imported' # Estimate gain (or set gain to 1.00 if cannot calculate) # Cribbed from estimate_gain.py by Richard Gildea if self.params.advanced.estimate_gain: from dxtbx.datablock import DataBlockFactory from dials.command_line.estimate_gain import estimate_gain with misc.Capturing() as junk_output: try: datablock = DataBlockFactory.from_filenames([self.raw_img])[0] imageset = datablock.extract_imagesets()[0] self.gain = estimate_gain(imageset) except Exception as e: self.gain = 1.0 else: self.gain = 1.0 return data, img_type
class xes_from_histograms(object): def __init__(self, pixel_histograms, output_dirname=".", gain_map_path=None, gain_map=None, estimated_gain=30, roi=None, run=None): self.sum_img = flex.double(flex.grid( 370, 391), 0) # XXX define the image size some other way? gain_img = flex.double(self.sum_img.accessor(), 0) assert [gain_map, gain_map_path].count(None) > 0 if gain_map_path is not None: d = easy_pickle.load(gain_map_path) gain_map = d["DATA"] mask = flex.int(self.sum_img.accessor(), 0) start_row = 370 end_row = 0 print len(pixel_histograms.histograms) pixels = list(pixel_histograms.pixels()) n_pixels = len(pixels) if roi is not None: for k, (i, j) in enumerate(reversed(pixels)): if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]): del pixels[n_pixels - k - 1] if gain_map is None: fixed_func = pixel_histograms.fit_one_histogram else: def fixed_func(pixel): return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1) chi_squared_list = flex.double() for i, pixel in enumerate(pixels): #print i,pixel LEG = False start_row = min(start_row, pixel[0]) end_row = max(end_row, pixel[0]) n_photons = 0 try: if LEG: gaussians, two_photon_flag = pixel_histograms.fit_one_histogram( pixel) alt_gaussians = pixel_histograms.fit_one_histogram_two_gaussians( pixel) except ZeroDivisionError: print "HEY DIVIDE BY ZERO" #pixel_histograms.plot_combo(pixel, gaussians) mask[pixel] = 1 continue except RuntimeError, e: print "Error fitting pixel %s" % str(pixel) print str(e) mask[pixel] = 1 continue hist = pixel_histograms.histograms[pixel] if not LEG: gs = alt_gaussians[1].params fit_photons = gs[0] * gs[2] * math.sqrt(2. * math.pi) n_photons = int(round(fit_photons, 0)) fit_interpretation = pixel_histograms.multiphoton_and_fit_residual( pixel_histograms.histograms[pixel], alt_gaussians) multi_photons = fit_interpretation.get_multiphoton_count() total_photons = n_photons + multi_photons if False and n_photons < 0: # Generally, do not mask negative values; if fit is still OK print "\n%d pixel %s altrn %d photons from curvefitting" % ( i, pixel, n_photons) pixel_histograms.plot_combo( pixel, alt_gaussians, interpretation=fit_interpretation) mask[ pixel] = 1 # do not mask out negative pixels if the Gaussian fit is good continue chi_squared_list.append(fit_interpretation.chi_squared()) suspect = False # don't know the optimal statistical test. Histograms vary primarily by total count & # photons if total_photons <= 3: if fit_interpretation.chi_squared( ) > 2.5 or fit_interpretation.quality_factor < 5: suspect = True elif 3 < total_photons <= 10: if fit_interpretation.chi_squared( ) > 5 or fit_interpretation.quality_factor < 10: suspect = True elif 10 < total_photons <= 33: if fit_interpretation.chi_squared( ) > 10 or fit_interpretation.quality_factor < 20: suspect = True elif 33 < total_photons <= 100: if fit_interpretation.chi_squared( ) > 20 or fit_interpretation.quality_factor < 20: suspect = True elif 100 < total_photons <= 330: if fit_interpretation.chi_squared( ) > 30 or fit_interpretation.quality_factor < 25: suspect = True elif 330 < total_photons <= 1000: if fit_interpretation.chi_squared( ) > 40 or fit_interpretation.quality_factor < 30: suspect = True elif 1000 < total_photons: if fit_interpretation.chi_squared( ) > 50 or fit_interpretation.quality_factor < 30: suspect = True if suspect: print "\n%d pixel %s Bad quality 0/1-photon fit" % ( i, pixel), fit_interpretation.quality_factor print " with chi-squared %10.5f" % fit_interpretation.chi_squared( ) print " Suspect", suspect print "%d fit photons, %d total photons" % (n_photons, total_photons) #pixel_histograms.plot_combo(pixel, alt_gaussians, # interpretation=fit_interpretation) mask[pixel] = 1 continue self.sum_img[pixel] = n_photons + multi_photons mask.set_selected(self.sum_img == 0, 1) unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask() mask.set_selected(unbound_pixel_mask > 0, 1) bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7() mask.set_selected(bad_pixel_mask > 0, 1) for row in range(self.sum_img.all()[0]): self.sum_img[row:row + 1, :].count(0) spectrum_focus = self.sum_img[start_row:end_row, :] mask_focus = mask[start_row:end_row, :] spectrum_focus.set_selected(mask_focus > 0, 0) xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus) print "Number of rows: %i" % spectrum_focus.all()[0] print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus) print "Number of images used: %i" % flex.sum( pixel_histograms.histograms.values()[0].slots()) d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=spectrum_focus, distance=1, ccd_image_saturation=2e8, # XXX ) if run is not None: runstr = "_%04d" % run else: runstr = "" cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr) plot_x, plot_y = xes_finalise.output_spectrum( spectrum_focus.iround(), mask_focus=mask_focus, output_dirname=output_dirname, run=run) self.spectrum = (plot_x, plot_y) self.spectrum_focus = spectrum_focus xes_finalise.output_matlab_form( spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr)) print output_dirname print "Average chi squared is", flex.mean( chi_squared_list), "on %d shots" % flex.sum(hist.slots())
def run(argv=None): import libtbx.option_parser if (argv is None): argv = sys.argv command_line = (libtbx.option_parser.option_parser( usage= "%s [-v] [-p poly_mask] [-c circle_mask] [-a avg_max] [-s stddev_max] [-m maxproj_min] [-x mask_pix_val] [-o output] avg_path stddev_path max_path" % libtbx.env.dispatcher_name ).option( None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress" ).option( None, "--poly_mask", "-p", type="string", default=None, dest="poly_mask", help="Polygon to mask out. Comma-seperated string of xy pairs." ).option( None, "--circle_mask", "-c", type="string", default=None, dest="circle_mask", help="Circle to mask out. Comma-seperated string of x, y, and radius." ).option( None, "--avg_max", "-a", type="float", default=2000.0, dest="avg_max", help= "Maximum ADU that pixels in the average image are allowed to have before masked out" ).option( None, "--stddev_max", "-s", type="float", default=10.0, dest="stddev_max", help= "Maximum ADU that pixels in the standard deviation image are allowed to have before masked out" ).option( None, "--maxproj_min", "-m", type="float", default=300.0, dest="maxproj_min", help= "Minimum ADU that pixels in the maximum projection image are allowed to have before masked out" ).option(None, "--mask_pix_val", "-x", type="int", default=-2, dest="mask_pix_val", help="Value for masked out pixels").option( None, "--detector_format_version", "-d", type="string", default=None, dest="detector_format_version", help="detector format version string").option( None, "--output", "-o", type="string", default="mask_.pickle", dest="destpath", help="output file path, should be *.pickle")).process( args=argv[1:]) # Must have exactly three remaining arguments. paths = command_line.args if (len(paths) != 3): command_line.parser.print_usage(file=sys.stderr) return if command_line.options.detector_format_version is None: address = timestamp = None else: from xfel.cxi.cspad_ana.cspad_tbx import evt_timestamp from iotbx.detectors.cspad_detector_formats import address_and_timestamp_from_detector_format_version address, timestamp = address_and_timestamp_from_detector_format_version( command_line.options.detector_format_version) timestamp = evt_timestamp((timestamp, 0)) poly_mask = None if not command_line.options.poly_mask == None: poly_mask = [] poly_mask_tmp = command_line.options.poly_mask.split(",") if len(poly_mask_tmp) % 2 != 0: command_line.parser.print_usage(file=sys.stderr) return odd = True for item in poly_mask_tmp: try: if odd: poly_mask.append(int(item)) else: poly_mask[-1] = (poly_mask[-1], int(item)) except ValueError: command_line.parser.print_usage(file=sys.stderr) return odd = not odd circle_mask = None if command_line.options.circle_mask is not None: circle_mask_tmp = command_line.options.circle_mask.split(",") if len(circle_mask_tmp) != 3: command_line.parser.print_usage(file=sys.stderr) return try: circle_mask = (int(circle_mask_tmp[0]), int(circle_mask_tmp[1]), int(circle_mask_tmp[2])) except ValueError: command_line.parser.print_usage(file=sys.stderr) return avg_path = paths[0] stddev_path = paths[1] max_path = paths[2] # load the three images format_class = Registry.find(avg_path) avg_f = format_class(avg_path) avg_i = avg_f.get_detectorbase() avg_d = avg_i.get_raw_data() stddev_f = format_class(stddev_path) stddev_i = stddev_f.get_detectorbase() stddev_d = stddev_i.get_raw_data() max_f = format_class(max_path) max_i = max_f.get_detectorbase() max_d = max_i.get_raw_data() # first find all the pixels in the average that are less than zero or greater # than a cutoff and set them to the masking value avg_d.set_selected((avg_d <= 0) | (avg_d > command_line.options.avg_max), command_line.options.mask_pix_val) # set all the rest of the pixels to zero. They will be accepted avg_d.set_selected(avg_d != command_line.options.mask_pix_val, 0) # mask out the overly noisy or flat pixels avg_d.set_selected(stddev_d <= 0, command_line.options.mask_pix_val) avg_d.set_selected(stddev_d >= command_line.options.stddev_max, command_line.options.mask_pix_val) # these are the non-bonded pixels avg_d.set_selected(max_d < command_line.options.maxproj_min, command_line.options.mask_pix_val) # calculate the beam center panel = avg_f.get_detector()[0] bcx, bcy = panel.get_beam_centre(avg_f.get_beam().get_s0()) if poly_mask is not None or circle_mask is not None: minx = miny = 0 maxx = avg_d.focus()[0] maxy = avg_d.focus()[1] if poly_mask is not None: minx = min([x[0] for x in poly_mask]) miny = min([y[1] for y in poly_mask]) maxx = max([x[0] for x in poly_mask]) maxy = max([y[1] for y in poly_mask]) if circle_mask is not None: circle_x, circle_y, radius = circle_mask if circle_x - radius < minx: minx = circle_x - radius if circle_y - radius < miny: miny = circle_y - radius if circle_x + radius > maxx: maxx = circle_x + radius if circle_y + radius > maxy: maxy = circle_y + radius sel = avg_d == command_line.options.mask_pix_val for j in range(miny, maxy): for i in range(minx, maxx): idx = j * avg_d.focus()[0] + i if not sel[idx]: if poly_mask is not None and point_in_polygon( (i, j), poly_mask): sel[idx] = True elif circle_mask is not None and point_inside_circle( i, j, circle_x, circle_y, radius): sel[idx] = True avg_d.set_selected(sel, command_line.options.mask_pix_val) # have to re-layout the data to match how it was stored originally shifted_int_data_old = avg_d shifted_int_data_new = shifted_int_data_old.__class__( flex.grid(shifted_int_data_old.focus())) shifted_int_data_new += command_line.options.mask_pix_val phil = avg_i.horizons_phil_cache manager = avg_i.get_tile_manager(phil) for i, shift in enumerate(manager.effective_translations()): shift_slow = shift[0] shift_fast = shift[1] ur_slow = phil.distl.detector_tiling[4 * i + 0] + shift_slow ur_fast = phil.distl.detector_tiling[4 * i + 1] + shift_fast ll_slow = phil.distl.detector_tiling[4 * i + 2] + shift_slow ll_fast = phil.distl.detector_tiling[4 * i + 3] + shift_fast #print "Shifting tile at (%d, %d) by (%d, %d)" % (ur_slow-shift_slow, ur_fast-shift_fast, -shift_slow, -shift_fast) shifted_int_data_new.matrix_paste_block_in_place( block=shifted_int_data_old.matrix_copy_block( i_row=ur_slow, i_column=ur_fast, n_rows=ll_slow - ur_slow, n_columns=ll_fast - ur_fast), i_row=ur_slow - shift_slow, i_column=ur_fast - shift_fast) d = dpack(active_areas=avg_i.parameters['ACTIVE_AREAS'], address=address, beam_center_x=bcx, beam_center_y=bcy, data=shifted_int_data_new, distance=avg_i.distance, timestamp=timestamp, wavelength=avg_i.wavelength, xtal_target=None, pixel_size=avg_i.pixel_size, saturated_value=avg_i.saturation) dwritef2(d, command_line.options.destpath) #the minimum number of pixels to mask out cooresponding to the interstitial regions for the CS-PAD min_count = 818265 # (1765 * 1765) - (194 * 185 * 64) masked_out = len(avg_d.as_1d().select( (avg_d == command_line.options.mask_pix_val).as_1d())) assert masked_out >= min_count print "Masked out %d pixels out of %d (%.2f%%)"% \ (masked_out-min_count,len(avg_d)-min_count,(masked_out-min_count)*100/(len(avg_d)-min_count))
class SingleImage(object): def __init__(self, img, init, verbose=True, imported_grid=None): """ Constructor for the SingleImage object using a raw image file or pickle """ # Initialize parameters self.params = init.params self.args = init.args self.raw_img = img[2] self.conv_img = img[2] self.img_index = img[0] self.status = None self.fail = None self.final = None self.log_info = [] self.gs_results = [] self.main_log = init.logfile self.verbose = verbose self.hmed = self.params.cctbx.grid_search.height_median self.amed = self.params.cctbx.grid_search.area_median self.input_base = init.input_base self.conv_base = init.conv_base self.int_base = init.int_base self.obj_base = init.obj_base self.fin_base = init.fin_base self.viz_base = init.viz_base self.tmp_base = init.tmp_base self.abort_file = os.path.join(self.int_base, '.abort.tmp') self.obj_path = None self.obj_file = None self.fin_path = None self.fin_file = None self.viz_path = None # ============================== SELECTION-ONLY FUNCTIONS ============================== # def import_int_file(self, init): """ Replaces path settings in imported image object with new settings NEED TO RE-DO LATER """ if os.path.isfile(self.abort_file): self.fail = 'aborted' return self # Generate paths to output files self.params = init.params self.main_log = init.logfile self.input_base = init.input_base self.conv_base = init.conv_base self.int_base = init.int_base self.obj_base = init.obj_base self.fin_base = init.fin_base self.viz_base = init.viz_base self.obj_path = misc.make_image_path(self.conv_img, self.input_base, self.obj_base) self.obj_file = os.path.abspath( os.path.join( self.obj_path, os.path.basename(self.conv_img).split('.')[0] + ".int")) self.fin_path = misc.make_image_path(self.conv_img, self.input_base, self.fin_base) self.fin_file = os.path.abspath( os.path.join( self.fin_path, os.path.basename(self.conv_img).split('.')[0] + "_int.pickle")) self.final['final'] = self.fin_file self.final['img'] = self.conv_img self.viz_path = misc.make_image_path(self.conv_img, self.input_base, self.viz_base) self.viz_file = os.path.join( self.viz_path, os.path.basename(self.conv_img).split('.')[0] + "_int.png") # Create actual folders (if necessary) try: if not os.path.isdir(self.obj_path): os.makedirs(self.obj_path) if not os.path.isdir(self.fin_path): os.makedirs(self.fin_path) if not os.path.isdir(self.viz_path): os.makedirs(self.viz_path) except OSError: pass # Grid search / integration log file self.int_log = os.path.join( self.fin_path, os.path.basename(self.conv_img).split('.')[0] + '.tmp') # Reset status to 'grid search' to pick up at selection (if no fail) if self.fail == None: self.status = 'bypass grid search' return self def determine_gs_result_file(self): """ For 'selection-only' cctbx.xfel runs, determine where the image objects are """ if self.params.cctbx.selection.select_only.grid_search_path != None: obj_path = os.path.abspath( self.params.cctbx.selection.select_only.grid_search_path) else: run_number = int(os.path.basename(self.int_base)) - 1 obj_path = "{}/integration/{:03d}/image_objects"\ "".format(os.path.abspath(os.curdir), run_number) gs_result_file = os.path.join(obj_path, os.path.basename(self.obj_file)) return gs_result_file # =============================== IMAGE IMPORT FUNCTIONS =============================== # def load_image(self): """ Reads raw image file and extracts data for conversion into pickle format. Also estimates gain if turned on.""" # Load raw image or image pickle try: with misc.Capturing() as junk_output: loaded_img = dxtbx.load(self.raw_img) except IOError, e: loaded_img = None pass # Extract image information if loaded_img is not None: raw_data = loaded_img.get_raw_data() detector = loaded_img.get_detector()[0] beam = loaded_img.get_beam() scan = loaded_img.get_scan() distance = detector.get_distance() pixel_size = detector.get_pixel_size()[0] overload = detector.get_trusted_range()[1] wavelength = beam.get_wavelength() beam_x = detector.get_beam_centre(beam.get_s0())[0] beam_y = detector.get_beam_centre(beam.get_s0())[1] if scan is None: timestamp = None img_type = 'pickle' else: img_type = 'raw' msec, sec = math.modf(scan.get_epochs()[0]) timestamp = evt_timestamp((sec, msec)) # Assemble datapack data = dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data['OSC_START'] = 0 #osc_start data['OSC_RANGE'] = 0 #osc_start data['TIME'] = scan.get_exposure_times()[0] else: data = None img_type = 'not imported' # Estimate gain (or set gain to 1.00 if cannot calculate) # Cribbed from estimate_gain.py by Richard Gildea if self.params.advanced.estimate_gain: try: from dials.algorithms.image.threshold import KabschDebug raw_data = [raw_data] gain_value = 1 kernel_size = (10, 10) gain_map = [ flex.double(raw_data[i].accessor(), gain_value) for i in range(len(loaded_img.get_detector())) ] mask = loaded_img.get_mask() min_local = 0 # dummy values, shouldn't affect results: REPLACE WITH SETTINGS! nsigma_b = 6 nsigma_s = 3 global_threshold = 0 kabsch_debug_list = [] for i_panel in range(len(loaded_img.get_detector())): kabsch_debug_list.append( KabschDebug(raw_data[i_panel].as_double(), mask[i_panel], gain_map[i_panel], kernel_size, nsigma_b, nsigma_s, global_threshold, min_local)) dispersion = flex.double() for kabsch in kabsch_debug_list: dispersion.extend( kabsch.coefficient_of_variation().as_1d()) sorted_dispersion = flex.sorted(dispersion) from libtbx.math_utils import nearest_integer as nint q1 = sorted_dispersion[nint(len(sorted_dispersion) / 4)] q2 = sorted_dispersion[nint(len(sorted_dispersion) / 2)] q3 = sorted_dispersion[nint(len(sorted_dispersion) * 3 / 4)] iqr = q3 - q1 inlier_sel = (sorted_dispersion > (q1 - 1.5 * iqr)) & (sorted_dispersion < (q3 + 1.5 * iqr)) sorted_dispersion = sorted_dispersion.select(inlier_sel) self.gain = sorted_dispersion[nint(len(sorted_dispersion) / 2)] except IndexError: self.gain = 1.0 else: self.gain = 1.0 return data, img_type
def run(argv=None): import libtbx.option_parser if (argv is None): argv = sys.argv command_line = (libtbx.option_parser.option_parser( usage="%s [-v] [-p poly_mask] [-c circle_mask] [-a avg_max] [-s stddev_max] [-m maxproj_min] [-x mask_pix_val] [-o output] avg_path stddev_path max_path" % libtbx.env.dispatcher_name) .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") .option(None, "--poly_mask", "-p", type="string", default=None, dest="poly_mask", help="Polygon to mask out. Comma-seperated string of xy pairs.") .option(None, "--circle_mask", "-c", type="string", default=None, dest="circle_mask", help="Circle to mask out. Comma-seperated string of x, y, and radius.") .option(None, "--avg_max", "-a", type="float", default=2000.0, dest="avg_max", help="Maximum ADU that pixels in the average image are allowed to have before masked out") .option(None, "--stddev_max", "-s", type="float", default=10.0, dest="stddev_max", help="Maximum ADU that pixels in the standard deviation image are allowed to have before masked out") .option(None, "--maxproj_min", "-m", type="float", default=300.0, dest="maxproj_min", help="Minimum ADU that pixels in the maximum projection image are allowed to have before masked out") .option(None, "--mask_pix_val", "-x", type="int", default=-2, dest="mask_pix_val", help="Value for masked out pixels") .option(None, "--detector_format_version", "-d", type="string", default=None, dest="detector_format_version", help="detector format version string") .option(None, "--output", "-o", type="string", default="mask_.pickle", dest="destpath", help="output file path, should be *.pickle") ).process(args=argv[1:]) # Must have exactly three remaining arguments. paths = command_line.args if (len(paths) != 3): command_line.parser.print_usage(file=sys.stderr) return if command_line.options.detector_format_version is None: address = timestamp = None else: from xfel.cxi.cspad_ana.cspad_tbx import evt_timestamp from xfel.detector_formats import address_and_timestamp_from_detector_format_version address, timestamp = address_and_timestamp_from_detector_format_version(command_line.options.detector_format_version) timestamp = evt_timestamp((timestamp,0)) poly_mask = None if not command_line.options.poly_mask == None: poly_mask = [] poly_mask_tmp = command_line.options.poly_mask.split(",") if len(poly_mask_tmp) % 2 != 0: command_line.parser.print_usage(file=sys.stderr) return odd = True for item in poly_mask_tmp: try: if odd: poly_mask.append(int(item)) else: poly_mask[-1] = (poly_mask[-1],int(item)) except ValueError: command_line.parser.print_usage(file=sys.stderr) return odd = not odd circle_mask = None if command_line.options.circle_mask is not None: circle_mask_tmp = command_line.options.circle_mask.split(",") if len(circle_mask_tmp) != 3: command_line.parser.print_usage(file=sys.stderr) return try: circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2])) except ValueError: command_line.parser.print_usage(file=sys.stderr) return avg_path = paths[0] stddev_path = paths[1] max_path = paths[2] # load the three images format_class = Registry.find(avg_path) avg_f = format_class(avg_path) avg_i = avg_f.get_detectorbase() avg_d = avg_i.get_raw_data() stddev_f = format_class(stddev_path) stddev_i = stddev_f.get_detectorbase() stddev_d = stddev_i.get_raw_data() max_f = format_class(max_path) max_i = max_f.get_detectorbase() max_d = max_i.get_raw_data() # first find all the pixels in the average that are less than zero or greater # than a cutoff and set them to the masking value avg_d.set_selected((avg_d <= 0) | (avg_d > command_line.options.avg_max), command_line.options.mask_pix_val) # set all the rest of the pixels to zero. They will be accepted avg_d.set_selected(avg_d != command_line.options.mask_pix_val, 0) # mask out the overly noisy or flat pixels avg_d.set_selected(stddev_d <= 0, command_line.options.mask_pix_val) avg_d.set_selected(stddev_d >= command_line.options.stddev_max, command_line.options.mask_pix_val) # these are the non-bonded pixels avg_d.set_selected(max_d < command_line.options.maxproj_min, command_line.options.mask_pix_val) # calculate the beam center panel = avg_f.get_detector()[0] bcx, bcy = panel.get_beam_centre(avg_f.get_beam().get_s0()) if poly_mask is not None or circle_mask is not None: minx = miny = 0 maxx = avg_d.focus()[0] maxy = avg_d.focus()[1] if poly_mask is not None: minx = min([x[0] for x in poly_mask]) miny = min([y[1] for y in poly_mask]) maxx = max([x[0] for x in poly_mask]) maxy = max([y[1] for y in poly_mask]) if circle_mask is not None: circle_x, circle_y, radius = circle_mask if circle_x - radius < minx: minx = circle_x - radius if circle_y - radius < miny: miny = circle_y - radius if circle_x + radius > maxx: maxx = circle_x + radius if circle_y + radius > maxy: maxy = circle_y + radius sel = avg_d == command_line.options.mask_pix_val for j in xrange(miny, maxy): for i in xrange(minx, maxx): idx = j * avg_d.focus()[0] + i if not sel[idx]: if poly_mask is not None and point_in_polygon((i,j),poly_mask): sel[idx] = True elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius): sel[idx] = True avg_d.set_selected(sel,command_line.options.mask_pix_val) # have to re-layout the data to match how it was stored originally shifted_int_data_old = avg_d shifted_int_data_new = shifted_int_data_old.__class__( flex.grid(shifted_int_data_old.focus())) shifted_int_data_new += command_line.options.mask_pix_val phil = avg_i.horizons_phil_cache manager = avg_i.get_tile_manager(phil) for i,shift in enumerate(manager.effective_translations()): shift_slow = shift[0] shift_fast = shift[1] ur_slow = phil.distl.detector_tiling[4 * i + 0] + shift_slow ur_fast = phil.distl.detector_tiling[4 * i + 1] + shift_fast ll_slow = phil.distl.detector_tiling[4 * i + 2] + shift_slow ll_fast = phil.distl.detector_tiling[4 * i + 3] + shift_fast #print "Shifting tile at (%d, %d) by (%d, %d)" % (ur_slow-shift_slow, ur_fast-shift_fast, -shift_slow, -shift_fast) shifted_int_data_new.matrix_paste_block_in_place( block = shifted_int_data_old.matrix_copy_block( i_row=ur_slow,i_column=ur_fast, n_rows=ll_slow-ur_slow, n_columns=ll_fast-ur_fast), i_row = ur_slow - shift_slow, i_column = ur_fast - shift_fast ) d = dpack( active_areas=avg_i.parameters['ACTIVE_AREAS'], address=address, beam_center_x=bcx, beam_center_y=bcy, data=shifted_int_data_new, distance=avg_i.distance, timestamp=timestamp, wavelength=avg_i.wavelength, xtal_target=None, pixel_size=avg_i.pixel_size, saturated_value=avg_i.saturation) dwritef2(d, command_line.options.destpath) #the minimum number of pixels to mask out cooresponding to the interstitial regions for the CS-PAD min_count = 818265 # (1765 * 1765) - (194 * 185 * 64) masked_out = len(avg_d.as_1d().select((avg_d == command_line.options.mask_pix_val).as_1d())) assert masked_out >= min_count print "Masked out %d pixels out of %d (%.2f%%)"% \ (masked_out-min_count,len(avg_d)-min_count,(masked_out-min_count)*100/(len(avg_d)-min_count))
break except KeyError, ValueError: n_fails += 1 print "Fail to read, attempt number", n_fails if n_fails > 100: raise Exception("Couldn't read the data") import time time.sleep(n_fails * 0.1) imgdict = cspad_tbx.dpack( data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, address="Sacla.MPCCD.8tile", active_areas=active_areas, ) imgdict = crop_image_pickle(imgdict, preserve_active_areas_even_though_cropping_would_invalidate_them=True) dest_path = os.path.join(dest_dir, dest_base + "_%06d.pickle" % img_no) print "Saving image", img_no, "to", dest_path easy_pickle.dump(dest_path, imgdict) easy_mp.pool_map(args=range(data.get_num_images()), func=do_work, processes=4)
quad_asics.append(numpy.concatenate((a,b),axis=1)) quad_data = numpy.dstack(quad_asics) quad_data = numpy.rollaxis(quad_data, 2,0) data3d.append(fake_cspad_ElementV2(quad_data, i_quad)) env = fake_env(fake_config()) evt = fake_evt(data3d) beam_center, active_areas = cbcaa(fake_config(),sections) data = flex.int(CsPadDetector(address, evt, env, sections).astype(numpy.float64)) img_dict = dpack( active_areas=active_areas, address=address, beam_center_x=beam_center[0]*pixel_size, beam_center_y=beam_center[1]*pixel_size, data=data, distance=params.distance, pixel_size=pixel_size, timestamp=timestamp, wavelength=params.wavelength) img = NpyImage("", source_data=img_dict) args = ["distl.detector_format_version=%s"%params.detector_format_version] horizons_phil = cxi_phil.cxi_versioned_extract(args) img.readHeader(horizons_phil) img.translate_tiles(horizons_phil) tm = img.get_tile_manager(horizons_phil) effective_active_areas = tm.effective_tiling_as_flex_int()
def event(self, evt, env): """The event() function is called for every L1Accept transition. XXX more? Previously, common-mode correction was applied only after initial threshold filtering. Since the common_mode class applies the (lengthy) common-mode correction immediately after reading the image from the stream, this optimisation is currently not (elegantly) doable. @param evt Event data object, a configure object @param env Environment object """ super(mod_hitfind, self).event(evt, env) if (evt.get("skip_event")): return # This module only applies to detectors for which a distance is # available. distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: self.nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), "skip_event") return device = cspad_tbx.address_split(self.address)[2] # ***** HITFINDING ***** XXX For hitfinding it may be interesting # to look at the fraction of subzero pixels in the dark-corrected # image. if (self.m_threshold is not None): # If a threshold value is given it can be applied in one of three ways: # 1. Apply it over the whole image if (self.m_roi is None and self.m_distl_min_peaks is None): vmax = flex.max(self.cspad_img) if (vmax < self.m_threshold): if not self.m_negate_hits: # Tell downstream modules to skip this event if the threshold was not met. evt.put(skip_event_flag(), "skip_event") return elif self.m_negate_hits: evt.put(skip_event_flag(), "skip_event") return # 2. Apply threshold over a rectangular region of interest. elif (self.m_roi is not None): vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3], self.m_roi[0]:self.m_roi[1]]) if (vmax < self.m_threshold): if not self.m_negate_hits: evt.put(skip_event_flag(), "skip_event") return elif self.m_negate_hits: evt.put(skip_event_flag(), "skip_event") return # 3. Determine the spotfinder spots within the central ASICS, and accept the # image as a hit if there are m_distl_min_peaks exceeding m_threshold. # As a further requirement, the peaks must exceed 2.5 * the 90-percentile # pixel value of the central ASICS. This filter was added to avoid high-background # false positives. elif (self.m_distl_min_peaks is not None): if device == 'marccd': self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0] self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1] elif device == 'Rayonix': self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0] self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1] peak_heights,outvalue = self.distl_filter( self.address, self.cspad_img.iround(), # XXX correct? distance, self.timestamp, self.wavelength) if ('permissive' in self.m_distl_flags): number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True) else: number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True) sec,ms = cspad_tbx.evt_time(evt) evt_time = sec + ms/1000 self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks)) skip_event = False if number_of_accepted_peaks < self.m_distl_min_peaks: self.logger.info("Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d" %( env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold)) if not self.m_negate_hits: skip_event = True else: self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %( env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold)) if self.m_negate_hits: skip_event = True if skip_event: if self.m_db_logging: # log misses to the database self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance, self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags)) evt.put(skip_event_flag(), "skip_event") return # the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this # hit will not be logged in the db. evt.put(number_of_accepted_peaks, 'sfspots') self.logger.info("Subprocess %02d: process image #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) # See r17537 of mod_average.py. if device == 'Cspad': pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value elif device == 'marccd': pixel_size = evt.get("marccd_pixel_size") saturated_value = evt.get("marccd_saturated_value") elif device == 'Rayonix': pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) saturated_value = rayonix_tbx.rayonix_saturated_value d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=self.cspad_img.iround(), # XXX ouch! distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=self.timestamp, wavelength=self.wavelength, xtal_target=self.m_xtal_target) if (self.m_dispatch == "index"): import sys from xfel.cxi.integrate_image_api import integrate_one_image info = integrate_one_image(d, integration_dirname = self.m_integration_dirname, integration_basename = self.m_integration_basename) sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ indexed = info is not None if indexed and self.m_progress_logging: # integration pickle dictionary is available here as info.last_saved_best if info.last_saved_best["identified_isoform"] is not None: #print info.last_saved_best.keys() from cxi_xdr_xes.cftbx.cspad_ana import db dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password) cursor = dbobj.cursor() if info.last_saved_best["identified_isoform"] in self.isoforms: PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]] else: from xfel.xpp.progress_support import progress_manager PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run()) indices, miller_id = PM.get_HKL(cursor) # cache these as they don't change for a given isoform self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id if self.m_sql_buffer_size > 1: self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False)) else: PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True) dbobj.commit() cursor.close() dbobj.close() if self.m_db_logging: sec,ms = cspad_tbx.evt_time(evt) evt_time = sec + ms/1000 sfspots = evt.get('sfspots') if sfspots is None: if indexed: n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total']) else: n_spots = 0 else: n_spots = sfspots if indexed: mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0] mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0] ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0] obs = info.last_saved_best['observations'][0] cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters() pointgroup = info.last_saved_best['pointgroup'] resolution = obs.d_min() else: mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \ cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0 self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance, self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation, mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution, self.m_db_tags)) if (not indexed): evt.put(skip_event_flag(), "skip_event") return elif (self.m_dispatch == "nop"): pass elif (self.m_dispatch == "view"): #interactive image viewer args = ["indexing.data=dummy"] detector_format_version = detector_format_function( self.address, evt.GetTime()) if detector_format_version is not None: args += ["distl.detector_format_version=%" % detector_format_version] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(self.m_xtal_target, args) horizons_phil.indexing.data = d from xfel.cxi import display_spots display_spots.parameters.horizons_phil = horizons_phil display_spots.wrapper_of_callback().display(horizons_phil.indexing.data) elif (self.m_dispatch == "spots"): #interactive spotfinder viewer args = ["indexing.data=dummy"] detector_format_version = detector_format_function( self.address, evt.GetTime()) if detector_format_version is not None: args += ["distl.detector_format_version=%s" % detector_format_version] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(self.m_xtal_target, args) horizons_phil.indexing.data = d from xfel.cxi import display_spots display_spots.parameters.horizons_phil = horizons_phil from rstbx.new_horizons.index import pre_indexing_validation,pack_names pre_indexing_validation(horizons_phil) imagefile_arguments = pack_names(horizons_phil) horizons_phil.persist.show() from spotfinder.applications import signal_strength info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments) work = display_spots.wrapper_of_callback(info) work.display_with_callback(horizons_phil.indexing.data) elif (self.m_dispatch == "write_dict"): self.logger.warning( "event(): deprecated dispatch 'write_dict', use mod_dump instead") if (self.m_out_dirname is not None or self.m_out_basename is not None): cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename) # Diagnostic message emitted only when all the processing is done. if (env.subprocess() >= 0): self.logger.info("Subprocess %02d: accepted #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) else: self.logger.info("Accepted #%05d @ %s" % (self.nshots, self.timestamp))
def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ super(average_mixin, self).event(evt, env) if evt.get('skip_event'): return # Get the distance for the detectors that should have it, and set # it to NaN for those that should not. if self.detector == 'CxiDs1' or \ self.detector == 'CxiDs2' or \ self.detector == 'CxiDsd' or \ self.detector == 'XppGon': distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: self._nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), 'skip_event') return else: distance = float('nan') if ("skew" in self.flags): # Take out inactive pixels if self.roi is not None: pixels = self.cspad_img[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]] dark_mask = self.dark_mask[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]] pixels = pixels.as_1d().select(dark_mask.as_1d()) else: pixels = self.cspad_img.as_1d().select(self.dark_mask.as_1d()).as_double() stats = scitbx.math.basic_statistics(pixels.as_double()) #stats.show() self.logger.info("skew: %.3f" %stats.skew) self.logger.info("kurtosis: %.3f" %stats.kurtosis) if 0: from matplotlib import pyplot hist_min, hist_max = flex.min(flex_cspad_img.as_double()), flex.max(flex_cspad_img.as_double()) print hist_min, hist_max n_slots = 100 n, bins, patches = pyplot.hist(flex_cspad_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max)) pyplot.show() # XXX This skew threshold probably needs fine-tuning skew_threshold = 0.35 if stats.skew < skew_threshold: self._nfail += 1 self.logger.warning("event(): skew < %f, shot skipped" % skew_threshold) evt.put(skip_event_flag(), 'skip_event') return #self.cspad_img *= stats.skew if ("inactive" in self.flags): self.cspad_img.set_selected(self.dark_stddev <= 0, 0) if ("noelastic" in self.flags): ELASTIC_THRESHOLD = self.elastic_threshold self.cspad_img.set_selected(self.cspad_img > ELASTIC_THRESHOLD, 0) if self.hot_threshold is not None: HOT_THRESHOLD = self.hot_threshold self.cspad_img.set_selected(self.dark_img > HOT_THRESHOLD, 0) if self.gain_map is not None and self.gain_threshold is not None: # XXX comparing each pixel to a moving average would probably be better # since the gain should vary approximately smoothly over different areas # of the detector GAIN_THRESHOLD = self.gain_threshold #self.logger.debug( #"rejecting: %i" %(self.gain_map > GAIN_THRESHOLD).count(True)) self.cspad_img.set_selected(self.gain_map > GAIN_THRESHOLD, 0) if ("nonoise" in self.flags): NOISE_THRESHOLD = self.noise_threshold self.cspad_img.set_selected(self.cspad_img < NOISE_THRESHOLD, 0) if ("sigma_scaling" in self.flags): self.do_sigma_scaling() if ("symnoise" in self.flags): SYMNOISE_THRESHOLD = self.symnoise_threshold self.cspad_img.set_selected((-SYMNOISE_THRESHOLD < self.cspad_img) & ( self.cspad_img < SYMNOISE_THRESHOLD), 0) if ("output" in self.flags): try: import cPickle as pickle except ImportError: import pickle import os if (not os.path.isdir(self.pickle_dirname)): os.makedirs(self.pickle_dirname) flexdata = flex.int(self.cspad_img.astype(numpy.int32)) d = cspad_tbx.dpack( address=self.address, data=flexdata, timestamp=cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) ) G = open(os.path.join(".",self.pickle_dirname)+"/"+self.pickle_basename, "ab") pickle.dump(d,G,pickle.HIGHEST_PROTOCOL) G.close() if self.photon_threshold is not None and self.two_photon_threshold is not None: self.do_photon_counting() if self.background_path is not None: self.cspad_img -= self.background_img # t and self._sum_time are a two-long arrays of seconds and # milliseconds which hold time with respect to the base time. t = [t1 - t2 for (t1, t2) in zip(cspad_tbx.evt_time(evt), self._metadata['time_base'])] if self._nmemb == 0: # The peers metadata item is a bit field where a bit is set if # the partial sum from the corresponding worker process is # pending. If this is the first frame a worker process sees, # set its corresponding bit in the bit field since it will # contribute a partial sum. if env.subprocess() >= 0: self._lock.acquire() if 'peers' in self._metadata.keys(): self._metadata['peers'] |= (1 << env.subprocess()) else: self._metadata['peers'] = (1 << env.subprocess()) self._lock.release() self._sum_distance = distance self._sum_time = (t[0], t[1]) self._sum_wavelength = self.wavelength if self._have_max: self._max_img = self.cspad_img.deep_copy() if self._have_mean: self._sum_img = self.cspad_img.deep_copy() if self._have_std: self._ssq_img = flex.pow2(self.cspad_img) else: self._sum_distance += distance self._sum_time = (self._sum_time[0] + t[0], self._sum_time[1] + t[1]) self._sum_wavelength += self.wavelength if self._have_max: sel = (self.cspad_img > self._max_img).as_1d() self._max_img.as_1d().set_selected( sel, self.cspad_img.as_1d().select(sel)) if self._have_mean: self._sum_img += self.cspad_img if self._have_std: self._ssq_img += flex.pow2(self.cspad_img) self._nmemb += 1
try: raw_data = data.get_raw_data(img_no) break except KeyError, ValueError: n_fails += 1 print "Fail to read, attempt number", n_fails if n_fails > 100: raise Exception("Couldn't read the data") import time time.sleep(n_fails * 0.1) imgdict = cspad_tbx.dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, address="Sacla.MPCCD.8tile", active_areas=active_areas) imgdict = crop_image_pickle( imgdict, preserve_active_areas_even_though_cropping_would_invalidate_them=True) dest_path = os.path.join(dest_dir, dest_base + "_%06d.pickle" % img_no) print "Saving image", img_no, "to", dest_path easy_pickle.dump(dest_path, imgdict) easy_mp.pool_map(args=range(data.get_num_images()), func=do_work, processes=4)
def load_image(self): """ Reads raw image file and extracts data for conversion into pickle format. Also estimates gain if turned on.""" # Load raw image or image pickle try: with misc.Capturing() as junk_output: loaded_img = dxtbx.load(self.raw_img) except IOError: loaded_img = None pass # Extract image information if loaded_img is not None: raw_data = loaded_img.get_raw_data() detector = loaded_img.get_detector()[0] beam = loaded_img.get_beam() scan = loaded_img.get_scan() distance = detector.get_distance() pixel_size = detector.get_pixel_size()[0] overload = detector.get_trusted_range()[1] wavelength = beam.get_wavelength() beam_x = detector.get_beam_centre(beam.get_s0())[0] beam_y = detector.get_beam_centre(beam.get_s0())[1] if scan is None: timestamp = None if abs(beam_x - beam_y) <= 0.1 or self.params.image_conversion.square_mode == "None": img_type = 'converted' else: img_type = 'unconverted' else: msec, sec = math.modf(scan.get_epochs()[0]) timestamp = evt_timestamp((sec,msec)) if self.params.image_conversion.beamstop != 0 or\ self.params.image_conversion.beam_center.x != 0 or\ self.params.image_conversion.beam_center.y != 0 or\ self.params.image_conversion.rename_pickle_prefix != 'Auto' or\ self.params.image_conversion.rename_pickle_prefix != None: img_type = 'unconverted' # Assemble datapack data = dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp ) #print "data: ", type(raw_data) #print "pixel size: ", type(pixel_size) #print 'wavelength: ', type(wavelength) #print "beamX: ", type(beam_x) #print "saturation: ", type(overload) #print "timestamp: ", type(timestamp) #for i in dir(raw_data): print i #exit() if scan is not None: osc_start, osc_range = scan.get_oscillation() img_type = 'unconverted' if osc_start != osc_range: data['OSC_START'] = osc_start data['OSC_RANGE'] = osc_range data['TIME'] = scan.get_exposure_times()[0] # Estimate gain (or set gain to 1.00 if cannot calculate) # Cribbed from estimate_gain.py by Richard Gildea if self.params.advanced.estimate_gain: try: from dials.algorithms.image.threshold import KabschDebug raw_data = [raw_data] gain_value = 1 kernel_size=(10,10) gain_map = [flex.double(raw_data[i].accessor(), gain_value) for i in range(len(loaded_img.get_detector()))] mask = loaded_img.get_mask() min_local = 0 # dummy values, shouldn't affect results nsigma_b = 6 nsigma_s = 3 global_threshold = 0 kabsch_debug_list = [] for i_panel in range(len(loaded_img.get_detector())): kabsch_debug_list.append( KabschDebug( raw_data[i_panel].as_double(), mask[i_panel], gain_map[i_panel], kernel_size, nsigma_b, nsigma_s, global_threshold, min_local)) dispersion = flex.double() for kabsch in kabsch_debug_list: dispersion.extend(kabsch.coefficient_of_variation().as_1d()) sorted_dispersion = flex.sorted(dispersion) from libtbx.math_utils import nearest_integer as nint q1 = sorted_dispersion[nint(len(sorted_dispersion)/4)] q2 = sorted_dispersion[nint(len(sorted_dispersion)/2)] q3 = sorted_dispersion[nint(len(sorted_dispersion)*3/4)] iqr = q3-q1 inlier_sel = (sorted_dispersion > (q1 - 1.5*iqr)) & (sorted_dispersion < (q3 + 1.5*iqr)) sorted_dispersion = sorted_dispersion.select(inlier_sel) self.gain = sorted_dispersion[nint(len(sorted_dispersion)/2)] except IndexError: self.gain = 1.0 else: self.gain = 1.0 else: data = None return data, img_type
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ import libtbx.load_env from libtbx import easy_pickle, option_parser from scitbx.array_family import flex from xfel.cxi.cspad_ana import cspad_tbx from iotbx.detectors.cspad_detector_formats import reverse_timestamp if argv is None: argv = sys.argv command_line = (option_parser.option_parser( usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \ "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name) .option(None, "--average-path", "-a", type="string", default=None, dest="avg_path", metavar="PATH", help="Write average image to PATH") .option(None, "--maximum-path", "-m", type="string", default=None, dest="max_path", metavar="PATH", help="Write maximum projection image to PATH") .option(None, "--stddev-path", "-s", type="string", default=None, dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") ).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 # Loop over all images and accumulate statistics. nfail = 0 nmemb = 0 for path in paths: if command_line.options.verbose: sys.stdout.write("Processing %s...\n" % path) try: # Promote the image to double-precision floating point type. # All real-valued flex arrays have the as_double() function. d = easy_pickle.load(path) distance = d['DISTANCE'] img = d['DATA'].as_1d().as_double() wavelength = d['WAVELENGTH'] time_tuple = reverse_timestamp(d['TIMESTAMP']) # Warn if the header items across the set of images do not match # up. Note that discrepancies regarding the image size are # fatal. if 'active_areas' in locals(): if (active_areas != d['ACTIVE_AREAS']).count(True) != 0: sys.stderr.write("Active areas do not match\n") else: active_areas = d['ACTIVE_AREAS'] if 'beam_center' in locals(): if beam_center != (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']): sys.stderr.write("Beam centers do not match\n") else: beam_center = (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']) if 'detector_address' in locals(): if detector_address != d['DETECTOR_ADDRESS']: sys.stderr.write("Detector addresses do not match\n") else: detector_address = d['DETECTOR_ADDRESS'] if 'saturated_value' in locals(): if saturated_value != d['SATURATED_VALUE']: sys.stderr.write("Saturated values do not match\n") else: saturated_value = d['SATURATED_VALUE'] if 'size' in locals(): if size != (d['SIZE1'], d['SIZE2']): sys.stderr.write("Image sizes do not match\n") return 1 else: size = (d['SIZE1'], d['SIZE2']) if size != d['DATA'].focus(): sys.stderr.write("Image size does not match pixel array\n") return 1 if 'pixel_size' in locals(): if pixel_size != d['PIXEL_SIZE']: sys.stderr.write("Pixel sizes do not match\n") return 1 else: if 'PIXEL_SIZE' in d: pixel_size = d['PIXEL_SIZE'] else: pixel_size = None except Exception: try: # Fall back on reading the image with dxtbx, and shoehorn the # extracted information into what would have been found in a # pickle file. XXX This code assumes a monolithic detector! from dxtbx.format.Registry import Registry format_class = Registry.find(path) i = format_class(path) beam = i.get_beam() assert len(i.get_detector()) == 1 detector = i.get_detector()[0] beam_center = detector.get_beam_centre(beam.get_s0()) detector_address = format_class.__name__ distance = detector.get_distance() img = i.get_raw_data().as_1d().as_double() pixel_size = 0.5 * sum(detector.get_pixel_size()) saturated_value = int(round(detector.get_trusted_range()[1])) size = detector.get_image_size() time_tuple = (i.get_scan().get_epochs()[0], 0) wavelength = beam.get_wavelength() active_areas = flex.int((0, 0, size[0], size[1])) except Exception: nfail += 1 continue # See also event() in xfel.cxi.cspad_ana.average_tbx. Record the # base time as the timestamp of the first image. # # The sum-of-squares image is accumulated using long integers, as # this delays the point where overflow occurs. But really, this # is just a band-aid... if nmemb == 0: max_img = img.deep_copy() sum_distance = distance sum_img = img.deep_copy() ssq_img = flex.pow2(img) sum_wavelength = wavelength sum_time = (0, 0) time_base = time_tuple else: sel = (img > max_img).as_1d() max_img.set_selected(sel, img.select(sel)) sum_distance += distance sum_img += img ssq_img += flex.pow2(img) sum_wavelength += wavelength sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]), sum_time[1] + (time_tuple[1] - time_base[1])) nmemb += 1 # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = sum_img.as_double() / nmemb avg_distance = sum_distance / nmemb avg_timestamp = cspad_tbx.evt_timestamp( (time_base[0] + int(round(sum_time[0] / nmemb)), time_base[1] + int(round(sum_time[1] / nmemb)))) avg_wavelength = sum_wavelength / nmemb # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: avg_img.resize(flex.grid(size[0], size[1])) d = cspad_tbx.dpack( active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=avg_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.avg_path, d) if command_line.options.max_path is not None: max_img.resize(flex.grid(size[0], size[1])) d = cspad_tbx.dpack( active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=max_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.max_path, d) if command_line.options.stddev_path is not None: stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. stddev_img.set_selected(stddev_img < 0, 0) if nmemb == 1: stddev_img = flex.sqrt(stddev_img) else: stddev_img = flex.sqrt(stddev_img / (nmemb - 1)) stddev_img.resize(flex.grid(size[0], size[1])) d = cspad_tbx.dpack( active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=stddev_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.stddev_path, d) return 0
def HitFinder(IO,XSetup,HFParams,Frelon,DataCorr,AI,index): hit=0 fname=IO.fname_list[index] peaks=[] peakslist=[] try : img = fabio.open(fname) except: print 'Warning : problem while opening file %s, file skiped '%fname return if img.data.shape == Frelon.resolution: #Apply the dark, flatfield and distortion correction (as specified by the user) img.data=DataCorr.apply_correction(img.data,HFParams.DoDarkCorr,HFParams.DoFlatCorr,HFParams.DoDist) #Remove beam stop area (i.e = 0) extend=15 #BkgCorr with pyFAI Azimuthal Integrator working=AI.ai.separate(img.data,npt_rad=1024, npt_azim=512, unit="2th_deg",percentile=50, mask=AI.mask,restore_mask=False)[0] imgmax,imgmin,imgmed = np.max(working), np.min(working), np.median(working) working[XSetup.beam_y-extend:XSetup.beam_y+extend,XSetup.beam_x-extend:XSetup.beam_x+extend]=0 # Get number of peaks above threshold in the current frame cropped=working[20:Frelon.resolution[1]-20,20:Frelon.resolution[1]-20] #print cropped.shape peaks=cropped[np.where(cropped>float(HFParams.threshold))] #peaks=working[np.where(working[20:Frelon.resolution[1]-20][20:Frelon.resolution[0]-20]>float(HFParams.threshold))] #peaks=working[np.where(working>float(HFParams.threshold))]# If enough peaks in current frame - This is a hit - Save it !! if len(peaks) >= HFParams.npixels: hit = 1 root=os.path.basename(fname) root=os.path.splitext(root)[0] if HFParams.DoPeakSearch: local_max=pf.find_local_max(working[0:1023,0:1004].astype(np.float),d_rad=1,threshold=HFParams.threshold) local_max_crop=pf.local_max_crop(working[0:1023,0:1004].astype(np.float), local_max, 3) peakslist=np.array(pf.subpixel_centroid(working[0:1023,0:1004],local_max_crop,3)) if IO.edf: OutputFileName =os.path.join(IO.procdir, IO.EDFDir,"%s.edf" %root) img.data = working img.write(OutputFileName) #Conversion to H5 if IO.H5: OutputFileName =os.path.join(IO.procdir, IO.H5Dir,"%s.h5" %root) #OutputFileName =os.path.join(IO.procdir, "HDF5/%s.h5" %root) OutputFile = h5py.File(OutputFileName,'w') working[0:1023,1004:1023]=0 OutputFile.create_dataset("data",data=working.astype(np.int32)) if HFParams.DoPeakSearch: OutputFile.create_dataset("processing/hitfinder/peakinfo",data=peakslist.astype(np.int32)) #g1=OutputFile.create_group("processing") #g2=g1.create_group("hitfinder") #g2.create_dataset("peakinfo",data=peakslist.astype(np.int32)) OutputFile.close() #if conversion to Pickle if IO.pickle: pixels=flex.int(working.astype(np.int32)) pixel_size=Frelon.pixel_size data = dpack(data=pixels, distance=XSetup.distance, pixel_size=pixel_size, wavelength=XSetup.wavelength, beam_center_x=XSetup.beam_y*pixel_size, beam_center_y=XSetup.beam_x*pixel_size, ccd_image_saturation=Frelon.overload, saturated_value=Frelon.overload) data=crop_image_pickle(data) OutputFileName =os.path.join(IO.procdir, IO.PicklesDir , "%s.pickle" %root) easy_pickle.dump(OutputFileName,data) else: working =0 else: print 'Warning : data shape problem for file %s, file skiped '%fname #continue return [hit,imgmax,imgmin,imgmed,index,peakslist,fname,working]
#beamX=602.015+(ds.env().epicsStore().value("robot_x")) #beamY=51.682+(ds.env().epicsStore().value("robot_y")) #beamDist=-563.083+60.+ (ds.env().epicsStore().value("robot_z")) beamX = 87.96 beamY = 85.57 beamDist = 150 print 'beam: ',beamX,beamY,beamDist # Assemble data pack data = cspad_tbx.dpack(data=image, distance=beamDist, pixel_size=pixelSize, wavelength=lambda_wavelength, beam_center_x=beamX, beam_center_y=beamY, ccd_image_saturation=saturation, saturated_value=saturation, timestamp = evt_timestamp) # output image pickle fpklname='/reg/d/psdm/xpp/%s/ftc/Run%s/Run_%s_idx_{}.pickle'%(expname,run,run) ep.dump(fpklname.format(ievt), data) print 'Importing image {}, beamX = {}, beamY = {}, Z-offset = {}, wavelength = {} '.format(fpklname, beamX, beamY, beamDist, lambda_wavelength) # if ievt>5: # break print 'is done '
def endjob(self, obj1, obj2=None): """The endjob() function writes the mean and standard deviation images to disk. @param evt Event object (psana only) @param env Environment object """ if obj2 is None: env = obj1 else: evt = obj1 env = obj2 stats = super(mod_average, self).endjob(env) if stats is None: return device = cspad_tbx.address_split(self.address)[2] if device == 'Andor': beam_center = (0, 0) # XXX Fiction! pixel_size = 13.5e-3 # XXX Should not be hardcoded here! saturated_value = 10000 elif device == 'Cspad' or device == 'Cspad2x2': beam_center = self.beam_center pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value elif device == 'marccd': beam_center = tuple(t // 2 for t in d['mean_img'].focus()) pixel_size = 0.079346 saturated_value = 2**16 - 1 if stats['nmemb'] > 0: if self.avg_dirname is not None or \ self.avg_basename is not None or \ self._mean_out is not None: d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=stats['mean_img'], distance=stats['distance'], pixel_size=pixel_size, saturated_value=saturated_value, timestamp=cspad_tbx.evt_timestamp(stats['time']), wavelength=stats['wavelength']) if self._mean_out is not None: p = cspad_tbx.dwritef2(d, self._mean_out) else: p = cspad_tbx.dwritef(d, self.avg_dirname, self.avg_basename) self.logger.info("Average written to %s" % p) if self.stddev_dirname is not None or \ self.stddev_basename is not None or \ self._std_out is not None: d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=stats['std_img'], distance=stats['distance'], pixel_size=pixel_size, saturated_value=saturated_value, timestamp=cspad_tbx.evt_timestamp(stats['time']), wavelength=stats['wavelength']) if self._std_out is not None: p = cspad_tbx.dwritef2(d, self._std_out) else: p = cspad_tbx.dwritef(d, self.stddev_dirname, self.stddev_basename) self.logger.info("Standard deviation written to %s" % p) if self.max_dirname is not None or \ self.max_basename is not None or \ self._max_out is not None: d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=stats['max_img'], distance=stats['distance'], pixel_size=pixel_size, saturated_value=saturated_value, timestamp=cspad_tbx.evt_timestamp(stats['time']), wavelength=stats['wavelength']) if self._max_out is not None: p = cspad_tbx.dwritef2(d, self._max_out) else: p = cspad_tbx.dwritef(d, self.max_dirname, self.max_basename) self.logger.info("Max written to %s" % p) if stats['nfail'] == 0: self.logger.info("%d images processed" % stats['nmemb']) else: self.logger.warning( "%d images processed, %d failed" % (stats['nmemb'], stats['nfail']))
spectrum_focus = self.sum_img[start_row:end_row, :] mask_focus = mask[start_row:end_row, :] spectrum_focus.set_selected(mask_focus > 0, 0) xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus) print "Number of rows: %i" % spectrum_focus.all()[0] print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus) print "Number of images used: %i" % flex.sum( pixel_histograms.histograms.values()[0].slots()) d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=spectrum_focus, distance=1, ccd_image_saturation=2e8, # XXX ) if run is not None: runstr = "_%04d" % run else: runstr = "" cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr) if gain_map is None: gain_map = flex.double(gain_img.accessor(), 0) img_sel = (gain_img > 0).as_1d() d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0', data=gain_img, distance=1) cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_') gain_map.as_1d().set_selected(img_sel.iselection(), 1 / gain_img.as_1d().select(img_sel))
def run(argv=None): import libtbx.option_parser if (argv is None): argv = sys.argv command_line = (libtbx.option_parser.option_parser( usage="%s [-v] [-p poly_mask] [-c circle_mask] [-x mask_pix_val] [-o output] -W mask_width -H mask_height" % libtbx.env.dispatcher_name) .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") .option(None, "--poly_mask", "-p", type="string", default=None, dest="poly_mask", help="Polygon to mask out. Comma-seperated string of xy pairs.") .option(None, "--circle_mask", "-c", type="string", default=None, dest="circle_mask", help="Circle to mask out. Comma-seperated string of x, y, and radius.") .option(None, "--mask_pix_val", "-x", type="int", default=-2, dest="mask_pix_val", help="Value for masked out pixels") .option(None, "--mask_width", "-W", type="int", default=None, dest="mask_width", help="Width of output mask") .option(None, "--mask_height", "-H", type="int", default=None, dest="mask_height", help="Height of output mask") .option(None, "--output", "-o", type="string", default="mask.pickle", dest="destpath", help="Output file path, should be *.pickle") .option(None, "--pixel_size", "-s", type="float", default=None, dest="pixel_size", help="Pixel size for detector") ).process(args=argv[1:]) # Must have width and height set if command_line.options.mask_height is None or command_line.options.mask_width is None: command_line.parser.print_usage(file=sys.stderr) return poly_mask = None if not command_line.options.poly_mask == None: poly_mask = [] poly_mask_tmp = command_line.options.poly_mask.split(",") if len(poly_mask_tmp) % 2 != 0: command_line.parser.print_usage(file=sys.stderr) return odd = True for item in poly_mask_tmp: try: if odd: poly_mask.append(int(item)) else: poly_mask[-1] = (poly_mask[-1],int(item)) except ValueError: command_line.parser.print_usage(file=sys.stderr) return odd = not odd circle_mask = None if command_line.options.circle_mask is not None: circle_mask_tmp = command_line.options.circle_mask.split(",") if len(circle_mask_tmp) != 3: command_line.parser.print_usage(file=sys.stderr) return try: circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2])) except ValueError: command_line.parser.print_usage(file=sys.stderr) return mask = flex.int(flex.grid(command_line.options.mask_width, command_line.options.mask_height)) if poly_mask is not None or circle_mask is not None: minx = miny = 0 maxx = mask.focus()[0] maxy = mask.focus()[1] if poly_mask is not None: minx = min([x[0] for x in poly_mask]) miny = min([y[1] for y in poly_mask]) maxx = max([x[0] for x in poly_mask]) maxy = max([y[1] for y in poly_mask]) if circle_mask is not None: circle_x, circle_y, radius = circle_mask if circle_x - radius < minx: minx = circle_x - radius if circle_y - radius < miny: miny = circle_y - radius if circle_x + radius > maxx: maxx = circle_x + radius if circle_y + radius > maxy: maxy = circle_y + radius sel = mask == command_line.options.mask_pix_val for j in xrange(miny, maxy): for i in xrange(minx, maxx): idx = j * mask.focus()[0] + i if not sel[idx]: if poly_mask is not None and point_in_polygon((i,j),poly_mask): sel[idx] = True elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius): sel[idx] = True mask.set_selected(sel,command_line.options.mask_pix_val) masked_out = len(mask.as_1d().select((mask == command_line.options.mask_pix_val).as_1d())) print "Masked out %d pixels out of %d (%.2f%%)"% \ (masked_out,len(mask),(masked_out)*100/(len(mask))) easy_pickle.dump(command_line.options.destpath, mask) d = dpack( active_areas=[0,0,command_line.options.mask_width,command_line.options.mask_height], address=None, beam_center_x=None, beam_center_y=None, data=mask, distance=None, timestamp=None, wavelength=1, xtal_target=None, pixel_size=command_line.options.pixel_size, saturated_value=None) dwritef2(d, command_line.options.destpath)
def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ super(average_mixin, self).event(evt, env) if evt.get('skip_event'): return # Get the distance for the detectors that should have it, and set # it to NaN for those that should not. if self.detector == 'CxiDs1' or \ self.detector == 'CxiDs2' or \ self.detector == 'CxiDsd' or \ self.detector == 'XppGon': distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: self._nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), 'skip_event') return else: distance = float('nan') if ("skew" in self.flags): # Take out inactive pixels if self.roi is not None: pixels = self.cspad_img[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]] dark_mask = self.dark_mask[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]] pixels = pixels.as_1d().select(dark_mask.as_1d()) else: pixels = self.cspad_img.as_1d().select(self.dark_mask.as_1d()).as_double() stats = scitbx.math.basic_statistics(pixels.as_double()) #stats.show() self.logger.info("skew: %.3f" %stats.skew) self.logger.info("kurtosis: %.3f" %stats.kurtosis) if 0: from matplotlib import pyplot hist_min, hist_max = flex.min(flex_cspad_img.as_double()), flex.max(flex_cspad_img.as_double()) print hist_min, hist_max n_slots = 100 n, bins, patches = pyplot.hist(flex_cspad_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max)) pyplot.show() # XXX This skew threshold probably needs fine-tuning skew_threshold = 0.35 if stats.skew < skew_threshold: self._nfail += 1 self.logger.warning("event(): skew < %f, shot skipped" % skew_threshold) evt.put(skip_event_flag(), 'skip_event') return #self.cspad_img *= stats.skew if ("inactive" in self.flags): self.cspad_img.set_selected(self.dark_stddev <= 0, 0) if ("noelastic" in self.flags): ELASTIC_THRESHOLD = self.elastic_threshold self.cspad_img.set_selected(self.cspad_img > ELASTIC_THRESHOLD, 0) if self.hot_threshold is not None: HOT_THRESHOLD = self.hot_threshold self.cspad_img.set_selected(self.dark_img > HOT_THRESHOLD, 0) if self.gain_map is not None and self.gain_threshold is not None: # XXX comparing each pixel to a moving average would probably be better # since the gain should vary approximately smoothly over different areas # of the detector GAIN_THRESHOLD = self.gain_threshold #self.logger.debug( #"rejecting: %i" %(self.gain_map > GAIN_THRESHOLD).count(True)) self.cspad_img.set_selected(self.gain_map > GAIN_THRESHOLD, 0) if ("nonoise" in self.flags): NOISE_THRESHOLD = self.noise_threshold self.cspad_img.set_selected(self.cspad_img < NOISE_THRESHOLD, 0) if ("sigma_scaling" in self.flags): self.do_sigma_scaling() if ("symnoise" in self.flags): SYMNOISE_THRESHOLD = self.symnoise_threshold self.cspad_img.set_selected((-SYMNOISE_THRESHOLD < self.cspad_img) & ( self.cspad_img < SYMNOISE_THRESHOLD), 0) if ("output" in self.flags): import pickle,os if (not os.path.isdir(self.pickle_dirname)): os.makedirs(self.pickle_dirname) flexdata = flex.int(self.cspad_img.astype(numpy.int32)) d = cspad_tbx.dpack( address=self.address, data=flexdata, timestamp=cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) ) G = open(os.path.join(".",self.pickle_dirname)+"/"+self.pickle_basename, "ab") pickle.dump(d,G,pickle.HIGHEST_PROTOCOL) G.close() if self.photon_threshold is not None and self.two_photon_threshold is not None: self.do_photon_counting() if self.background_path is not None: self.cspad_img -= self.background_img # t and self._sum_time are a two-long arrays of seconds and # milliseconds which hold time with respect to the base time. t = [t1 - t2 for (t1, t2) in zip(cspad_tbx.evt_time(evt), self._metadata['time_base'])] if self._nmemb == 0: # The peers metadata item is a bit field where a bit is set if # the partial sum from the corresponding worker process is # pending. If this is the first frame a worker process sees, # set its corresponding bit in the bit field since it will # contribute a partial sum. if env.subprocess() >= 0: self._lock.acquire() if 'peers' in self._metadata.keys(): self._metadata['peers'] |= (1 << env.subprocess()) else: self._metadata['peers'] = (1 << env.subprocess()) self._lock.release() self._sum_distance = distance self._sum_time = (t[0], t[1]) self._sum_wavelength = self.wavelength if self._have_max: self._max_img = self.cspad_img.deep_copy() if self._have_mean: self._sum_img = self.cspad_img.deep_copy() if self._have_std: self._ssq_img = flex.pow2(self.cspad_img) else: self._sum_distance += distance self._sum_time = (self._sum_time[0] + t[0], self._sum_time[1] + t[1]) self._sum_wavelength += self.wavelength if self._have_max: sel = (self.cspad_img > self._max_img).as_1d() self._max_img.as_1d().set_selected( sel, self.cspad_img.as_1d().select(sel)) if self._have_mean: self._sum_img += self.cspad_img if self._have_std: self._ssq_img += flex.pow2(self.cspad_img) self._nmemb += 1
def run(argv=None): if argv is None: argv = sys.argv[1:] command_line = ( libtbx.option_parser.option_parser( usage="%s [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files" % libtbx.env.dispatcher_name ) .option( None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress", ) .option( None, "--crop", "-c", action="store_true", default=False, dest="crop", help="Crop the image such that the beam center is in the middle", ) .option( None, "--skip_converted", "-s", action="store_true", default=False, dest="skip_converted", help="Skip converting if an image already exist that matches the destination file name", ) .option( None, "--wavelength", "-w", type="float", default=None, dest="wavelength", help="Override the image's wavelength (angstroms)", ) .option( None, "--distance", "-d", type="float", default=None, dest="distance", help="Override the detector distance (mm)", ) .option( None, "--pixel_size", "-p", type="float", default=None, dest="pixel_size", help="Override the detector pixel size (mm)", ) .option( None, "--beam_x", "-x", type="float", default=None, dest="beam_center_x", help="Override the beam x position (pixels)", ) .option( None, "--beam_y", "-y", type="float", default=None, dest="beam_center_y", help="Override the beam y position (pixels)", ) .option( None, "--overload", "-o", type="float", default=None, dest="overload", help="Override the detector overload value (ADU)", ) ).process(args=argv) paths = command_line.args if len(paths) <= 0: raise Usage("No files specified") for imgpath in paths: destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + ".pickle") if command_line.options.skip_converted and os.path.isfile(destpath): if command_line.options.verbose: print "Skipping %s, file exists" % imgpath continue if command_line.options.verbose: print "Converting %s to %s..." % (imgpath, destpath) try: img = dxtbx.load(imgpath) except IOError: img = None pass if img is None: import numpy as np try: raw_data = np.loadtxt(imgpath) from scitbx.array_family import flex raw_data = flex.double(raw_data.astype(np.double)) except ValueError: raise Usage("Couldn't load %s, no supported readers" % imgpath) detector = None beam = None scan = None else: raw_data = img.get_raw_data() detector = img.get_detector() beam = img.get_beam() scan = img.get_scan() if detector is None: if command_line.options.distance is None: raise Usage("Can't get distance from image. Override with -d") if command_line.options.pixel_size is None: raise Usage("Can't get pixel size from image. Override with -p") if command_line.options.overload is None: raise Usage("Can't get overload value from image. Override with -o") distance = command_line.options.distance pixel_size = command_line.options.pixel_size overload = command_line.options.overload else: detector = detector[0] if command_line.options.distance is None: distance = detector.get_distance() else: distance = command_line.options.distance if command_line.options.pixel_size is None: pixel_size = detector.get_pixel_size()[0] else: pixel_size = command_line.options.pixel_size if command_line.options.overload is None: overload = detector.get_trusted_range()[1] else: overload = command_line.options.overload if beam is None: if command_line.options.wavelength is None: raise Usage("Can't get wavelength from image. Override with -w") wavelength = command_line.options.wavelength else: if command_line.options.wavelength is None: wavelength = beam.get_wavelength() else: wavelength = command_line.options.wavelength if beam is None and detector is None: if command_line.options.beam_center_x is None: print "Can't get beam x position from image. Using image center. Override with -x" beam_x = raw_data.focus()[0] * pixel_size else: beam_x = command_line.options.beam_center_x * pixel_size if command_line.options.beam_center_y is None: print "Can't get beam y position from image. Using image center. Override with -y" beam_y = raw_data.focus()[1] * pixel_size else: beam_y = command_line.options.beam_center_y * pixel_size else: if command_line.options.beam_center_x is None: beam_x = detector.get_beam_centre(beam.get_s0())[0] else: beam_x = command_line.options.beam_center_x * pixel_size if command_line.options.beam_center_y is None: beam_y = detector.get_beam_centre(beam.get_s0())[1] else: beam_y = command_line.options.beam_center_y * pixel_size if scan is None: timestamp = None else: msec, sec = math.modf(scan.get_epochs()[0]) timestamp = evt_timestamp((sec, msec)) data = dpack( data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp, ) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data["OSC_START"] = osc_start data["OSC_RANGE"] = osc_range data["TIME"] = scan.get_exposure_times()[0] if command_line.options.crop: data = crop_image_pickle(data) easy_pickle.dump(destpath, data)
quad_asics.append(numpy.concatenate((a, b), axis=1)) quad_data = numpy.dstack(quad_asics) quad_data = numpy.rollaxis(quad_data, 2, 0) data3d.append(fake_cspad_ElementV2(quad_data, i_quad)) env = fake_env(fake_config()) evt = fake_evt(data3d) beam_center, active_areas = cbcaa(fake_config(), sections) data = flex.int( CsPadDetector(address, evt, env, sections).astype(numpy.float64)) img_dict = dpack(active_areas=active_areas, address=address, beam_center_x=beam_center[0] * pixel_size, beam_center_y=beam_center[1] * pixel_size, data=data, distance=params.distance, pixel_size=pixel_size, timestamp=timestamp, wavelength=params.wavelength) img = NpyImage("", source_data=img_dict) args = [ "distl.detector_format_version=%s" % params.detector_format_version ] horizons_phil = cxi_phil.cxi_versioned_extract(args) img.readHeader(horizons_phil) img.translate_tiles(horizons_phil) tm = img.get_tile_manager(horizons_phil)
def run(argv=None): import libtbx.option_parser if (argv is None): argv = sys.argv command_line = (libtbx.option_parser.option_parser( usage="%s [-v] [-p poly_mask] [-c circle_mask] [-x mask_pix_val] [-o output] -W mask_width -H mask_height" % libtbx.env.dispatcher_name) .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") .option(None, "--poly_mask", "-p", type="string", default=None, dest="poly_mask", help="Polygon to mask out. Comma-seperated string of xy pairs.") .option(None, "--circle_mask", "-c", type="string", default=None, dest="circle_mask", help="Circle to mask out. Comma-seperated string of x, y, and radius.") .option(None, "--mask_pix_val", "-x", type="int", default=-2, dest="mask_pix_val", help="Value for masked out pixels") .option(None, "--mask_width", "-W", type="int", default=None, dest="mask_width", help="Width of output mask") .option(None, "--mask_height", "-H", type="int", default=None, dest="mask_height", help="Height of output mask") .option(None, "--output", "-o", type="string", default="mask.pickle", dest="destpath", help="Output file path, should be *.pickle") .option(None, "--pixel_size", "-s", type="float", default=None, dest="pixel_size", help="Pixel size for detector") ).process(args=argv[1:]) # Must have width and height set if command_line.options.mask_height is None or command_line.options.mask_width is None: command_line.parser.print_usage(file=sys.stderr) return poly_mask = None if not command_line.options.poly_mask == None: poly_mask = [] poly_mask_tmp = command_line.options.poly_mask.split(",") if len(poly_mask_tmp) % 2 != 0: command_line.parser.print_usage(file=sys.stderr) return odd = True for item in poly_mask_tmp: try: if odd: poly_mask.append(int(item)) else: poly_mask[-1] = (poly_mask[-1],int(item)) except ValueError: command_line.parser.print_usage(file=sys.stderr) return odd = not odd circle_mask = None if command_line.options.circle_mask is not None: circle_mask_tmp = command_line.options.circle_mask.split(",") if len(circle_mask_tmp) != 3: command_line.parser.print_usage(file=sys.stderr) return try: circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2])) except ValueError: command_line.parser.print_usage(file=sys.stderr) return mask = flex.int(flex.grid(command_line.options.mask_width, command_line.options.mask_height)) if poly_mask is not None or circle_mask is not None: minx = miny = 0 maxx = mask.focus()[0] maxy = mask.focus()[1] if poly_mask is not None: minx = min([x[0] for x in poly_mask]) miny = min([y[1] for y in poly_mask]) maxx = max([x[0] for x in poly_mask]) maxy = max([y[1] for y in poly_mask]) if circle_mask is not None: circle_x, circle_y, radius = circle_mask if circle_x - radius < minx: minx = circle_x - radius if circle_y - radius < miny: miny = circle_y - radius if circle_x + radius > maxx: maxx = circle_x + radius if circle_y + radius > maxy: maxy = circle_y + radius sel = mask == command_line.options.mask_pix_val for j in range(miny, maxy): for i in range(minx, maxx): idx = j * mask.focus()[0] + i if not sel[idx]: if poly_mask is not None and point_in_polygon((i,j),poly_mask): sel[idx] = True elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius): sel[idx] = True mask.set_selected(sel,command_line.options.mask_pix_val) masked_out = len(mask.as_1d().select((mask == command_line.options.mask_pix_val).as_1d())) print("Masked out %d pixels out of %d (%.2f%%)"% \ (masked_out,len(mask),(masked_out)*100/(len(mask)))) easy_pickle.dump(command_line.options.destpath, mask) d = dpack( active_areas=[0,0,command_line.options.mask_width,command_line.options.mask_height], address=None, beam_center_x=None, beam_center_y=None, data=mask, distance=None, timestamp=None, wavelength=1, xtal_target=None, pixel_size=command_line.options.pixel_size, saturated_value=None) dwritef2(d, command_line.options.destpath)
def load_image_file(self, filepath): """ Reads a single image file (e.g. a CBF or MCCD) and returns raw data, experiment information, and an image type (e.g. pickle, raw image, or none) :param img: path to image file :return: data: raw image data and experiment info img_type: which type of image this was, or None if not loaded """ error = None try: with util.Capturing() as junk_output: loaded_img = dxtbx.load(filepath) except Exception as e: error = 'IOTA IMPORTER ERROR: DXTBX failed! {}'.format(e) print(e) loaded_img = None # Extract image information if loaded_img is not None: raw_data = loaded_img.get_raw_data() detector = loaded_img.get_detector()[0] beam = loaded_img.get_beam() scan = loaded_img.get_scan() distance = detector.get_distance() pixel_size = detector.get_pixel_size()[0] overload = detector.get_trusted_range()[1] wavelength = beam.get_wavelength() beam_x = detector.get_beam_centre(beam.get_s0())[0] beam_y = detector.get_beam_centre(beam.get_s0())[1] if scan is None: timestamp = None else: msec, sec = math.modf(scan.get_epochs()[0]) timestamp = evt_timestamp((sec, msec)) # Assemble datapack data = dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data['OSC_START'] = 0 #osc_start data['OSC_RANGE'] = 0 #osc_start data['TIME'] = scan.get_exposure_times()[0] # Populate image object information self.img_object.final['pixel_size'] = pixel_size self.img_object.final['img_size'] = (data['SIZE1'], data['SIZE2']) self.img_object.final['beamX'] = beam_x self.img_object.final['beamY'] = beam_y self.img_object.final['gain'] = detector.get_gain() self.img_object.final['distance'] = distance self.img_object.final['wavelength'] = wavelength else: data = None return data, error
spectrum_focus = self.sum_img[start_row:end_row,:] mask_focus = mask[start_row:end_row,:] spectrum_focus.set_selected(mask_focus > 0, 0) xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus) print "Number of rows: %i" %spectrum_focus.all()[0] print "Estimated no. photons counted: %i" %flex.sum(spectrum_focus) print "Number of images used: %i" %flex.sum( pixel_histograms.histograms.values()[0].slots()) d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=spectrum_focus, distance=1, ccd_image_saturation=2e8, # XXX ) if run is not None: runstr="_%04d"%run else: runstr="" cspad_tbx.dwritef(d, output_dirname, 'sum%s_'%runstr) if gain_map is None: gain_map = flex.double(gain_img.accessor(), 0) img_sel = (gain_img > 0).as_1d() d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=gain_img, distance=1 ) cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ import libtbx.load_env from libtbx import easy_pickle, option_parser from scitbx.array_family import flex from xfel.cxi.cspad_ana import cspad_tbx if argv is None: argv = sys.argv command_line = (option_parser.option_parser( usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \ "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name) .option(None, "--average-path", "-a", type="string", default=None, dest="avg_path", metavar="PATH", help="Write average image to PATH") .option(None, "--maximum-path", "-m", type="string", default=None, dest="max_path", metavar="PATH", help="Write maximum projection image to PATH") .option(None, "--stddev-path", "-s", type="string", default=None, dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") ).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 # Loop over all images and accumulate statistics. nfail = 0 nmemb = 0 if len(paths) == 1: # test if the iamge is a multi-image from dxtbx.format.Registry import Registry from dxtbx.format.FormatMultiImage import FormatMultiImage format_class = Registry.find(paths[0]) if not issubclass(format_class, FormatMultiImage): from libtbx.utils import Usage raise Usage("Supply more than one image") print "Loading image..." i = format_class(paths[0]) print "Loaded" def read_single_image(n): if command_line.options.verbose: sys.stdout.write("Processing %s: %d...\n" % (paths[0], n)) beam = i.get_beam(n) assert len(i.get_detector(n)) == 1 detector = i.get_detector(n)[0] beam_center = detector.get_beam_centre(beam.get_s0()) detector_address = format_class.__name__ distance = detector.get_distance() img = i.get_raw_data(n).as_1d().as_double() pixel_size = 0.5 * sum(detector.get_pixel_size()) saturated_value = int(round(detector.get_trusted_range()[1])) size = detector.get_image_size() scan = i.get_scan(n) if scan is None: time_tuple = (0, 0) else: time_tuple = (scan.get_epochs()[0], 0) wavelength = beam.get_wavelength() active_areas = flex.int((0, 0, size[0], size[1])) return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas iterable = xrange(i.get_num_images()) else: def read_single_image(path): if command_line.options.verbose: sys.stdout.write("Processing %s...\n" % path) from dxtbx.format.Registry import Registry format_class = Registry.find(path) i = format_class(path) beam = i.get_beam() assert len(i.get_detector()) == 1 detector = i.get_detector()[0] beam_center = detector.get_beam_centre(beam.get_s0()) detector_address = format_class.__name__ distance = detector.get_distance() img = i.get_raw_data().as_1d().as_double() pixel_size = 0.5 * sum(detector.get_pixel_size()) saturated_value = int(round(detector.get_trusted_range()[1])) size = detector.get_image_size() scan = i.get_scan() if scan is None: time_tuple = (0, 0) else: time_tuple = (scan.get_epochs()[0], 0) wavelength = beam.get_wavelength() active_areas = flex.int((0, 0, size[0], size[1])) return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas iterable = paths for item in iterable: try: #XXX This code assumes a monolithic detector! beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas = \ read_single_image(item) except Exception: nfail += 1 continue # See also event() in xfel.cxi.cspad_ana.average_tbx. Record the # base time as the timestamp of the first image. # # The sum-of-squares image is accumulated using long integers, as # this delays the point where overflow occurs. But really, this # is just a band-aid... if nmemb == 0: max_img = img.deep_copy() sum_distance = distance sum_img = img.deep_copy() ssq_img = flex.pow2(img) sum_wavelength = wavelength sum_time = (0, 0) time_base = time_tuple else: sel = (img > max_img).as_1d() max_img.set_selected(sel, img.select(sel)) sum_distance += distance sum_img += img ssq_img += flex.pow2(img) sum_wavelength += wavelength sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]), sum_time[1] + (time_tuple[1] - time_base[1])) nmemb += 1 # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = sum_img.as_double() / nmemb avg_distance = sum_distance / nmemb avg_timestamp = cspad_tbx.evt_timestamp( (time_base[0] + int(round(sum_time[0] / nmemb)), time_base[1] + int(round(sum_time[1] / nmemb)))) avg_wavelength = sum_wavelength / nmemb # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: avg_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=avg_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.avg_path, d) if command_line.options.max_path is not None: max_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=max_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.max_path, d) if command_line.options.stddev_path is not None: stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. stddev_img.set_selected(stddev_img < 0, 0) if nmemb == 1: stddev_img = flex.sqrt(stddev_img) else: stddev_img = flex.sqrt(stddev_img / (nmemb - 1)) stddev_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=stddev_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.stddev_path, d) return 0
def average(argv=None): if argv == None: argv = sys.argv[1:] try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") command_line = (libtbx.option_parser.option_parser( usage=""" %s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] To write image pickles use -p, otherwise the program writes CSPAD CBFs. Writing CBFs requires the geometry to be already deployed. Examples: cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 Use one process on the current node to process all the events from run 25 of experiment cxi49812, using a detz_offset of 571. mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 As above, using 16 cores on the current node. bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812 As above, using the psanaq and 100 cores, putting the log in average.out and the output images in the folder cxi49812. """ % libtbx.env.dispatcher_name) .option(None, "--as_pickle", "-p", action="store_true", default=False, dest="as_pickle", help="Write results as image pickle files instead of cbf files") .option(None, "--config", "-c", type="string", default=None, dest="config", metavar="PATH", help="psana config file") .option(None, "--experiment", "-x", type="string", default=None, dest="experiment", help="experiment name (eg cxi84914)") .option(None, "--run", "-r", type="int", default=None, dest="run", help="run number") .option(None, "--address", "-a", type="string", default="CxiDs2.0:Cspad.0", dest="address", help="detector address name (eg CxiDs2.0:Cspad.0)") .option(None, "--detz_offset", "-d", type="float", default=None, dest="detz_offset", help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)") .option(None, "--outputdir", "-o", type="string", default=".", dest="outputdir", metavar="PATH", help="Optional path to output directory for output files") .option(None, "--averagebase", "-A", type="string", default="{experiment!l}_avg-r{run:04d}", dest="averagepath", metavar="PATH", help="Path to output average image without extension. String substitution allowed") .option(None, "--stddevbase", "-S", type="string", default="{experiment!l}_stddev-r{run:04d}", dest="stddevpath", metavar="PATH", help="Path to output standard deviation image without extension. String substitution allowed") .option(None, "--maxbase", "-M", type="string", default="{experiment!l}_max-r{run:04d}", dest="maxpath", metavar="PATH", help="Path to output maximum projection image without extension. String substitution allowed") .option(None, "--numevents", "-n", type="int", default=None, dest="numevents", help="Maximum number of events to process. Default: all") .option(None, "--skipevents", "-s", type="int", default=0, dest="skipevents", help="Number of events in the beginning of the run to skip. Default: 0") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") .option(None, "--pickle-optical-metrology", "-m", action="store_true", default=False, dest="pickle_optical_metrology", help="If writing pickle files, use the optical metrology in the experiment's calib directory") .option(None, "--bin_size", "-b", type="int", default=None, dest="bin_size", help="Rayonix detector bin size") .option(None, "--override_beam_x", "-X", type="float", default=None, dest="override_beam_x", help="Rayonix detector beam center x coordinate") .option(None, "--override_beam_y", "-Y", type="float", default=None, dest="override_beam_y", help="Rayonix detector beam center y coordinate") .option(None, "--calib_dir", "-C", type="string", default=None, dest="calib_dir", metavar="PATH", help="calibration directory") .option(None, "--xtc_dir", "-D", type="string", default=None, dest="xtc_dir", metavar="PATH", help="xtc stream directory") .option(None, "--use_ffb", "-f", action="store_true", default=False, dest="use_ffb", help="Use the fast feedback filesystem at LCLS. Only for the active experiment!") ).process(args=argv) if len(command_line.args) > 0 or \ command_line.options.as_pickle is None or \ command_line.options.experiment is None or \ command_line.options.run is None or \ command_line.options.address is None or \ command_line.options.detz_offset is None or \ command_line.options.averagepath is None or \ command_line.options.stddevpath is None or \ command_line.options.maxpath is None or \ command_line.options.pickle_optical_metrology is None: command_line.parser.show_help() return # set this to sys.maxint to analyze all events if command_line.options.numevents is None: maxevents = sys.maxint else: maxevents = command_line.options.numevents comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if command_line.options.config is not None: psana.setConfigFile(command_line.options.config) dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run) if command_line.options.xtc_dir is not None: if command_line.options.use_ffb: raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system") dataset_name += ":dir=%s"%command_line.options.xtc_dir elif command_line.options.use_ffb: # as ffb is only at SLAC, ok to hardcode /reg/d here dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment) ds = psana.DataSource(dataset_name) address = command_line.options.address src = psana.Source('DetInfo(%s)'%address) if not command_line.options.as_pickle: psana_det = psana.Detector(address, ds.env()) nevent = np.array([0.]) for run in ds.runs(): runnumber = run.run() # list of all events if command_line.options.skipevents > 0: print "Skipping first %d events"%command_line.options.skipevents times = run.times()[command_line.options.skipevents:] nevents = min(len(times),maxevents) # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0] for i in xrange(len(mytimes)): if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes) evt = run.event(mytimes[i]) #print "Event #",rank*mylength+i," has id:",evt.get(EventId) if 'Rayonix' in command_line.options.address: data = evt.get(Camera.FrameV1,src) if data is None: print "No data" continue data=data.data16().astype(np.float64) elif command_line.options.as_pickle: data = evt.get(psana.ndarray_float64_3, src, 'image0') else: # get numpy array, 32x185x388 data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations if data is None: print "No data" continue d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset) if d is None: print "No distance, skipping shot" continue if 'distance' in locals(): distance += d else: distance = np.array([float(d)]) w = cspad_tbx.evt_wavelength(evt) if w is None: print "No wavelength, skipping shot" continue if 'wavelength' in locals(): wavelength += w else: wavelength = np.array([w]) t = cspad_tbx.evt_time(evt) if t is None: print "No timestamp, skipping shot" continue if 'timestamp' in locals(): timestamp += t[0] + (t[1]/1000) else: timestamp = np.array([t[0] + (t[1]/1000)]) if 'sum' in locals(): sum+=data else: sum=np.array(data, copy=True) if 'sumsq' in locals(): sumsq+=data*data else: sumsq=data*data if 'maximum' in locals(): maximum=np.maximum(maximum,data) else: maximum=np.array(data, copy=True) nevent += 1 #sum the images across mpi cores if size > 1: print "Synchronizing rank", rank totevent = np.zeros(nevent.shape) comm.Reduce(nevent,totevent) if rank == 0 and totevent[0] == 0: raise Sorry("No events found in the run") sumall = np.zeros(sum.shape).astype(sum.dtype) comm.Reduce(sum,sumall) sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype) comm.Reduce(sumsq,sumsqall) maxall = np.zeros(maximum.shape).astype(maximum.dtype) comm.Reduce(maximum,maxall, op=MPI.MAX) waveall = np.zeros(wavelength.shape).astype(wavelength.dtype) comm.Reduce(wavelength,waveall) distall = np.zeros(distance.shape).astype(distance.dtype) comm.Reduce(distance,distall) timeall = np.zeros(timestamp.shape).astype(timestamp.dtype) comm.Reduce(timestamp,timeall) if rank==0: if size > 1: print "Synchronized" # Accumulating floating-point numbers introduces errors, # which may cause negative variances. Since a two-pass # approach is unacceptable, the standard deviation is # clamped at zero. mean = sumall / float(totevent[0]) variance = (sumsqall / float(totevent[0])) - (mean**2) variance[variance < 0] = 0 stddev = np.sqrt(variance) wavelength = waveall[0] / totevent[0] distance = distall[0] / totevent[0] pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value timestamp = timeall[0] / totevent[0] timestamp = (int(timestamp), timestamp % int(timestamp) * 1000) timestamp = cspad_tbx.evt_timestamp(timestamp) if command_line.options.as_pickle: extension = ".pickle" else: extension = ".cbf" dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env())] dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths] if 'Rayonix' in command_line.options.address: from xfel.cxi.cspad_ana import rayonix_tbx pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size) beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y] detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size) active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]]) split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3] for data, path in zip([mean, stddev, maxall], dest_paths): print "Saving", path d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=flex.double(data), distance=distance, pixel_size=pixel_size, saturated_value=rayonix_tbx.rayonix_saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) elif command_line.options.as_pickle: split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3] xpp = 'xpp' in address.lower() if xpp: evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format from xfel.detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0]) assert version_lookup is not None active_areas = xpp_active_areas[version_lookup]['active_areas'] beam_center = [1765 // 2, 1765 // 2] else: if command_line.options.calib_dir is not None: metro_path = command_line.options.calib_dir elif command_line.options.pickle_optical_metrology: from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path metro_path = get_calib_file_path(run.env(), address, run) else: metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0") sections = parse_calib.calib2sections(metro_path) beam_center, active_areas = cspad_tbx.cbcaa( cspad_tbx.getConfig(address, ds.env()), sections) class fake_quad(object): def __init__(self, q, d): self.q = q self.d = d def quad(self): return self.q def data(self): return self.d if xpp: quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)] mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) mean = flex.double(mean.astype(np.float64)) quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)] stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) stddev = flex.double(stddev.astype(np.float64)) quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)] maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) maxall = flex.double(maxall.astype(np.float64)) else: quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)] mean = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)] stddev = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)] maxall = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) maxall = flex.double(maxall.astype(np.float64)) for data, path in zip([mean, stddev, maxall], dest_paths): print "Saving", path d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=data, distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) else: # load a header only cspad cbf from the slac metrology from xfel.cftbx.detector import cspad_cbf_tbx import pycbf base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d"%run.run()) for data, path in zip([mean, stddev, maxall], dest_paths): print "Saving", path cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address) cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ import libtbx.load_env from libtbx import easy_pickle, option_parser from xfel.cxi.cspad_ana import cspad_tbx if argv is None: argv = sys.argv command_line = (option_parser.option_parser( usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \ "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name) .option(None, "--average-path", "-a", type="string", default=None, dest="avg_path", metavar="PATH", help="Write average image to PATH") .option(None, "--maximum-path", "-m", type="string", default=None, dest="max_path", metavar="PATH", help="Write maximum projection image to PATH") .option(None, "--stddev-path", "-s", type="string", default=None, dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") .option(None, "--nproc", "-n", type="int", default=1, dest="nproc", help="Number of processors") .option(None, "--num-images-max", "-N", type="int", default=None, dest="num_images_max", help="Maximum number of frames to average") ).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 if len(paths) == 1: # test if the iamge is a multi-image from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_filenames([paths[0]]) assert len(datablocks) == 1 datablock = datablocks[0] imagesets = datablock.extract_imagesets() assert len(imagesets) == 1 imageset = imagesets[0] if not imageset.reader().is_single_file_reader(): from libtbx.utils import Usage raise Usage("Supply more than one image") worker = multi_image_worker(command_line, paths[0], imageset) if command_line.options.num_images_max is not None and command_line.options.num_images_max < len( imageset): iterable = range(command_line.options.num_images_max) else: iterable = range(len(imageset)) else: # Multiple images provided worker = single_image_worker(command_line) if command_line.options.num_images_max is not None and command_line.options.num_images_max < len( paths): iterable = paths[:command_line.options.num_images_max] else: iterable = paths if command_line.options.nproc > 1: iterable = splitit(iterable, command_line.options.nproc) from libtbx import easy_mp if command_line.options.nproc == 1: results = [worker(iterable)] else: results = easy_mp.parallel_map(func=worker, iterable=iterable, processes=command_line.options.nproc) nfail = 0 nmemb = 0 for i, (r_nfail, r_nmemb, r_max_img, r_sum_distance, r_sum_img, r_ssq_img, r_sum_wavelength, size, active_areas, detector_address, beam_center, pixel_size, saturated_value) in enumerate(results): nfail += r_nfail nmemb += r_nmemb if i == 0: max_img = r_max_img sum_distance = r_sum_distance sum_img = r_sum_img ssq_img = r_ssq_img sum_wavelength = r_sum_wavelength else: sel = (r_max_img > max_img).as_1d() max_img.set_selected(sel, r_max_img.select(sel)) sum_distance += r_sum_distance sum_img += r_sum_img ssq_img += r_ssq_img sum_wavelength += r_sum_wavelength # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = sum_img.as_double() / nmemb avg_distance = sum_distance / nmemb avg_wavelength = sum_wavelength / nmemb # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: avg_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=avg_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.avg_path, d) if command_line.options.max_path is not None: max_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=max_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.max_path, d) if command_line.options.stddev_path is not None: stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. stddev_img.set_selected(stddev_img < 0, 0) if nmemb == 1: stddev_img = flex.sqrt(stddev_img) else: stddev_img = flex.sqrt(stddev_img / (nmemb - 1)) stddev_img.resize(flex.grid(size[1], size[0])) d = cspad_tbx.dpack(active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=stddev_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.stddev_path, d) return 0
def event(self, evt, env): """The event() function is called for every L1Accept transition. XXX more? Previously, common-mode correction was applied only after initial threshold filtering. Since the common_mode class applies the (lengthy) common-mode correction immediately after reading the image from the stream, this optimisation is currently not (elegantly) doable. @param evt Event data object, a configure object @param env Environment object """ super(mod_hitfind, self).event(evt, env) if (evt.get("skip_event")): return # This module only applies to detectors for which a distance is # available. distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: self.nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), "skip_event") return device = cspad_tbx.address_split(self.address)[2] # ***** HITFINDING ***** XXX For hitfinding it may be interesting # to look at the fraction of subzero pixels in the dark-corrected # image. if (self.m_threshold is not None): # If a threshold value is given it can be applied in one of three ways: # 1. Apply it over the whole image if (self.m_roi is None and self.m_distl_min_peaks is None): vmax = flex.max(self.cspad_img) if (vmax < self.m_threshold): if not self.m_negate_hits: # Tell downstream modules to skip this event if the threshold was not met. evt.put(skip_event_flag(), "skip_event") return elif self.m_negate_hits: evt.put(skip_event_flag(), "skip_event") return # 2. Apply threshold over a rectangular region of interest. elif (self.m_roi is not None): vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3], self.m_roi[0]:self.m_roi[1]]) if (vmax < self.m_threshold): if not self.m_negate_hits: evt.put(skip_event_flag(), "skip_event") return elif self.m_negate_hits: evt.put(skip_event_flag(), "skip_event") return # 3. Determine the spotfinder spots within the central ASICS, and accept the # image as a hit if there are m_distl_min_peaks exceeding m_threshold. # As a further requirement, the peaks must exceed 2.5 * the 90-percentile # pixel value of the central ASICS. This filter was added to avoid high-background # false positives. elif (self.m_distl_min_peaks is not None): if device == 'marccd': self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0] self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1] elif device == 'Rayonix': self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0] self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1] peak_heights, outvalue = self.distl_filter( self.address, self.cspad_img.iround(), # XXX correct? distance, self.timestamp, self.wavelength) if ('permissive' in self.m_distl_flags): number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True) else: number_of_accepted_peaks = (( peak_heights > self.m_threshold).__and__( outvalue == 0)).count(True) sec, ms = cspad_tbx.evt_time(evt) evt_time = sec + ms / 1000 self.stats_logger.info("BRAGG %.3f %d" % (evt_time, number_of_accepted_peaks)) skip_event = False if number_of_accepted_peaks < self.m_distl_min_peaks: self.logger.info( "Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d" % (env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold)) if not self.m_negate_hits: skip_event = True else: self.logger.info( "Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" % (env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold)) if self.m_negate_hits: skip_event = True if skip_event: if self.m_db_logging: # log misses to the database self.queue_entry( (self.trial, evt.run(), "%.3f" % evt_time, number_of_accepted_peaks, distance, self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags)) evt.put(skip_event_flag(), "skip_event") return # the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this # hit will not be logged in the db. evt.put(number_of_accepted_peaks, 'sfspots') self.logger.info("Subprocess %02d: process image #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) # See r17537 of mod_average.py. if device == 'Cspad': pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value elif device == 'marccd': pixel_size = evt.get("marccd_pixel_size") saturated_value = evt.get("marccd_saturated_value") elif device == 'Rayonix': pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) saturated_value = rayonix_tbx.rayonix_saturated_value d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=self.cspad_img.iround(), # XXX ouch! distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=self.timestamp, wavelength=self.wavelength, xtal_target=self.m_xtal_target) if (self.m_dispatch == "index"): import sys from xfel.cxi.integrate_image_api import integrate_one_image info = integrate_one_image( d, integration_dirname=self.m_integration_dirname, integration_basename=self.m_integration_basename) sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ indexed = info is not None and hasattr(info, 'spotfinder_results') if self.m_progress_logging: if self.m_db_version == 'v1': if indexed: # integration pickle dictionary is available here as info.last_saved_best if info.last_saved_best[ "identified_isoform"] is not None: #print info.last_saved_best.keys() from cxi_xdr_xes.cftbx.cspad_ana import db dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password) cursor = dbobj.cursor() if info.last_saved_best[ "identified_isoform"] in self.isoforms: PM, indices, miller_id = self.isoforms[ info.last_saved_best["identified_isoform"]] else: from xfel.xpp.progress_support import progress_manager PM = progress_manager(info.last_saved_best, self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run()) indices, miller_id = PM.get_HKL(cursor) # cache these as they don't change for a given isoform self.isoforms[info.last_saved_best[ "identified_isoform"]] = PM, indices, miller_id if self.m_sql_buffer_size > 1: self.queue_progress_entry( PM.scale_frame_detail(self.timestamp, cursor, do_inserts=False)) else: PM.scale_frame_detail(self.timestamp, cursor, do_inserts=True) dbobj.commit() cursor.close() dbobj.close() elif self.m_db_version == 'v2': key_low = 'cctbx.xfel.radial_average.two_theta_low' key_high = 'cctbx.xfel.radial_average.two_theta_high' tt_low = evt.get(key_low) tt_high = evt.get(key_high) from xfel.ui.db.dxtbx_db import log_frame if indexed: n_spots = len(info.spotfinder_results.images[ info.frames[0]]['spots_total']) else: sfspots = evt.get('sfspots') if sfspots is None: if info is None or not isinstance(info, int): n_spots = 0 else: n_spots = info else: n_spots = sfspots if indexed: known_setting = info.horizons_phil.known_setting indexed_setting = info.organizer.info[ 'best_integration']['counter'] if known_setting is None or known_setting == indexed_setting: from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list c = construct_reflection_table_and_experiment_list( info.last_saved_best, None, pixel_size, proceed_without_image=True) c.assemble_experiments() c.assemble_reflections() log_frame(c.experiment_list, c.reflections, self.db_params, evt.run(), n_spots, self.timestamp, tt_low, tt_high) else: print( "Not logging %s, wrong bravais setting (expecting %d, got %d)" % (self.timestamp, known_setting, indexed_setting)) else: log_frame(None, None, self.db_params, evt.run(), n_spots, self.timestamp, tt_low, tt_high) if self.m_db_logging: sec, ms = cspad_tbx.evt_time(evt) evt_time = sec + ms / 1000 sfspots = evt.get('sfspots') if sfspots is None: if indexed: n_spots = len(info.spotfinder_results.images[ info.frames[0]]['spots_total']) else: n_spots = 0 else: n_spots = sfspots if indexed: mosaic_bloc_rotation = info.last_saved_best.get( 'ML_half_mosaicity_deg', [0])[0] mosaic_block_size = info.last_saved_best.get( 'ML_domain_size_ang', [0])[0] ewald_proximal_volume = info.last_saved_best.get( 'ewald_proximal_volume', [0])[0] obs = info.last_saved_best['observations'][0] cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell( ).parameters() pointgroup = info.last_saved_best['pointgroup'] resolution = obs.d_min() else: mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \ cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0 self.queue_entry( (self.trial, evt.run(), "%.3f" % evt_time, n_spots, distance, self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation, mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution, self.m_db_tags)) if (not indexed): evt.put(skip_event_flag(), "skip_event") return elif (self.m_dispatch == "nop"): pass elif (self.m_dispatch == "view"): #interactive image viewer args = ["indexing.data=dummy"] detector_format_version = detector_format_function( self.address, evt.GetTime()) if detector_format_version is not None: args += [ "distl.detector_format_version=%" % detector_format_version ] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(self.m_xtal_target, args) horizons_phil.indexing.data = d from xfel.cxi import display_spots display_spots.parameters.horizons_phil = horizons_phil display_spots.wrapper_of_callback().display( horizons_phil.indexing.data) elif (self.m_dispatch == "spots"): #interactive spotfinder viewer args = ["indexing.data=dummy"] detector_format_version = detector_format_function( self.address, evt.GetTime()) if detector_format_version is not None: args += [ "distl.detector_format_version=%s" % detector_format_version ] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(self.m_xtal_target, args) horizons_phil.indexing.data = d from xfel.cxi import display_spots display_spots.parameters.horizons_phil = horizons_phil from rstbx.new_horizons.index import pre_indexing_validation, pack_names pre_indexing_validation(horizons_phil) imagefile_arguments = pack_names(horizons_phil) horizons_phil.persist.show() from spotfinder.applications import signal_strength info = signal_strength.run_signal_strength_core( horizons_phil, imagefile_arguments) work = display_spots.wrapper_of_callback(info) work.display_with_callback(horizons_phil.indexing.data) elif (self.m_dispatch == "write_dict"): self.logger.warning( "event(): deprecated dispatch 'write_dict', use mod_dump instead" ) if (self.m_out_dirname is not None or self.m_out_basename is not None): cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename) # Diagnostic message emitted only when all the processing is done. if (env.subprocess() >= 0): self.logger.info("Subprocess %02d: accepted #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) else: self.logger.info("Accepted #%05d @ %s" % (self.nshots, self.timestamp))
def average(argv=None): if argv == None: argv = sys.argv[1:] try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") command_line = (libtbx.option_parser.option_parser(usage=""" %s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath] To write image pickles use -p, otherwise the program writes CSPAD CBFs. Writing CBFs requires the geometry to be already deployed. Examples: cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 Use one process on the current node to process all the events from run 25 of experiment cxi49812, using a detz_offset of 571. mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 As above, using 16 cores on the current node. bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812 As above, using the psanaq and 100 cores, putting the log in average.out and the output images in the folder cxi49812. """ % libtbx.env.dispatcher_name).option( None, "--as_pickle", "-p", action="store_true", default=False, dest="as_pickle", help="Write results as image pickle files instead of cbf files" ).option( None, "--raw_data", "-R", action="store_true", default=False, dest="raw_data", help= "Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)" ).option( None, "--background_pickle", "-B", default=None, dest="background_pickle", help="" ).option( None, "--config", "-c", type="string", default=None, dest="config", metavar="PATH", help="psana config file" ).option( None, "--experiment", "-x", type="string", default=None, dest="experiment", help="experiment name (eg cxi84914)" ).option( None, "--run", "-r", type="int", default=None, dest="run", help="run number" ).option( None, "--address", "-a", type="string", default="CxiDs2.0:Cspad.0", dest="address", help="detector address name (eg CxiDs2.0:Cspad.0)" ).option( None, "--detz_offset", "-d", type="float", default=None, dest="detz_offset", help= "offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)" ).option( None, "--outputdir", "-o", type="string", default=".", dest="outputdir", metavar="PATH", help="Optional path to output directory for output files" ).option( None, "--averagebase", "-A", type="string", default="{experiment!l}_avg-r{run:04d}", dest="averagepath", metavar="PATH", help= "Path to output average image without extension. String substitution allowed" ).option( None, "--stddevbase", "-S", type="string", default="{experiment!l}_stddev-r{run:04d}", dest="stddevpath", metavar="PATH", help= "Path to output standard deviation image without extension. String substitution allowed" ).option( None, "--maxbase", "-M", type="string", default="{experiment!l}_max-r{run:04d}", dest="maxpath", metavar="PATH", help= "Path to output maximum projection image without extension. String substitution allowed" ).option( None, "--numevents", "-n", type="int", default=None, dest="numevents", help="Maximum number of events to process. Default: all" ).option( None, "--skipevents", "-s", type="int", default=0, dest="skipevents", help="Number of events in the beginning of the run to skip. Default: 0" ).option( None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress" ).option( None, "--pickle-optical-metrology", "-m", action="store_true", default=False, dest="pickle_optical_metrology", help= "If writing pickle files, use the optical metrology in the experiment's calib directory" ).option( None, "--bin_size", "-b", type="int", default=None, dest="bin_size", help="Rayonix detector bin size" ).option( None, "--override_beam_x", "-X", type="float", default=None, dest="override_beam_x", help="Rayonix detector beam center x coordinate" ).option( None, "--override_beam_y", "-Y", type="float", default=None, dest="override_beam_y", help="Rayonix detector beam center y coordinate" ).option( None, "--calib_dir", "-C", type="string", default=None, dest="calib_dir", metavar="PATH", help="calibration directory" ).option( None, "--pickle_calib_dir", "-P", type="string", default=None, dest="pickle_calib_dir", metavar="PATH", help= "pickle calibration directory specification. Replaces --calib_dir functionality." ).option( None, "--xtc_dir", "-D", type="string", default=None, dest="xtc_dir", metavar="PATH", help="xtc stream directory" ).option( None, "--use_ffb", "-f", action="store_true", default=False, dest="use_ffb", help= "Use the fast feedback filesystem at LCLS. Only for the active experiment!" ).option( None, "--gain_mask_value", "-g", type="float", default=None, dest="gain_mask_value", help= "Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode." ).option( None, "--min", None, action="store_true", default=False, dest="do_minimum_projection", help="Output a minimum projection" ).option( None, "--minpath", None, type="string", default="{experiment!l}_min-r{run:04d}", dest="minpath", metavar="PATH", help= "Path to output minimum image without extension. String substitution allowed" )).process(args=argv) if len(command_line.args) > 0 or \ command_line.options.as_pickle is None or \ command_line.options.experiment is None or \ command_line.options.run is None or \ command_line.options.address is None or \ command_line.options.detz_offset is None or \ command_line.options.averagepath is None or \ command_line.options.stddevpath is None or \ command_line.options.maxpath is None or \ command_line.options.pickle_optical_metrology is None: command_line.parser.show_help() return # set this to sys.maxint to analyze all events if command_line.options.numevents is None: maxevents = sys.maxsize else: maxevents = command_line.options.numevents comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if command_line.options.config is not None: psana.setConfigFile(command_line.options.config) dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment, command_line.options.run) if command_line.options.xtc_dir is not None: if command_line.options.use_ffb: raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system") dataset_name += ":dir=%s" % command_line.options.xtc_dir elif command_line.options.use_ffb: # as ffb is only at SLAC, ok to hardcode /reg/d here dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % ( command_line.options.experiment[0:3], command_line.options.experiment) if command_line.options.calib_dir is not None: psana.setOption('psana.calib-dir', command_line.options.calib_dir) ds = psana.DataSource(dataset_name) address = command_line.options.address src = psana.Source('DetInfo(%s)' % address) nevent = np.array([0.]) if command_line.options.background_pickle is not None: background = easy_pickle.load( command_line.options.background_pickle)['DATA'].as_numpy_array() for run in ds.runs(): runnumber = run.run() if not command_line.options.as_pickle: psana_det = psana.Detector(address, ds.env()) # list of all events if command_line.options.skipevents > 0: print("Skipping first %d events" % command_line.options.skipevents) elif "Rayonix" in command_line.options.address: print("Skipping first image in the Rayonix detector" ) # Shuttering issue command_line.options.skipevents = 1 for i, evt in enumerate(run.events()): if i % size != rank: continue if i < command_line.options.skipevents: continue if i >= maxevents: break if i % 10 == 0: print('Rank', rank, 'processing event', i) #print "Event #",rank*mylength+i," has id:",evt.get(EventId) if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: data = evt.get(psana.Camera.FrameV1, src) if data is None: print("No data") continue data = data.data16().astype(np.float64) elif command_line.options.as_pickle: data = evt.get(psana.ndarray_float64_3, src, 'image0') else: # get numpy array, 32x185x388 from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data if command_line.options.raw_data: data = get_psana_corrected_data(psana_det, evt, use_default=False, dark=False, common_mode=None, apply_gain_mask=False, per_pixel_gain=False) else: if command_line.options.gain_mask_value is None: data = get_psana_corrected_data(psana_det, evt, use_default=True) else: data = get_psana_corrected_data( psana_det, evt, use_default=False, dark=True, common_mode=None, apply_gain_mask=True, gain_mask_value=command_line.options. gain_mask_value, per_pixel_gain=False) if data is None: print("No data") continue if command_line.options.background_pickle is not None: data -= background if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: distance = np.array([0.0]) wavelength = np.array([1.0]) else: d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset) if d is None: print("No distance, using distance", command_line.options.detz_offset) assert command_line.options.detz_offset is not None if 'distance' not in locals(): distance = np.array([command_line.options.detz_offset]) else: distance += command_line.options.detz_offset else: if 'distance' in locals(): distance += d else: distance = np.array([float(d)]) w = cspad_tbx.evt_wavelength(evt) if w is None: print("No wavelength") if 'wavelength' not in locals(): wavelength = np.array([1.0]) else: if 'wavelength' in locals(): wavelength += w else: wavelength = np.array([w]) t = cspad_tbx.evt_time(evt) if t is None: print("No timestamp, skipping shot") continue if 'timestamp' in locals(): timestamp += t[0] + (t[1] / 1000) else: timestamp = np.array([t[0] + (t[1] / 1000)]) if 'sum' in locals(): sum += data else: sum = np.array(data, copy=True) if 'sumsq' in locals(): sumsq += data * data else: sumsq = data * data if 'maximum' in locals(): maximum = np.maximum(maximum, data) else: maximum = np.array(data, copy=True) if command_line.options.do_minimum_projection: if 'minimum' in locals(): minimum = np.minimum(minimum, data) else: minimum = np.array(data, copy=True) nevent += 1 #sum the images across mpi cores if size > 1: print("Synchronizing rank", rank) totevent = np.zeros(nevent.shape) comm.Reduce(nevent, totevent) if rank == 0 and totevent[0] == 0: raise Sorry("No events found in the run") sumall = np.zeros(sum.shape).astype(sum.dtype) comm.Reduce(sum, sumall) sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype) comm.Reduce(sumsq, sumsqall) maxall = np.zeros(maximum.shape).astype(maximum.dtype) comm.Reduce(maximum, maxall, op=MPI.MAX) if command_line.options.do_minimum_projection: minall = np.zeros(maximum.shape).astype(minimum.dtype) comm.Reduce(minimum, minall, op=MPI.MIN) waveall = np.zeros(wavelength.shape).astype(wavelength.dtype) comm.Reduce(wavelength, waveall) distall = np.zeros(distance.shape).astype(distance.dtype) comm.Reduce(distance, distall) timeall = np.zeros(timestamp.shape).astype(timestamp.dtype) comm.Reduce(timestamp, timeall) if rank == 0: if size > 1: print("Synchronized") # Accumulating floating-point numbers introduces errors, # which may cause negative variances. Since a two-pass # approach is unacceptable, the standard deviation is # clamped at zero. mean = sumall / float(totevent[0]) variance = (sumsqall / float(totevent[0])) - (mean**2) variance[variance < 0] = 0 stddev = np.sqrt(variance) wavelength = waveall[0] / totevent[0] distance = distall[0] / totevent[0] pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value timestamp = timeall[0] / totevent[0] timestamp = (int(timestamp), timestamp % int(timestamp) * 1000) timestamp = cspad_tbx.evt_timestamp(timestamp) if command_line.options.as_pickle: extension = ".pickle" else: extension = ".cbf" dest_paths = [ cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env()) ] if command_line.options.do_minimum_projection: dest_paths.append( cspad_tbx.pathsubst(command_line.options.minpath + extension, evt, ds.env())) dest_paths = [ os.path.join(command_line.options.outputdir, path) for path in dest_paths ] if 'Rayonix' in command_line.options.address: all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) from xfel.cxi.cspad_ana import rayonix_tbx pixel_size = rayonix_tbx.get_rayonix_pixel_size( command_line.options.bin_size) beam_center = [ command_line.options.override_beam_x, command_line.options.override_beam_y ] active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]]) split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] for data, path in zip(all_data, dest_paths): print("Saving", path) d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=flex.double(data), distance=distance, pixel_size=pixel_size, saturated_value=rayonix_tbx.rayonix_saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: all_data = [mean, stddev, maxall] split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): d = cspad_tbx.dpack(address=old_style_address, data=flex.double(data), distance=distance, pixel_size=0.1, timestamp=timestamp, wavelength=wavelength) print("Saving", path) easy_pickle.dump(path, d) elif command_line.options.as_pickle: split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] xpp = 'xpp' in address.lower() if xpp: evt_time = cspad_tbx.evt_time( evt) # tuple of seconds, milliseconds timestamp = cspad_tbx.evt_timestamp( evt_time) # human readable format from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version( old_style_address, reverse_timestamp(timestamp)[0]) assert version_lookup is not None active_areas = xpp_active_areas[version_lookup]['active_areas'] beam_center = [1765 // 2, 1765 // 2] else: if command_line.options.pickle_calib_dir is not None: metro_path = command_line.options.pickle_calib_dir elif command_line.options.pickle_optical_metrology: from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path metro_path = get_calib_file_path(run.env(), address, run) else: metro_path = libtbx.env.find_in_repositories( "xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0") sections = parse_calib.calib2sections(metro_path) beam_center, active_areas = cspad_tbx.cbcaa( cspad_tbx.getConfig(address, ds.env()), sections) class fake_quad(object): def __init__(self, q, d): self.q = q self.d = d def quad(self): return self.q def data(self): return self.d if xpp: quads = [ fake_quad(i, mean[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [ fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [ fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [ fake_quad(i, minall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] minall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) minall = flex.double(minall.astype(np.float64)) else: quads = [ fake_quad(i, mean[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] mean = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [ fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] stddev = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [ fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] maxall = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [ fake_quad(i, minall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] minall = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) minall = flex.double(minall.astype(np.float64)) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print("Saving", path) d = cspad_tbx.dpack(active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=data, distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) else: # load a header only cspad cbf from the slac metrology from xfel.cftbx.detector import cspad_cbf_tbx import pycbf base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology( run, address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d" % run.run()) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print("Saving", path) cspad_img = cspad_cbf_tbx.format_object_from_data( base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=False) cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def __init__(self, pixel_histograms, output_dirname=".", gain_map_path=None, gain_map=None, method="photon_counting", estimated_gain=30, nproc=None, photon_threshold=2 / 3, roi=None, run=None): assert method in ("sum_adu", "photon_counting") self.sum_img = flex.double(flex.grid( 370, 391), 0) # XXX define the image size some other way? gain_img = flex.double(self.sum_img.accessor(), 0) assert [gain_map, gain_map_path].count(None) > 0 if gain_map_path is not None: d = easy_pickle.load(gain_map_path) gain_map = d["DATA"] two_photon_threshold = photon_threshold + 1 mask = flex.int(self.sum_img.accessor(), 0) start_row = 370 end_row = 0 print(len(pixel_histograms.histograms)) pixels = list(pixel_histograms.pixels()) n_pixels = len(pixels) if roi is not None: for k, (i, j) in enumerate(reversed(pixels)): if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]): del pixels[n_pixels - k - 1] if gain_map is None: fixed_func = pixel_histograms.fit_one_histogram else: def fixed_func(pixel): return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1) results = None if nproc is None: nproc = easy_mp.Auto nproc = easy_mp.get_processes(nproc) print("nproc: ", nproc) stdout_and_results = easy_mp.pool_map( processes=nproc, fixed_func=fixed_func, args=pixels, func_wrapper="buffer_stdout_stderr") results = [r for so, r in stdout_and_results] gains = flex.double() for i, pixel in enumerate(pixels): start_row = min(start_row, pixel[0]) end_row = max(end_row, pixel[0]) n_photons = 0 if results is None: # i.e. not multiprocessing try: gaussians = pixel_histograms.fit_one_histogram(pixel) except RuntimeError as e: print("Error fitting pixel %s" % str(pixel)) print(str(e)) mask[pixel] = 1 continue else: gaussians = results[i] hist = pixel_histograms.histograms[pixel] if gaussians is None: # Presumably the peak fitting failed in some way print("Skipping pixel %s" % str(pixel)) continue zero_peak_diff = gaussians[0].params[1] if gain_map is None: try: view_pixel_histograms.check_pixel_histogram_fit( hist, gaussians) except view_pixel_histograms.PixelFitError as e: print("PixelFitError:", str(pixel), str(e)) mask[pixel] = 1 continue gain = gaussians[1].params[1] - gaussians[0].params[1] gain_img[pixel] = gain gain_ratio = gain / estimated_gain else: gain = gain_map[pixel] if gain == 0: print("bad gain!!!!!", pixel) continue gain = 30 / gain gain_ratio = 1 / gain gains.append(gain) #for g in gaussians: #sigma = abs(g.params[2]) #if sigma < 1 or sigma > 10: #print "bad sigma!!!!!", pixel, sigma #mask[pixel] = 1 #continue if method == "sum_adu": sum_adu = 0 one_photon_cutoff, two_photon_cutoff = [ (threshold * gain + zero_peak_diff) for threshold in (photon_threshold, two_photon_threshold) ] i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff) slots = hist.slots().as_double() slot_centers = hist.slot_centers() slots -= gaussians[0](slot_centers) for j in range(i_one_photon_cutoff, len(slots)): center = slot_centers[j] sum_adu += slots[j] * (center - zero_peak_diff) * 30 / gain self.sum_img[pixel] = sum_adu elif method == "photon_counting": one_photon_cutoff, two_photon_cutoff = [ (threshold * gain + zero_peak_diff) for threshold in (photon_threshold, two_photon_threshold) ] i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff) i_two_photon_cutoff = hist.get_i_slot(two_photon_cutoff) slots = hist.slots() for j in range(i_one_photon_cutoff, len(slots)): if j == i_one_photon_cutoff: center = hist.slot_centers()[j] upper = center + 0.5 * hist.slot_width() n_photons += int( round((upper - one_photon_cutoff) / hist.slot_width() * slots[j])) elif j == i_two_photon_cutoff: center = hist.slot_centers()[j] upper = center + 0.5 * hist.slot_width() n_photons += 2 * int( round((upper - two_photon_cutoff) / hist.slot_width() * slots[j])) elif j < i_two_photon_cutoff: n_photons += int(round(slots[j])) else: n_photons += 2 * int(round(slots[j])) self.sum_img[pixel] = n_photons stats = scitbx.math.basic_statistics(gains) print("gain statistics:") stats.show() mask.set_selected(self.sum_img == 0, 1) unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask() mask.set_selected(unbound_pixel_mask > 0, 1) bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7() mask.set_selected(bad_pixel_mask > 0, 1) for row in range(self.sum_img.all()[0]): self.sum_img[row:row + 1, :].count(0) spectrum_focus = self.sum_img[start_row:end_row, :] mask_focus = mask[start_row:end_row, :] spectrum_focus.set_selected(mask_focus > 0, 0) xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus) print("Number of rows: %i" % spectrum_focus.all()[0]) print("Estimated no. photons counted: %i" % flex.sum(spectrum_focus)) print("Number of images used: %i" % flex.sum(pixel_histograms.histograms.values()[0].slots())) d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=spectrum_focus, distance=1, ccd_image_saturation=2e8, # XXX ) if run is not None: runstr = "_%04d" % run else: runstr = "" cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr) if gain_map is None: gain_map = flex.double(gain_img.accessor(), 0) img_sel = (gain_img > 0).as_1d() d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0', data=gain_img, distance=1) cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_') gain_map.as_1d().set_selected(img_sel.iselection(), 1 / gain_img.as_1d().select(img_sel)) gain_map /= flex.mean(gain_map.as_1d().select(img_sel)) d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0', data=gain_map, distance=1) cspad_tbx.dwritef(d, output_dirname, 'gain_map_') plot_x, plot_y = xes_finalise.output_spectrum( spectrum_focus.iround(), mask_focus=mask_focus, output_dirname=output_dirname, run=run) self.spectrum = (plot_x, plot_y) self.spectrum_focus = spectrum_focus xes_finalise.output_matlab_form( spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr)) print(output_dirname)
def __init__(self, runs, output_dirname=".", roi=None): avg_basename = "avg_" stddev_basename = "stddev" self.sum_img = None self.sumsq_img = None self.nmemb = 0 self.roi = cspad_tbx.getOptROI(roi) self.unbound_pixel_mask = cspad_unbound_pixel_mask() for i_run, run in enumerate(runs): run_scratch_dir = run result = finalise_one_run(run_scratch_dir) if result.sum_img is None: continue if self.sum_img is None: self.sum_img = result.sum_img self.sumsq_img = result.sumsq_img else: self.sum_img += result.sum_img self.sumsq_img += result.sumsq_img self.nmemb += result.nmemb self.avg_img = self.sum_img.as_double() / self.nmemb self.stddev_img = flex.sqrt( (self.sumsq_img.as_double() - self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1)) self.mask = flex.int(self.sum_img.accessor(), 0) self.mask.set_selected(self.sum_img == 0, 1) self.mask.set_selected(self.unbound_pixel_mask > 0, 1) if (output_dirname is not None and avg_basename is not None): if (not os.path.isdir(output_dirname)): os.makedirs(output_dirname) d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=self.avg_img, distance=1, ) cspad_tbx.dwritef(d, output_dirname, avg_basename) d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=self.sum_img, distance=1, ) cspad_tbx.dwritef(d, output_dirname, "sum_") if 1: output_image(self.avg_img, "%s/avg.png" % output_dirname) output_image(self.avg_img, "%s/avg_inv.png" % output_dirname, invert=True) if 1: output_matlab_form(self.sum_img, "%s/sum.m" % output_dirname) output_matlab_form(self.avg_img, "%s/avg.m" % output_dirname) output_matlab_form(self.stddev_img, "%s/stddev.m" % output_dirname) if (stddev_basename is not None): d = cspad_tbx.dpack( address='CxiSc1-0|Cspad2x2-0', data=self.stddev_img, distance=1, ) cspad_tbx.dwritef(d, output_dirname, stddev_basename) # XXX we should really figure out automatically the area where the spectrum is #write an integrated spectrum from lines 186-227 #spectrum_focus = self.sum_img.as_numpy_array()[186:228,:] img = self.sum_img if self.roi is None: spectrum_focus = img mask_focus = self.mask else: slices = (slice(self.roi[2], self.roi[3]), slice(self.roi[0], self.roi[1])) spectrum_focus = img[slices] mask_focus = self.mask[slices] if False: from matplotlib import pylab pylab.imshow(spectrum_focus.as_numpy_array()) pylab.show() output_spectrum(spectrum_focus, mask_focus=mask_focus, output_dirname=output_dirname) print "Total number of images used from %i runs: %i" % (i_run + 1, self.nmemb)
def run(args): command_line = (option_parser().option( "-o", "--output_filename", action="store", type="string", help="Filename for the output pickle file", default="gain_map.pickle" ).option( "-f", "--detector_format_version", action="store", type="string", help= "Detector format version to use for generating active areas and laying out tiles", default=None ).option( "-m", "--optical_metrology_path", action="store", type="string", help= "Path to slac optical metrology file. If not set, use Run 4 metrology", default=None ).option( "-d", "--distance", action="store", type="int", help= "Detector distance put into the gain pickle file. Not needed for processing.", default="0" ).option( "-w", "--wavelength", action="store", type="float", help= "Incident beam wavelength put into the gain pickle file. Not needed for processing.", default="0")).process(args=args) output_filename = command_line.options.output_filename detector_format_version = command_line.options.detector_format_version if detector_format_version is None or 'XPP' not in detector_format_version: beam_center_x = None beam_center_y = None else: beam_center_x = 1765 // 2 * 0.11 beam_center_y = 1765 // 2 * 0.11 address, timestamp = address_and_timestamp_from_detector_format_version( detector_format_version) # if no detector format version is provided, make sure to write no address to the image pickle # but CsPadDetector (called later), needs an address, so give it a fake one save_address = address is not None if not save_address: address = "CxiDs1-0|Cspad-0" # time stamp will still be None timestamp = evt_timestamp((timestamp, 0)) args = command_line.args assert len(args) == 1 if args[0].endswith('.npy'): data = numpy.load(args[0]) det, active_areas = convert_2x2(data, detector_format_version, address) elif args[0].endswith('.txt') or args[0].endswith('.gain'): raw_data = numpy.loadtxt(args[0]) assert raw_data.shape in [(5920, 388), (11840, 194)] det, active_areas = convert_detector( raw_data, detector_format_version, address, command_line.options.optical_metrology_path) img_diff = det img_sel = (img_diff > 0).as_1d() gain_map = flex.double(img_diff.accessor(), 0) gain_map.as_1d().set_selected(img_sel.iselection(), 1 / img_diff.as_1d().select(img_sel)) gain_map /= flex.mean(gain_map.as_1d().select(img_sel)) if not save_address: address = None d = cspad_tbx.dpack(data=gain_map, address=address, active_areas=active_areas, timestamp=timestamp, distance=command_line.options.distance, wavelength=command_line.options.wavelength, beam_center_x=beam_center_x, beam_center_y=beam_center_y) easy_pickle.dump(output_filename, d)
class SingleImage(object): def __init__(self, img, init, verbose=True, imported_grid=None): """ Constructor for the SingleImage object using a raw image file or pickle """ # Initialize parameters self.params = init.params self.args = init.args self.user_id = init.user_id self.raw_img = img[2] self.conv_img = img[2] self.img_index = img[0] self.center_int = 0 self.status = None self.fail = None self.final = None self.log_info = [] self.gs_results = [] self.main_log = init.logfile self.verbose = verbose self.hmed = self.params.cctbx.grid_search.height_median self.amed = self.params.cctbx.grid_search.area_median self.input_base = init.input_base self.conv_base = init.conv_base self.int_base = init.int_base self.obj_base = init.obj_base self.fin_base = init.fin_base self.viz_base = init.viz_base self.log_base = init.log_base self.tmp_base = init.tmp_base self.abort_file = os.path.join(self.int_base, '.abort.tmp') self.obj_path = None self.obj_file = None self.fin_path = None self.fin_file = None self.viz_path = None # ============================== SELECTION-ONLY FUNCTIONS ============================== # def import_int_file(self, init): """ Replaces path settings in imported image object with new settings NEED TO RE-DO LATER """ if os.path.isfile(self.abort_file): self.fail = 'aborted' return self # Generate paths to output files self.params = init.params self.main_log = init.logfile self.input_base = init.input_base self.conv_base = init.conv_base self.int_base = init.int_base self.obj_base = init.obj_base self.fin_base = init.fin_base self.log_base = init.log_base self.viz_base = init.viz_base self.obj_path = misc.make_image_path(self.conv_img, self.input_base, self.obj_base) self.obj_file = os.path.abspath( os.path.join( self.obj_path, os.path.basename(self.conv_img).split('.')[0] + ".int")) self.fin_path = misc.make_image_path(self.conv_img, self.input_base, self.fin_base) self.log_path = misc.make_image_path(self.conv_img, self.input_base, self.log_base) self.fin_file = os.path.abspath( os.path.join( self.fin_path, os.path.basename(self.conv_img).split('.')[0] + "_int.pickle")) self.final['final'] = self.fin_file self.final['img'] = self.conv_img self.viz_path = misc.make_image_path(self.conv_img, self.input_base, self.viz_base) self.viz_file = os.path.join( self.viz_path, os.path.basename(self.conv_img).split('.')[0] + "_int.png") # Create actual folders (if necessary) try: if not os.path.isdir(self.obj_path): os.makedirs(self.obj_path) if not os.path.isdir(self.fin_path): os.makedirs(self.fin_path) if not os.path.isdir(self.viz_path): os.makedirs(self.viz_path) except OSError: pass # Grid search / integration log file self.int_log = os.path.join( self.log_path, os.path.basename(self.conv_img).split('.')[0] + '.tmp') # Reset status to 'grid search' to pick up at selection (if no fail) if self.fail == None: self.status = 'bypass grid search' return self def determine_gs_result_file(self): """ For 'selection-only' cctbx.xfel runs, determine where the image objects are """ if self.params.cctbx.selection.select_only.grid_search_path != None: obj_path = os.path.abspath( self.params.cctbx.selection.select_only.grid_search_path) else: run_number = int(os.path.basename(self.int_base)) - 1 obj_path = "{}/integration/{:03d}/image_objects"\ "".format(os.path.abspath(os.curdir), run_number) gs_result_file = os.path.join(obj_path, os.path.basename(self.obj_file)) return gs_result_file # =============================== IMAGE IMPORT FUNCTIONS =============================== # def load_image(self): """ Reads raw image file and extracts data for conversion into pickle format. Also estimates gain if turned on.""" # Load raw image or image pickle try: with misc.Capturing() as junk_output: loaded_img = dxtbx.load(self.raw_img) except Exception, e: print 'IOTA IMPORT ERROR:', e loaded_img = None pass # Extract image information if loaded_img is not None: raw_data = loaded_img.get_raw_data() detector = loaded_img.get_detector()[0] beam = loaded_img.get_beam() scan = loaded_img.get_scan() distance = detector.get_distance() pixel_size = detector.get_pixel_size()[0] overload = detector.get_trusted_range()[1] wavelength = beam.get_wavelength() beam_x = detector.get_beam_centre(beam.get_s0())[0] beam_y = detector.get_beam_centre(beam.get_s0())[1] if scan is None: timestamp = None img_type = 'pickle' else: img_type = 'raw' msec, sec = math.modf(scan.get_epochs()[0]) timestamp = evt_timestamp((sec, msec)) # Assemble datapack data = dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data['OSC_START'] = 0 #osc_start data['OSC_RANGE'] = 0 #osc_start data['TIME'] = scan.get_exposure_times()[0] else: data = None img_type = 'not imported' # Estimate gain (or set gain to 1.00 if cannot calculate) # Cribbed from estimate_gain.py by Richard Gildea if self.params.advanced.estimate_gain: from dxtbx.datablock import DataBlockFactory from dials.command_line.estimate_gain import estimate_gain with misc.Capturing() as junk_output: try: datablock = DataBlockFactory.from_filenames([self.raw_img ])[0] imageset = datablock.extract_imagesets()[0] self.gain = estimate_gain(imageset) except Exception, e: self.gain = 1.0
def run(argv=None): if argv is None: argv = sys.argv[1:] command_line = (libtbx.option_parser.option_parser( usage= "%s [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files" % libtbx.env.dispatcher_name ).option( None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress" ).option( None, "--crop", "-c", action="store_true", default=False, dest="crop", help="Crop the image such that the beam center is in the middle" ).option( None, "--skip_converted", "-s", action="store_true", default=False, dest="skip_converted", help= "Skip converting if an image already exist that matches the destination file name" ).option( None, "--wavelength", "-w", type="float", default=None, dest="wavelength", help="Override the image's wavelength (angstroms)").option( None, "--distance", "-d", type="float", default=None, dest="distance", help="Override the detector distance (mm)").option( None, "--pixel_size", "-p", type="float", default=None, dest="pixel_size", help="Override the detector pixel size (mm)").option( None, "--beam_x", "-x", type="float", default=None, dest="beam_center_x", help="Override the beam x position (pixels)").option( None, "--beam_y", "-y", type="float", default=None, dest="beam_center_y", help="Override the beam y position (pixels)").option( None, "--overload", "-o", type="float", default=None, dest="overload", help="Override the detector overload value (ADU)") ).process(args=argv) paths = command_line.args if len(paths) <= 0: raise Usage("No files specified") for imgpath in paths: destpath = os.path.join( os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + ".pickle") if command_line.options.skip_converted and os.path.isfile(destpath): if command_line.options.verbose: print "Skipping %s, file exists" % imgpath continue if command_line.options.verbose: print "Converting %s to %s..." % (imgpath, destpath) try: img = dxtbx.load(imgpath) except IOError: img = None pass if img is None: import numpy as np try: raw_data = np.loadtxt(imgpath) from scitbx.array_family import flex raw_data = flex.double(raw_data.astype(np.double)) except ValueError: raise Usage("Couldn't load %s, no supported readers" % imgpath) detector = None beam = None scan = None else: raw_data = img.get_raw_data() detector = img.get_detector() beam = img.get_beam() scan = img.get_scan() if detector is None: if command_line.options.distance is None: raise Usage("Can't get distance from image. Override with -d") if command_line.options.pixel_size is None: raise Usage( "Can't get pixel size from image. Override with -p") if command_line.options.overload is None: raise Usage( "Can't get overload value from image. Override with -o") distance = command_line.options.distance pixel_size = command_line.options.pixel_size overload = command_line.options.overload else: detector = detector[0] if command_line.options.distance is None: distance = detector.get_directed_distance() else: distance = command_line.options.distance if command_line.options.pixel_size is None: pixel_size = detector.get_pixel_size()[0] else: pixel_size = command_line.options.pixel_size if command_line.options.overload is None: overload = detector.get_trusted_range()[1] else: overload = command_line.options.overload if beam is None: if command_line.options.wavelength is None: raise Usage( "Can't get wavelength from image. Override with -w") wavelength = command_line.options.wavelength else: if command_line.options.wavelength is None: wavelength = beam.get_wavelength() else: wavelength = command_line.options.wavelength if beam is None and detector is None: if command_line.options.beam_center_x is None: print "Can't get beam x position from image. Using image center. Override with -x" beam_x = raw_data.focus()[0] * pixel_size else: beam_x = command_line.options.beam_center_x * pixel_size if command_line.options.beam_center_y is None: print "Can't get beam y position from image. Using image center. Override with -y" beam_y = raw_data.focus()[1] * pixel_size else: beam_y = command_line.options.beam_center_y * pixel_size else: if command_line.options.beam_center_x is None: beam_x = detector.get_beam_centre(beam.get_s0())[0] else: beam_x = command_line.options.beam_center_x * pixel_size if command_line.options.beam_center_y is None: beam_y = detector.get_beam_centre(beam.get_s0())[1] else: beam_y = command_line.options.beam_center_y * pixel_size if scan is None: timestamp = None else: msec, sec = math.modf(scan.get_epochs()[0]) timestamp = evt_timestamp((sec, msec)) data = dpack(data=raw_data, distance=distance, pixel_size=pixel_size, wavelength=wavelength, beam_center_x=beam_x, beam_center_y=beam_y, ccd_image_saturation=overload, saturated_value=overload, timestamp=timestamp) if scan is not None: osc_start, osc_range = scan.get_oscillation() if osc_start != osc_range: data['OSC_START'] = osc_start data['OSC_RANGE'] = osc_range data['TIME'] = scan.get_exposure_times()[0] if command_line.options.crop: data = crop_image_pickle(data) easy_pickle.dump(destpath, data)
def endjob(self, obj1, obj2=None): """The endjob() function writes the mean and standard deviation images to disk. @param evt Event object (psana only) @param env Environment object """ if obj2 is None: env = obj1 else: evt = obj1 env = obj2 stats = super(mod_average, self).endjob(env) if stats is None: return device = cspad_tbx.address_split(self.address)[2] if device == 'Andor': beam_center = (0, 0) # XXX Fiction! pixel_size = 13.5e-3 # XXX Should not be hardcoded here! saturated_value = 10000 elif device == 'Cspad' or device == 'Cspad2x2': beam_center = self.beam_center pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value elif device == 'marccd': beam_center = tuple(t // 2 for t in d['mean_img'].focus()) pixel_size = 0.079346 saturated_value = 2**16 - 1 if stats['nmemb'] > 0: if self.avg_dirname is not None or \ self.avg_basename is not None or \ self._mean_out is not None: d = cspad_tbx.dpack(active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=stats['mean_img'], distance=stats['distance'], pixel_size=pixel_size, saturated_value=saturated_value, timestamp=cspad_tbx.evt_timestamp( stats['time']), wavelength=stats['wavelength']) if self._mean_out is not None: p = cspad_tbx.dwritef2(d, self._mean_out) else: p = cspad_tbx.dwritef(d, self.avg_dirname, self.avg_basename) self.logger.info("Average written to %s" % p) if self.stddev_dirname is not None or \ self.stddev_basename is not None or \ self._std_out is not None: d = cspad_tbx.dpack(active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=stats['std_img'], distance=stats['distance'], pixel_size=pixel_size, saturated_value=saturated_value, timestamp=cspad_tbx.evt_timestamp( stats['time']), wavelength=stats['wavelength']) if self._std_out is not None: p = cspad_tbx.dwritef2(d, self._std_out) else: p = cspad_tbx.dwritef(d, self.stddev_dirname, self.stddev_basename) self.logger.info("Standard deviation written to %s" % p) if self.max_dirname is not None or \ self.max_basename is not None or \ self._max_out is not None: d = cspad_tbx.dpack(active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=stats['max_img'], distance=stats['distance'], pixel_size=pixel_size, saturated_value=saturated_value, timestamp=cspad_tbx.evt_timestamp( stats['time']), wavelength=stats['wavelength']) if self._max_out is not None: p = cspad_tbx.dwritef2(d, self._max_out) else: p = cspad_tbx.dwritef(d, self.max_dirname, self.max_basename) self.logger.info("Max written to %s" % p) if stats['nfail'] == 0: self.logger.info("%d images processed" % stats['nmemb']) else: self.logger.warning("%d images processed, %d failed" % (stats['nmemb'], stats['nfail']))
def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ super(mod_image_dict, self).event(evt, env) if (evt.get("skip_event")): return if self.cspad_img is None: return # This module only applies to detectors for which a distance is # available. distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: self.nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), "skip_event") return device = cspad_tbx.address_split(self.address)[2] self.logger.info("Subprocess %02d: process image #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) # See r17537 of mod_average.py. if device == 'Cspad': pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value elif device == 'Rayonix': pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) saturated_value = rayonix_tbx.rayonix_saturated_value elif device == 'marccd': pixel_size = evt.get("marccd_pixel_size") saturated_value = evt.get("marccd_saturated_value") if distance == 0: distance = evt.get("marccd_distance") d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=self.cspad_img.iround(), # XXX ouch! distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=self.timestamp, wavelength=self.wavelength) evt.put(d, self.m_out_key) # Diagnostic message emitted only when all the processing is done. if (env.subprocess() >= 0): self.logger.info("Subprocess %02d: accepted #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) else: self.logger.info("Accepted #%05d @ %s" % (self.nshots, self.timestamp))