def _beam(self, index=None): """Returns a simple model for the beam""" if index is None: index = 0 if self._beam_index != index: self._beam_index = index evt = self._get_event(index) if self.params.spectrum_eV_per_pixel is not None: if self._fee is None: self._fee = psana.Detector("FEE-SPEC0") fee = self._fee.get(evt) if fee is None: wavelength = cspad_tbx.evt_wavelength(evt) else: x = (self.params.spectrum_eV_per_pixel * np.array(range(len( fee.hproj())))) + self.params.spectrum_eV_offset wavelength = factor_ev_angstrom / np.average( x, weights=fee.hproj()) else: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: self._beam_cache = None else: if self.params.wavelength_offset is not None: wavelength += self.params.wavelength_offset self._beam_cache = self._beam_factory.simple(wavelength) s, nsec = evt.get(psana.EventId).time() evttime = time.gmtime(s) if (evttime.tm_year == 2020 and evttime.tm_mon >= 7) or evttime.tm_year > 2020: self._beam_cache.set_polarization_normal((1, 0, 0)) return self._beam_cache
def _beam(self, index=None): '''Returns a simple model for the beam ''' if index is None: index = 0 evt = self._get_event(index) wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: return None return self._beam_factory.simple(wavelength)
def event (self, evt, env) : if (evt.get("skip_event")) : return self.nshots += 1 s = None t = evt.getTime() if (t is not None): s = t.seconds() + (t.nanoseconds() / 1000000000) else : self.nfail += 1 self.logger.warning("event(): no timestamp, shot skipped") evt.put(skip_event_flag(), "skip_event") return if (not isinstance(s, float)) : raise RuntimeError("Wrong type for 's': %s" % type(s).__name__) # XXX This hardcodes the address for the front detector! det_z = cspad_tbx.env_detz('CxiDs1-0|Cspad-0', env) if (det_z is None): self.nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), "skip_event") return laser01 = cspad_tbx.env_laser_status(env, 1) if laser01 is None: self.nfail += 1 self.logger.warning("event(): no status for laser 1, shot skipped") evt.put(skip_event_flag(), 'skip_event') return laser04 = cspad_tbx.env_laser_status(env, 4) if laser04 is None: self.nfail += 1 self.logger.warning("event(): no status for laser 4, shot skipped") evt.put(skip_event_flag(), 'skip_event') return # Laser power for fourth laser. The control name was provided by # Jan Kern. XXX Move to its own function in cspad_tbx? laser04_power = None if env is not None: pv = env.epicsStore().value('CXI:LAS:MMN:02:ROT.RBV') if pv is not None and len(pv.values) == 1: laser04_power = pv.values[0] if laser04_power is None: self.nfail += 1 self.logger.warning("event(): no power for laser 4, shot skipped") evt.put(skip_event_flag(), 'skip_event') return si_foil = cspad_tbx.env_sifoil(env) if (si_foil is None): self.nfail += 1 self.logger.warning("event(): no Si-foil thickness, shot skipped") evt.put(skip_event_flag(), "skip_event") return if (not (isinstance(si_foil, float) or isinstance(si_foil, int))) : raise RuntimeError("Wrong type for 'si_foil': %s"% type(si_foil).__name__) wavelength = cspad_tbx.evt_wavelength(evt) if (wavelength is None): self.nfail += 1 self.logger.warning("event(): no wavelength, shot skipped") evt.put(skip_event_flag(), "skip_event") return # In order to keep all arrays the same length, only append once # all values have been successfully obtained. XXX Still bugs: see # June run 119. self._t.append(s) self._si_foil.append(si_foil) self._wavelength.append(wavelength) self._det_z.append(det_z) self._laser01.append(laser01) self._laser04.append(laser04) self._laser04_power.append(laser04_power) if (self.nshots % 120 == 0) : self.update_plot()
def process_event(self, run, timestamp): """ Process a single event from a run @param run psana run object @param timestamp psana timestamp object """ ts = cspad_tbx.evt_timestamp((timestamp.seconds(),timestamp.nanoseconds()/1e6)) if ts is None: print "No timestamp, skipping shot" return if len(self.params_cache.debug.event_timestamp) > 0 and ts not in self.params_cache.debug.event_timestamp: return if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events: if ts in self.known_events: if self.known_events[ts] not in ["stop", "done", "fail"]: if self.params_cache.debug.skip_bad_events: print "Skipping event %s: possibly caused an unknown exception previously"%ts return elif self.params_cache.debug.skip_processed_events: print "Skipping event %s: processed successfully previously"%ts return else: if self.params_cache.debug.skip_unprocessed_events: print "Skipping event %s: not processed previously"%ts return self.debug_start(ts) evt = run.event(timestamp) if evt.get("skip_event") or "skip_event" in [key.key() for key in evt.keys()]: print "Skipping event",ts self.debug_write("psana_skip", "skip") return print "Accepted", ts self.params = copy.deepcopy(self.params_cache) # the data needs to have already been processed and put into the event by psana if self.params.format.file_format == 'cbf': # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data(self.psana_det, evt, use_default=False, dark=True, common_mode=self.common_mode, apply_gain_mask=self.params.format.cbf.gain_mask_value is not None, gain_mask_value=self.params.format.cbf.gain_mask_value, per_pixel_gain=False) if data is None: print "No data" self.debug_write("no_data", "skip") return if self.params.format.cbf.override_distance is None: distance = cspad_tbx.env_distance(self.params.input.address, run.env(), self.params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" self.debug_write("no_distance", "skip") return else: distance = self.params.format.cbf.override_distance if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" self.debug_write("no_wavelength", "skip") return else: wavelength = 12398.4187/self.params.format.cbf.override_energy if self.params.format.file_format == 'pickle': image_dict = evt.get(self.params.format.pickle.out_key) data = image_dict['DATA'] timestamp = t = ts s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23] print "Processing shot", s if self.params.format.file_format == 'cbf': # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data(self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address) if self.params.input.reference_geometry is not None: from dxtbx.model import Detector # copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead. cspad_img._detector_instance = Detector.from_dict(self.reference_detector.to_dict()) cspad_img.sync_detector_to_cbf() elif self.params.format.file_format == 'pickle': from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory cspad_img = FormatPYunspecifiedStillInMemory(image_dict) cspad_img.timestamp = s if self.params.dispatch.dump_all: self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s)) self.cache_ranges(cspad_img, self.params) imgset = MemImageSet([cspad_img]) if self.params.dispatch.estimate_gain_only: from dials.command_line.estimate_gain import estimate_gain estimate_gain(imgset) return if not self.params.dispatch.find_spots: self.debug_write("data_loaded", "done") return datablock = DataBlockFactory.from_imageset(imgset)[0] # before calling DIALS for processing, set output paths according to the templates if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template: self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.indexed_filename_template%("idx-" + s)) if "%s" in self.refined_experiments_filename_template: self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.refined_experiments_filename_template%("idx-" + s)) if "%s" in self.integrated_filename_template: self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.integrated_filename_template%("idx-" + s)) if "%s" in self.reindexedstrong_filename_template: self.params.output.reindexedstrong_filename = os.path.join(self.params.output.output_dir, self.reindexedstrong_filename_template%("idx-" + s)) # Load a dials mask from the trusted range and psana mask from dials.util.masking import MaskGenerator generator = MaskGenerator(self.params.border_mask) mask = generator.generate(imgset) if self.params.format.file_format == "cbf": mask = tuple([a&b for a, b in zip(mask,self.dials_mask)]) if self.spotfinder_mask is None: self.params.spotfinder.lookup.mask = mask else: self.params.spotfinder.lookup.mask = tuple([a&b for a, b in zip(mask,self.spotfinder_mask)]) if self.integration_mask is None: self.params.integration.lookup.mask = mask else: self.params.integration.lookup.mask = tuple([a&b for a, b in zip(mask,self.integration_mask)]) self.debug_write("spotfind_start") try: observed = self.find_spots(datablock) except Exception, e: import traceback; traceback.print_exc() print str(e), "event", timestamp self.debug_write("spotfinding_exception", "fail") return
def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ # Increase the event counter, even if this event is to be skipped. self.nshots += 1 if (evt.get("skip_event")): return sifoil = cspad_tbx.env_sifoil(env) if self.check_beam_status and sifoil is None: self.nfail += 1 self.logger.warning("event(): no Si-foil thickness, shot skipped") evt.put(skip_event_flag(), "skip_event") return if (self.sifoil is not None and self.sifoil != sifoil): self.logger.warning( "event(): Si-foil changed mid-run: % 8i -> % 8d" % (self.sifoil, sifoil)) self.sifoil = sifoil if self.verbose: self.logger.info("Si-foil thickness: %i" % sifoil) self.evt_time = cspad_tbx.evt_time( evt) # tuple of seconds, milliseconds self.timestamp = cspad_tbx.evt_timestamp( self.evt_time) # human readable format if (self.timestamp is None): self.nfail += 1 self.logger.warning("event(): no timestamp, shot skipped") evt.put(skip_event_flag(), "skip_event") return if self.verbose: self.logger.info(self.timestamp) if self.override_energy is None: self.wavelength = cspad_tbx.evt_wavelength(evt, self.delta_k) if self.wavelength is None: if self.check_beam_status: self.nfail += 1 self.logger.warning("event(): no wavelength, shot skipped") evt.put(skip_event_flag(), "skip_event") return else: self.wavelength = 0 else: self.wavelength = 12398.4187 / self.override_energy if self.verbose: self.logger.info("Wavelength: %.4f" % self.wavelength) self.pulse_length = cspad_tbx.evt_pulse_length(evt) if self.pulse_length is None: if self.check_beam_status: self.nfail += 1 self.logger.warning("event(): no pulse length, shot skipped") evt.put(skip_event_flag(), "skip_event") return else: self.pulse_length = 0 if self.verbose: self.logger.info("Pulse length: %s" % self.pulse_length) self.beam_charge = cspad_tbx.evt_beam_charge(evt) if self.beam_charge is None: if self.check_beam_status: self.nfail += 1 self.logger.warning("event(): no beam charge, shot skipped") evt.put(skip_event_flag(), "skip_event") return else: self.beam_charge = 0 if self.verbose: self.logger.info("Beam charge: %s" % self.beam_charge) self.injector_xyz = cspad_tbx.env_injector_xyz(env) #if self.injector_xyz is not None: #self.logger.info("injector_z: %i" %self.injector_xyz[2].value) self.laser_1_status.set_status( cspad_tbx.env_laser_status(env, laser_id=1), self.evt_time) self.laser_4_status.set_status( cspad_tbx.env_laser_status(env, laser_id=4), self.evt_time) self.laser_1_ms_since_change = self.laser_1_status.ms_since_last_status_change( self.evt_time) self.laser_4_ms_since_change = self.laser_4_status.ms_since_last_status_change( self.evt_time) if self.verbose: if self.laser_1_ms_since_change is not None: self.logger.info("ms since laser 1 status change: %i" % self.laser_1_ms_since_change) if self.laser_4_ms_since_change is not None: self.logger.info("ms since laser 4 status change: %i" % self.laser_4_ms_since_change) if self.laser_1_status is not None and self.laser_1_status.status is not None: self.logger.info("Laser 1 status: %i" % int(self.laser_1_status.status)) if self.laser_4_status is not None and self.laser_4_status.status is not None: self.logger.info("Laser 4 status: %i" % int(self.laser_4_status.status))
def __init__(self, dataset_name=None, detector_address=None, data_type='idx', mask_path=None, mask_angles=None, mask_widths=None, backimg_path=None, backmsk_path=None, geom_path=None, det_dist=None, det_pix=0.075, beam_l=None, mask_thr=None, nQ=None, nPhi=None, dQ=1, dPhi=1, cent0=None, r_max=None, dr=None, dx=None, dy=None, r_0=None, q_bound=None, peak=None, dpeak=None): """The fluctuation scattering class stores processing parameters, initiates mask and background data and retrieves 2D images from events. Processing options of the 2D images include: * Transform from cartesian to polar coordinates * Beam center refinement * Dynamic masking * Normalization % SAXS calculation * Particle sizing * Computation of in-frame 2-point angular auto-correlations using FFTs @param dataset_name Experiment name and run number @param detector_address Adress to back or front detector @param data_type Type of data file format (h5 or xtc in the formats: idx|idx_ffb|smd|smd_ffb|h5) @param mask_path Full path to static image mask @param mask_angles Center of angluar slices (deg) that should be masked out (due to jet streaks etc), [Ang1 Ang2 ...] @param mask_widths Width of angular slices (deg) that should be masked out (due to jet streaks etc), [delta1 delta2 ...] @param backimg_path Full path to background image @param backmsk_path Full path to background mask @param geom_path Full path to geometry file (for h5 format) @param det_dist Override of detecor distance (in mm) @param det_pix Pixel size (in mm) @param beam_l Override of beam wavelength (in Angstrom) @param mask_thr Threshold for dynamic masking @param nQ Number of Q-bins to consider (in pixels) @param nPhi Number of Phi-bins to consider (in pixels) @param dQ Stepsize in Q (in pixels) @param dPhi Stepsize in Phi (in pixels) @param cent0 Initial beam center coordinates [xc,yc] @param r_max Maximum radial value to use for beamcenter refinement (in pixels) @param dr Stepsize in r (in pixels) @param dx Gridsize for beam center refinement in x, i.e xc+/-dx (in pixels) @param dy Gridsize for beam center refinement in y, i.e yc+/-dy (in pixles) @param r_0 Starting value for particle radius refinement [in Ang] @param q_bound Upper and Lower boundaries of q for Particle radius refinement [in Ang^-1] @param peak Q-values for peak maxima [q_peak1 q_peak2 ...] @param dpeak Delta Q used for peak integration of peak maxima [delta_q1 delta_q2 ...] """ # Initialize parameters and configuration files once self.data_type = data_type self.dataset_name = dataset_name self.detector_address = detector_address if (self.data_type == 'idx') or (self.data_type == 'idx_ffb') or ( self.data_type == 'smd') or (self.data_type == 'smd_ffb') or (self.data_type == 'xtc'): self.ds = DataSource(self.dataset_name) self.src = Detector(self.detector_address, self.ds.env()) if mask_path is None: # Create a binary mask of ones, default mask only works for xtc/ffb evt = next(self.ds.events()) self.mask_address = self.src.mask(evt, calib=True, status=True) self.msk = self.src.image(evt, self.mask_address) self.mask = np.copy(self.msk) else: self.msk = np.loadtxt(mask_path) self.mask = np.copy(self.msk) if geom_path is not None: geom = np.genfromtxt(geom_path, skiprows=1) self.gap = geom[0] self.shift = geom[1] self.orient = geom[2] self.comm = geom[3] self.param1 = geom[4] self.param2 = geom[5] self.param3 = geom[6] self.param4 = geom[7] if (self.data_type == 'h5'): if mask_path is None: # Create a binary mask of ones ## Add default binary mask here ## Default pnCCD dimensions dim1 = 1024 dim2 = 1024 self.mask = np.ones((dim1, dim2)) else: self.mask = np.loadtxt(mask_path) # Apply geometry self.msk = pnccd_tbx.get_geometry(img=self.mask, gap=self.gap, shift=self.shift, orient=self.orient) if self.detector_address == 'pnccdFront': evt = next(self.ds.events()) gain = self.src.gain(evt) self.gain = self.src.image(evt, gain) self.cart = 0 self.flat = 0 if backimg_path is None: self.backimg = None else: self.backimg = np.loadtxt(backimg_path).astype(np.float64) # Check if background image in cartesian coordinates exists if (self.backimg.shape == self.msk.shape): self.cart = 1 # Remove bg before transform to polar coordinates if backmsk_path is None: self.backmsk = None else: self.backmsk = np.loadtxt(backmsk_path).astype(np.float64) # Check if flat-field image exists if (self.backmsk is None) and (self.backimg is not None) and (self.cart == 0): self.pcflat = pnccd_tbx.dynamic_flatfield(self.backimg) self.flat = 1 if det_dist is None: # Get detector distance from events for run in self.ds.runs(): self.det_dist = cspad_tbx.env_distance(self.detector_address, run.env(), 577) else: self.det_dist = det_dist self.det_pix = det_pix if beam_l is None: # Get wavelength from event, note it can change slightly between events. So in the future use average. self.beam_l = cspad_tbx.evt_wavelength(next(self.ds.events())) else: self.beam_l = beam_l if mask_thr is None: # No dynamic masking self.thr = None else: self.thr = mask_thr if nQ is None: # Use image dimensions as a guide, leave room for offset beamC if self.msk.shape[0] > self.msk.shape[ 1]: # nQ determined by smallest dimension self.nQ = int(self.msk.shape[1] / 2) - 20 else: self.nQ = int(self.msk.shape[0] / 2) - 20 else: self.nQ = nQ if (self.nQ % 10): # Ascert even number, speeds things up massively for FFT self.nQ = np.floor(self.nQ / 10) * 10 if (self.nQ % dQ): # Ascert clean divisor self.nQ = np.floor(self.nQ / dQ) * dQ if nPhi is None: # Estimate based on 2*pi*nQ self.nPhi = np.ceil(2 * np.pi * self.nQ) else: self.nPhi = nPhi if (self.nPhi % 10): # Ascert even number, speeds things up massively for FFT self.nPhi = np.ceil(self.nPhi / 10) * 10 if (self.nPhi % dPhi): # Ascert clean divisor self.nPhi = np.ceil(self.nPhi / dPhi) * dPhi self.dQ = dQ self.dPhi = dPhi self.mask_angles = mask_angles self.mask_widths = mask_widths # Compute slices that should be masked in static mask if (self.mask_angles is not None) and (self.mask_widths is not None): self.mask_angles = (self.mask_angles / 360) * self.nPhi self.mask_widths = (self.mask_widths / 360) * self.nPhi # Check if nQ > smallest dimension/2 then we extend the image with zero values if self.nQ > min(self.msk.shape[0] / 2, self.msk.shape[1] / 2): self.msk = pnccd_tbx.extend_image(img=self.msk) self.mask = np.copy(self.msk) if (cent0 is None) or ( sum(cent0) == 0): # Use center of gravity to estimate starting beamC self.cent0 = [ int(round(self.msk.shape[1] / 2)), int(round(self.msk.shape[0] / 2)) ] else: self.cent0 = cent0 self.cent = self.cent0 # Default center if r_max is None: # Default, Use half of nQ self.r_max = int(self.nQ * (3 / 4)) else: self.r_max = r_max if (self.r_max % dr): # Ascert clean divisor self.r_max = np.floor(self.r_max / dr) * dr self.dr = dr self.dx = dx self.dy = dy if r_0 is None: self.radius = 0 self.score = 0 self.r_0 = r_0 if q_bound is None or sum(q_bound) == 0: self.q_bound = [None, None] else: self.q_bound = [None, self.q_bound] # Compute q-spacing self.q = np.arange(0, self.nQ, self.dQ) self.q = self.q * self.det_pix / self.det_dist * 4 * np.pi / self.beam_l / 2 # Compute Phi (Not accounting for curvature) self.phi = np.linspace(0, 2 * np.pi, self.nPhi / self.dPhi, endpoint=False) # Compute indices for Peak maxima if (peak is not None) and (dpeak is not None): self.peak = peak self.dpeak = dpeak self.ind1 = (self.q >= (self.peak[0] - self.dpeak[0])) & ( self.q <= (self.peak[0] + self.dpeak[0])) self.ind2 = (self.q >= (self.peak[1] - self.dpeak[1])) & ( self.q <= (self.peak[1] + self.dpeak[1])) else: self.peak = None self.dpeak = None
def run(self): """ Process all images assigned to this thread """ params, options = self.parser.parse_args(show_diff_phil=True) if params.input.experiment is None or \ params.input.run_num is None or \ params.input.address is None: raise Usage(self.usage) if params.format.file_format == "cbf": if params.format.cbf.detz_offset is None: raise Usage(self.usage) elif params.format.file_format == "pickle": if params.format.pickle.cfg is None: raise Usage(self.usage) else: raise Usage(self.usage) if not os.path.exists(params.output.output_dir): raise Sorry("Output path not found:" + params.output.output_dir) # Save the paramters self.params = params self.options = options from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank( ) # each process in MPI has a unique id, 0-indexed size = comm.Get_size() # size: number of processes running in this job # set up psana if params.format.file_format == "pickle": psana.setConfigFile(params.format.pickle.cfg) dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment, params.input.run_num) ds = psana.DataSource(dataset_name) if params.format.file_format == "cbf": src = psana.Source('DetInfo(%s)' % params.input.address) psana_det = psana.Detector(params.input.address, ds.env()) # set this to sys.maxint to analyze all events if params.dispatch.max_events is None: max_events = sys.maxint else: max_events = params.dispatch.max_events for run in ds.runs(): if params.format.file_format == "cbf": # load a header only cspad cbf from the slac metrology base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology( run, params.input.address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d" % run.run()) # list of all events times = run.times() nevents = min(len(times), max_events) # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes mytimes = [ times[i] for i in xrange(nevents) if (i + rank) % size == 0 ] for i in xrange(len(mytimes)): evt = run.event(mytimes[i]) id = evt.get(psana.EventId) print "Event #", i, " has id:", id timestamp = cspad_tbx.evt_timestamp( cspad_tbx.evt_time(evt)) # human readable format if timestamp is None: print "No timestamp, skipping shot" continue t = timestamp s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[ 17:19] + t[20:23] print "Processing shot", s if params.format.file_format == "pickle": if evt.get("skip_event"): print "Skipping event", id continue # the data needs to have already been processed and put into the event by psana data = evt.get(params.format.pickle.out_key) if data is None: print "No data" continue # set output paths according to the templates path = os.path.join(params.output.output_dir, "shot-" + s + ".pickle") print "Saving", path easy_pickle.dump(path, data) elif params.format.file_format == "cbf": # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data( psana_det, evt, use_default=False, dark=True, common_mode=None, apply_gain_mask=params.format.cbf.gain_mask_value is not None, gain_mask_value=params.format.cbf.gain_mask_value, per_pixel_gain=False) distance = cspad_tbx.env_distance( params.input.address, run.env(), params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" continue if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" continue else: wavelength = 12398.4187 / self.params.format.cbf.override_energy # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data( base_dxtbx, data, distance, wavelength, timestamp, params.input.address) path = os.path.join(params.output.output_dir, "shot-" + s + ".cbf") print "Saving", path # write the file import pycbf cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0) run.end() ds.end()
def _beam(self, index=None): '''Returns a simple model for the beam ''' if index is None: index = 0 evt = self.events_list[index] return self._beam_factory.simple(cspad_tbx.evt_wavelength(evt))
def average(argv=None): if argv == None: argv = sys.argv[1:] try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") command_line = (libtbx.option_parser.option_parser( usage=""" %s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] To write image pickles use -p, otherwise the program writes CSPAD CBFs. Writing CBFs requires the geometry to be already deployed. Examples: cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 Use one process on the current node to process all the events from run 25 of experiment cxi49812, using a detz_offset of 571. mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 As above, using 16 cores on the current node. bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812 As above, using the psanaq and 100 cores, putting the log in average.out and the output images in the folder cxi49812. """ % libtbx.env.dispatcher_name) .option(None, "--as_pickle", "-p", action="store_true", default=False, dest="as_pickle", help="Write results as image pickle files instead of cbf files") .option(None, "--config", "-c", type="string", default=None, dest="config", metavar="PATH", help="psana config file") .option(None, "--experiment", "-x", type="string", default=None, dest="experiment", help="experiment name (eg cxi84914)") .option(None, "--run", "-r", type="int", default=None, dest="run", help="run number") .option(None, "--address", "-a", type="string", default="CxiDs2.0:Cspad.0", dest="address", help="detector address name (eg CxiDs2.0:Cspad.0)") .option(None, "--detz_offset", "-d", type="float", default=None, dest="detz_offset", help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)") .option(None, "--outputdir", "-o", type="string", default=".", dest="outputdir", metavar="PATH", help="Optional path to output directory for output files") .option(None, "--averagebase", "-A", type="string", default="{experiment!l}_avg-r{run:04d}", dest="averagepath", metavar="PATH", help="Path to output average image without extension. String substitution allowed") .option(None, "--stddevbase", "-S", type="string", default="{experiment!l}_stddev-r{run:04d}", dest="stddevpath", metavar="PATH", help="Path to output standard deviation image without extension. String substitution allowed") .option(None, "--maxbase", "-M", type="string", default="{experiment!l}_max-r{run:04d}", dest="maxpath", metavar="PATH", help="Path to output maximum projection image without extension. String substitution allowed") .option(None, "--numevents", "-n", type="int", default=None, dest="numevents", help="Maximum number of events to process. Default: all") .option(None, "--skipevents", "-s", type="int", default=0, dest="skipevents", help="Number of events in the beginning of the run to skip. Default: 0") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") .option(None, "--pickle-optical-metrology", "-m", action="store_true", default=False, dest="pickle_optical_metrology", help="If writing pickle files, use the optical metrology in the experiment's calib directory") .option(None, "--bin_size", "-b", type="int", default=None, dest="bin_size", help="Rayonix detector bin size") .option(None, "--override_beam_x", "-X", type="float", default=None, dest="override_beam_x", help="Rayonix detector beam center x coordinate") .option(None, "--override_beam_y", "-Y", type="float", default=None, dest="override_beam_y", help="Rayonix detector beam center y coordinate") .option(None, "--calib_dir", "-C", type="string", default=None, dest="calib_dir", metavar="PATH", help="calibration directory") .option(None, "--xtc_dir", "-D", type="string", default=None, dest="xtc_dir", metavar="PATH", help="xtc stream directory") .option(None, "--use_ffb", "-f", action="store_true", default=False, dest="use_ffb", help="Use the fast feedback filesystem at LCLS. Only for the active experiment!") ).process(args=argv) if len(command_line.args) > 0 or \ command_line.options.as_pickle is None or \ command_line.options.experiment is None or \ command_line.options.run is None or \ command_line.options.address is None or \ command_line.options.detz_offset is None or \ command_line.options.averagepath is None or \ command_line.options.stddevpath is None or \ command_line.options.maxpath is None or \ command_line.options.pickle_optical_metrology is None: command_line.parser.show_help() return # set this to sys.maxint to analyze all events if command_line.options.numevents is None: maxevents = sys.maxint else: maxevents = command_line.options.numevents comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if command_line.options.config is not None: psana.setConfigFile(command_line.options.config) dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run) if command_line.options.xtc_dir is not None: if command_line.options.use_ffb: raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system") dataset_name += ":dir=%s"%command_line.options.xtc_dir elif command_line.options.use_ffb: # as ffb is only at SLAC, ok to hardcode /reg/d here dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment) ds = psana.DataSource(dataset_name) address = command_line.options.address src = psana.Source('DetInfo(%s)'%address) if not command_line.options.as_pickle: psana_det = psana.Detector(address, ds.env()) nevent = np.array([0.]) for run in ds.runs(): runnumber = run.run() # list of all events if command_line.options.skipevents > 0: print "Skipping first %d events"%command_line.options.skipevents times = run.times()[command_line.options.skipevents:] nevents = min(len(times),maxevents) # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0] for i in xrange(len(mytimes)): if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes) evt = run.event(mytimes[i]) #print "Event #",rank*mylength+i," has id:",evt.get(EventId) if 'Rayonix' in command_line.options.address: data = evt.get(Camera.FrameV1,src) if data is None: print "No data" continue data=data.data16().astype(np.float64) elif command_line.options.as_pickle: data = evt.get(psana.ndarray_float64_3, src, 'image0') else: # get numpy array, 32x185x388 data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations if data is None: print "No data" continue d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset) if d is None: print "No distance, skipping shot" continue if 'distance' in locals(): distance += d else: distance = np.array([float(d)]) w = cspad_tbx.evt_wavelength(evt) if w is None: print "No wavelength, skipping shot" continue if 'wavelength' in locals(): wavelength += w else: wavelength = np.array([w]) t = cspad_tbx.evt_time(evt) if t is None: print "No timestamp, skipping shot" continue if 'timestamp' in locals(): timestamp += t[0] + (t[1]/1000) else: timestamp = np.array([t[0] + (t[1]/1000)]) if 'sum' in locals(): sum+=data else: sum=np.array(data, copy=True) if 'sumsq' in locals(): sumsq+=data*data else: sumsq=data*data if 'maximum' in locals(): maximum=np.maximum(maximum,data) else: maximum=np.array(data, copy=True) nevent += 1 #sum the images across mpi cores if size > 1: print "Synchronizing rank", rank totevent = np.zeros(nevent.shape) comm.Reduce(nevent,totevent) if rank == 0 and totevent[0] == 0: raise Sorry("No events found in the run") sumall = np.zeros(sum.shape).astype(sum.dtype) comm.Reduce(sum,sumall) sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype) comm.Reduce(sumsq,sumsqall) maxall = np.zeros(maximum.shape).astype(maximum.dtype) comm.Reduce(maximum,maxall, op=MPI.MAX) waveall = np.zeros(wavelength.shape).astype(wavelength.dtype) comm.Reduce(wavelength,waveall) distall = np.zeros(distance.shape).astype(distance.dtype) comm.Reduce(distance,distall) timeall = np.zeros(timestamp.shape).astype(timestamp.dtype) comm.Reduce(timestamp,timeall) if rank==0: if size > 1: print "Synchronized" # Accumulating floating-point numbers introduces errors, # which may cause negative variances. Since a two-pass # approach is unacceptable, the standard deviation is # clamped at zero. mean = sumall / float(totevent[0]) variance = (sumsqall / float(totevent[0])) - (mean**2) variance[variance < 0] = 0 stddev = np.sqrt(variance) wavelength = waveall[0] / totevent[0] distance = distall[0] / totevent[0] pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value timestamp = timeall[0] / totevent[0] timestamp = (int(timestamp), timestamp % int(timestamp) * 1000) timestamp = cspad_tbx.evt_timestamp(timestamp) if command_line.options.as_pickle: extension = ".pickle" else: extension = ".cbf" dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env())] dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths] if 'Rayonix' in command_line.options.address: from xfel.cxi.cspad_ana import rayonix_tbx pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size) beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y] detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size) active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]]) split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3] for data, path in zip([mean, stddev, maxall], dest_paths): print "Saving", path d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=flex.double(data), distance=distance, pixel_size=pixel_size, saturated_value=rayonix_tbx.rayonix_saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) elif command_line.options.as_pickle: split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3] xpp = 'xpp' in address.lower() if xpp: evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format from xfel.detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0]) assert version_lookup is not None active_areas = xpp_active_areas[version_lookup]['active_areas'] beam_center = [1765 // 2, 1765 // 2] else: if command_line.options.calib_dir is not None: metro_path = command_line.options.calib_dir elif command_line.options.pickle_optical_metrology: from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path metro_path = get_calib_file_path(run.env(), address, run) else: metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0") sections = parse_calib.calib2sections(metro_path) beam_center, active_areas = cspad_tbx.cbcaa( cspad_tbx.getConfig(address, ds.env()), sections) class fake_quad(object): def __init__(self, q, d): self.q = q self.d = d def quad(self): return self.q def data(self): return self.d if xpp: quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)] mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) mean = flex.double(mean.astype(np.float64)) quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)] stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) stddev = flex.double(stddev.astype(np.float64)) quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)] maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) maxall = flex.double(maxall.astype(np.float64)) else: quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)] mean = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)] stddev = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)] maxall = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) maxall = flex.double(maxall.astype(np.float64)) for data, path in zip([mean, stddev, maxall], dest_paths): print "Saving", path d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=data, distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) else: # load a header only cspad cbf from the slac metrology from xfel.cftbx.detector import cspad_cbf_tbx import pycbf base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d"%run.run()) for data, path in zip([mean, stddev, maxall], dest_paths): print "Saving", path cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address) cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def event(self, evt, env): if (evt.get("skip_event")): return self.nshots += 1 s = None t = evt.getTime() if (t is not None): s = t.seconds() + (t.nanoseconds() / 1000000000) else: self.nfail += 1 self.logger.warning("event(): no timestamp, shot skipped") evt.put(skip_event_flag(), "skip_event") return if (not isinstance(s, float)): raise RuntimeError("Wrong type for 's': %s" % type(s).__name__) # XXX This hardcodes the address for the front detector! det_z = cspad_tbx.env_detz('CxiDs1-0|Cspad-0', env) if (det_z is None): self.nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), "skip_event") return laser01 = cspad_tbx.env_laser_status(env, 1) if laser01 is None: self.nfail += 1 self.logger.warning("event(): no status for laser 1, shot skipped") evt.put(skip_event_flag(), 'skip_event') return laser04 = cspad_tbx.env_laser_status(env, 4) if laser04 is None: self.nfail += 1 self.logger.warning("event(): no status for laser 4, shot skipped") evt.put(skip_event_flag(), 'skip_event') return # Laser power for fourth laser. The control name was provided by # Jan Kern. XXX Move to its own function in cspad_tbx? laser04_power = None if env is not None: pv = env.epicsStore().value('CXI:LAS:MMN:02:ROT.RBV') if pv is not None and len(pv.values) == 1: laser04_power = pv.values[0] if laser04_power is None: self.nfail += 1 self.logger.warning("event(): no power for laser 4, shot skipped") evt.put(skip_event_flag(), 'skip_event') return si_foil = cspad_tbx.env_sifoil(env) if (si_foil is None): self.nfail += 1 self.logger.warning("event(): no Si-foil thickness, shot skipped") evt.put(skip_event_flag(), "skip_event") return if (not (isinstance(si_foil, float) or isinstance(si_foil, int))): raise RuntimeError("Wrong type for 'si_foil': %s" % type(si_foil).__name__) wavelength = cspad_tbx.evt_wavelength(evt) if (wavelength is None): self.nfail += 1 self.logger.warning("event(): no wavelength, shot skipped") evt.put(skip_event_flag(), "skip_event") return # In order to keep all arrays the same length, only append once # all values have been successfully obtained. XXX Still bugs: see # June run 119. self._t.append(s) self._si_foil.append(si_foil) self._wavelength.append(wavelength) self._det_z.append(det_z) self._laser01.append(laser01) self._laser04.append(laser04) self._laser04_power.append(laser04_power) if (self.nshots % 120 == 0): self.update_plot()
def run(self): """ Process all images assigned to this thread """ params, options = self.parser.parse_args(show_diff_phil=True) if params.input.experiment is None or params.input.run_num is None or params.input.address is None: raise Usage(self.usage) if params.format.file_format == "cbf": if params.format.cbf.detz_offset is None: raise Usage(self.usage) elif params.format.file_format == "pickle": if params.format.pickle.cfg is None: raise Usage(self.usage) else: raise Usage(self.usage) if not os.path.exists(params.output.output_dir): raise Sorry("Output path not found:" + params.output.output_dir) # Save the paramters self.params = params self.options = options from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed size = comm.Get_size() # size: number of processes running in this job # set up psana if params.format.file_format == "pickle": psana.setConfigFile(params.format.pickle.cfg) dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment, params.input.run_num) ds = psana.DataSource(dataset_name) if params.format.file_format == "cbf": src = psana.Source("DetInfo(%s)" % params.input.address) psana_det = psana.Detector(params.input.address, ds.env()) # set this to sys.maxint to analyze all events if params.dispatch.max_events is None: max_events = sys.maxint else: max_events = params.dispatch.max_events for run in ds.runs(): if params.format.file_format == "cbf": # load a header only cspad cbf from the slac metrology base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, params.input.address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d" % run.run()) if params.format.cbf.gain_mask_value is not None: gain_mask = psana_det.gain_mask(gain=params.format.cbf.gain_mask_value) # list of all events times = run.times() nevents = min(len(times), max_events) # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes mytimes = [times[i] for i in xrange(nevents) if (i + rank) % size == 0] for i in xrange(len(mytimes)): evt = run.event(mytimes[i]) id = evt.get(psana.EventId) print "Event #", i, " has id:", id timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format if timestamp is None: print "No timestamp, skipping shot" continue t = timestamp s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23] print "Processing shot", s if params.format.file_format == "pickle": if evt.get("skip_event"): print "Skipping event", id continue # the data needs to have already been processed and put into the event by psana data = evt.get(params.format.pickle.out_key) if data is None: print "No data" continue # set output paths according to the templates path = os.path.join(params.output.output_dir, "shot-" + s + ".pickle") print "Saving", path easy_pickle.dump(path, data) elif params.format.file_format == "cbf": # get numpy array, 32x185x388 data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations if params.format.cbf.gain_mask_value is not None: # apply gain mask data *= gain_mask distance = cspad_tbx.env_distance(params.input.address, run.env(), params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" continue if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" continue else: wavelength = 12398.4187 / self.params.format.cbf.override_energy # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data( base_dxtbx, data, distance, wavelength, timestamp, params.input.address ) path = os.path.join(params.output.output_dir, "shot-" + s + ".cbf") print "Saving", path # write the file import pycbf cspad_img._cbf_handle.write_widefile( path, pycbf.CBF, pycbf.MIME_HEADERS | pycbf.MSG_DIGEST | pycbf.PAD_4K, 0 ) run.end() ds.end()
def process_event(self, run, timestamp): """ Process a single event from a run @param run psana run object @param timestamp psana timestamp object """ ts = cspad_tbx.evt_timestamp((timestamp.seconds(),timestamp.nanoseconds()/1e6)) if ts is None: print "No timestamp, skipping shot" return if len(self.params_cache.debug.event_timestamp) > 0 and ts not in self.params_cache.debug.event_timestamp: return if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events: if ts in self.known_events: if self.known_events[ts] == "unknown": if self.params_cache.debug.skip_bad_events and self.known_events[ts] == "unknown": print "Skipping event %s: possibly caused an unknown exception previously"%ts return elif self.params_cache.debug.skip_processed_events: print "Skipping event %s: processed successfully previously"%ts return else: if self.params_cache.debug.skip_unprocessed_events: print "Skipping event %s: not processed previously"%ts return print "Accepted", ts self.debug_file_handle.write("%s,%s"%(socket.gethostname(), ts)) self.params = copy.deepcopy(self.params_cache) evt = run.event(timestamp) id = evt.get(psana.EventId) if evt.get("skip_event"): print "Skipping event",id self.debug_file_handle.write(",psana_skip\n") return # the data needs to have already been processed and put into the event by psana if self.params.format.file_format == 'cbf': # get numpy array, 32x185x388 data = self.psana_det.calib(evt) # applies psana's complex run-dependent calibrations if data is None: print "No data" self.debug_file_handle.write(",no_data\n") return if self.params.format.cbf.gain_mask_value is not None: # apply gain mask data *= self.gain_mask distance = cspad_tbx.env_distance(self.params.input.address, run.env(), self.params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" self.debug_file_handle.write(",no_distance\n") return if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" self.debug_file_handle.write(",no_wavelength\n") return else: wavelength = 12398.4187/self.params.format.cbf.override_energy if self.params.format.file_format == 'pickle': image_dict = evt.get(self.params.format.pickle.out_key) data = image_dict['DATA'] timestamp = t = ts s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23] print "Processing shot", s if self.params.format.file_format == 'cbf': # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data(self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address) elif self.params.format.file_format == 'pickle': from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory cspad_img = FormatPYunspecifiedStillInMemory(image_dict) cspad_img.timestamp = s if self.params.dispatch.dump_all: self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s)) self.cache_ranges(cspad_img, self.params) imgset = MemImageSet([cspad_img]) datablock = DataBlockFactory.from_imageset(imgset)[0] # before calling DIALS for processing, set output paths according to the templates if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template: self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.indexed_filename_template%("idx-" + s)) if "%s" in self.refined_experiments_filename_template: self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.refined_experiments_filename_template%("idx-" + s)) if "%s" in self.integrated_filename_template: self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.integrated_filename_template%("idx-" + s)) # if border is requested, generate a border only mask if self.params.border_mask.border > 0: from dials.command_line.generate_mask import MaskGenerator generator = MaskGenerator(self.params.border_mask) mask = generator.generate(imgset) self.params.spotfinder.lookup.mask = mask try: observed = self.find_spots(datablock) except Exception, e: import traceback; traceback.print_exc() print str(e), "event", timestamp self.debug_file_handle.write(",spotfinding_exception\n") return
def event(self, evt, env): """The event() function is called for every L1Accept transition. The event() function does not log shots skipped due to incompleteness in order to keep the output streams clean. Instead, the number of skipped shots is reported by endjob(). @param evt Event data object, a configure object @param env Environment object """ if (evt.get("skip_event")): return # XXX This hardcodes the address for the front detector! detz = cspad_tbx.env_detz('CxiDs1-0|Cspad-0', env) if (detz is None): self.m_no_detz += 1 sifoil = cspad_tbx.env_sifoil(env) if (sifoil is None): self.m_no_sifoil += 1 timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) if (timestamp is None): self.m_no_timestamp += 1 wavelength = cspad_tbx.evt_wavelength(evt) if (wavelength is None): self.m_no_wavelength += 1 if (detz is None or sifoil is None or timestamp is None or wavelength is None): self.m_nfail += 1 return if (detz != self.m_detz): if (self.m_detz is None): self.m_logger.info("%s: initial detz % 8.4f" % (timestamp, detz)) else: self.m_logger.info("%s: detz % 8.4f -> % 8.4f" % (timestamp, self.m_detz, detz)) self.m_detz = detz if (self.m_detz_max is None or detz > self.m_detz_max): self.m_detz_max = detz if (self.m_detz_min is None or detz < self.m_detz_min): self.m_detz_min = detz if (sifoil != self.m_sifoil): if (self.m_sifoil is None): self.m_logger.info("%s: initial Si-foil % 8d" % (timestamp, sifoil)) else: self.m_logger.info("%s: Si-foil % 8d -> % 8d" % (timestamp, self.m_sifoil, sifoil)) self.m_sifoil = sifoil if (self.m_sifoil_max is None or sifoil > self.m_sifoil_max): self.m_sifoil_max = sifoil if (self.m_sifoil_min is None or sifoil < self.m_sifoil_min): self.m_sifoil_min = sifoil # Accumulate the sum and the squared sum of the shifted the # wavelength. The shift is taken as the first wavelength # encountered. This may be more accurate than accumulating raw # values [Chan et al. (1983) Am. Stat. 37, 242-247]. if (self.m_nmemb == 0): self.m_wavelength_shift = wavelength self.m_wavelength_sum = (wavelength - self.m_wavelength_shift) self.m_wavelength_sumsq = (wavelength - self.m_wavelength_shift)**2 self.m_nmemb = 1 else: self.m_wavelength_sum += (wavelength - self.m_wavelength_shift) self.m_wavelength_sumsq += (wavelength - self.m_wavelength_shift)**2 self.m_nmemb += 1
def __init__(self, dataset_name = None, detector_address = None, data_type = 'idx', mask_path = None, mask_angles = None, mask_widths = None, backimg_path = None, backmsk_path = None, geom_path = None, det_dist = None, det_pix = 0.075, beam_l = None, mask_thr = None, nQ = None, nPhi = None, dQ = 1, dPhi = 1, cent0 = None, r_max = None, dr = None, dx = None, dy = None, r_0 = None, q_bound = None, peak = None, dpeak = None): """The fluctuation scattering class stores processing parameters, initiates mask and background data and retrieves 2D images from events. Processing options of the 2D images include: * Transform from cartesian to polar coordinates * Beam center refinement * Dynamic masking * Normalization % SAXS calculation * Particle sizing * Computation of in-frame 2-point angular auto-correlations using FFTs @param dataset_name Experiment name and run number @param detector_address Adress to back or front detector @param data_type Type of data file format (h5 or xtc in the formats: idx|idx_ffb|smd|smd_ffb|h5) @param mask_path Full path to static image mask @param mask_angles Center of angluar slices (deg) that should be masked out (due to jet streaks etc), [Ang1 Ang2 ...] @param mask_widths Width of angular slices (deg) that should be masked out (due to jet streaks etc), [delta1 delta2 ...] @param backimg_path Full path to background image @param backmsk_path Full path to background mask @param geom_path Full path to geometry file (for h5 format) @param det_dist Override of detecor distance (in mm) @param det_pix Pixel size (in mm) @param beam_l Override of beam wavelength (in Angstrom) @param mask_thr Threshold for dynamic masking @param nQ Number of Q-bins to consider (in pixels) @param nPhi Number of Phi-bins to consider (in pixels) @param dQ Stepsize in Q (in pixels) @param dPhi Stepsize in Phi (in pixels) @param cent0 Initial beam center coordinates [xc,yc] @param r_max Maximum radial value to use for beamcenter refinement (in pixels) @param dr Stepsize in r (in pixels) @param dx Gridsize for beam center refinement in x, i.e xc+/-dx (in pixels) @param dy Gridsize for beam center refinement in y, i.e yc+/-dy (in pixles) @param r_0 Starting value for particle radius refinement [in Ang] @param q_bound Upper and Lower boundaries of q for Particle radius refinement [in Ang^-1] @param peak Q-values for peak maxima [q_peak1 q_peak2 ...] @param dpeak Delta Q used for peak integration of peak maxima [delta_q1 delta_q2 ...] """ # Initialize parameters and configuration files once self.data_type = data_type self.dataset_name = dataset_name self.detector_address = detector_address if (self.data_type == 'idx') or (self.data_type == 'idx_ffb') or (self.data_type == 'smd') or (self.data_type == 'smd_ffb') or (self.data_type == 'xtc') : self.ds = DataSource(self.dataset_name) self.src = Detector(self.detector_address, self.ds.env()) if mask_path is None : # Create a binary mask of ones, default mask only works for xtc/ffb evt = self.ds.events().next() self.mask_address = self.src.mask(evt,calib=True,status=True) self.msk = self.src.image(evt,self.mask_address) self.mask = np.copy(self.msk) else: self.msk = np.loadtxt(mask_path) self.mask = np.copy(self.msk) if geom_path is not None : geom = np.genfromtxt(geom_path,skiprows=1) self.gap = geom[0] self.shift = geom[1] self.orient = geom[2 ] self.comm = geom[3] self.param1 = geom[4] self.param2 = geom[5] self.param3 = geom[6] self.param4 = geom[7] if (self.data_type == 'h5'): if mask_path is None : # Create a binary mask of ones ## Add default binary mask here ## Default pnCCD dimensions dim1 = 1024 dim2 = 1024 self.mask = np.ones((dim1,dim2)) else: self.mask = np.loadtxt(mask_path) # Apply geometry self.msk = pnccd_tbx.get_geometry(img = self.mask, gap = self.gap, shift = self.shift, orient = self.orient) if self.detector_address == 'pnccdFront' : evt = self.ds.events().next() gain = self.src.gain(evt) self.gain = self.src.image(evt,gain) self.cart = 0 self.flat = 0 if backimg_path is None : self.backimg = None else : self.backimg = np.loadtxt(backimg_path).astype(np.float64) # Check if background image in cartesian coordinates exists if (self.backimg.shape == self.msk.shape): self.cart = 1 # Remove bg before transform to polar coordinates if backmsk_path is None : self.backmsk = None else : self.backmsk = np.loadtxt(backmsk_path).astype(np.float64) # Check if flat-field image exists if (self.backmsk is None) and (self.backimg is not None) and (self.cart==0): self.pcflat = pnccd_tbx.dynamic_flatfield(self.backimg) self.flat = 1 if det_dist is None : # Get detector distance from events for run in self.ds.runs(): self.det_dist = cspad_tbx.env_distance(self.detector_address, run.env(), 577) else : self.det_dist = det_dist self.det_pix = det_pix if beam_l is None : # Get wavelength from event, note it can change slightly between events. So in the future use average. self.beam_l = cspad_tbx.evt_wavelength(self.ds.events().next()) else : self.beam_l = beam_l if mask_thr is None : # No dynamic masking self.thr = None else : self.thr = mask_thr if nQ is None : # Use image dimensions as a guide, leave room for offset beamC if self.msk.shape[0] > self.msk.shape[1] : # nQ determined by smallest dimension self.nQ = int(self.msk.shape[1]/2)-20 else : self.nQ = int(self.msk.shape[0]/2)-20 else : self.nQ = nQ if (self.nQ % 10): # Ascert even number, speeds things up massively for FFT self.nQ = np.floor(self.nQ/10)*10 if (self.nQ % dQ): # Ascert clean divisor self.nQ = np.floor(self.nQ/dQ)*dQ if nPhi is None : # Estimate based on 2*pi*nQ self.nPhi = np.ceil(2*np.pi*self.nQ) else : self.nPhi = nPhi if (self.nPhi % 10): # Ascert even number, speeds things up massively for FFT self.nPhi = np.ceil(self.nPhi/10)*10 if (self.nPhi % dPhi): # Ascert clean divisor self.nPhi = np.ceil(self.nPhi/dPhi)*dPhi self.dQ = dQ self.dPhi = dPhi self.mask_angles = mask_angles self.mask_widths = mask_widths # Compute slices that should be masked in static mask if (self.mask_angles is not None) and (self.mask_widths is not None) : self.mask_angles = (self.mask_angles/360) * self.nPhi self.mask_widths = (self.mask_widths/360) * self.nPhi # Check if nQ > smallest dimension/2 then we extend the image with zero values if self.nQ > min(self.msk.shape[0]/2,self.msk.shape[1]/2) : self.msk = pnccd_tbx.extend_image(img = self.msk) self.mask = np.copy(self.msk) if (cent0 is None) or (sum(cent0) == 0): # Use center of gravity to estimate starting beamC self.cent0 = [int(round(self.msk.shape[1]/2)) , int(round(self.msk.shape[0]/2))] else : self.cent0 = cent0 self.cent = self.cent0 # Default center if r_max is None : # Default, Use half of nQ self.r_max = int(self.nQ*(3/4)) else : self.r_max = r_max if (self.r_max % dr): # Ascert clean divisor self.r_max = np.floor(self.r_max/dr)*dr self.dr = dr self.dx = dx self.dy = dy if r_0 is None : self.radius = 0 self.score = 0 self.r_0 = r_0 if q_bound is None or sum(q_bound)==0 : self.q_bound = [None,None] else : self.q_bound = [None,self.q_bound] # Compute q-spacing self.q = np.arange(0, self.nQ, self.dQ) self.q = self.q*self.det_pix/self.det_dist*4*np.pi/self.beam_l/2 # Compute Phi (Not accounting for curvature) self.phi = np.linspace(0, 2*np.pi, self.nPhi/self.dPhi,endpoint=False) # Compute indices for Peak maxima if (peak is not None) and (dpeak is not None) : self.peak = peak self.dpeak = dpeak self.ind1 = (self.q >= (self.peak[0] - self.dpeak[0])) & (self.q <= (self.peak[0] + self.dpeak[0]) ) self.ind2 = (self.q >= (self.peak[1] - self.dpeak[1])) & (self.q <= (self.peak[1] + self.dpeak[1]) ) else: self.peak = None self.dpeak = None
def event(self, evt, env): """The event() function is called for every L1Accept transition. For now, log error and set bogus value to allow stuff to continue -- must check for the bogosity later XXX The dead time of the detector complicates checking how often things are updated! Move this to the ring buffer? @param evt Event data object, a configure object @param env Environment object """ from pyana.event import Event from acqiris_ext import acqiris_integrate, apd_hitfind super(mod_ledge, self).event(evt, env) if evt.status() != Event.Normal: pass # XXX return -- Never skip because arrays will end up # different length, so ignore this? # Get the time of the event, in fractional seconds since the # epoch. This is needed for all subsequent history-keeping, and # is hence determined first. XXX Is history-keeping even # justified? time = cspad_tbx.evt_time(evt) if time is None: time = float("nan") else: time = time[0] + time[1] / 1e3 self._timestamp.append(time) # The repetition rate is currently just used for sanity checking. repetition_rate = cspad_tbx.evt_repetition_rate(evt) if repetition_rate is None: repetition_rate = float("nan") self._repetition_rate.append(repetition_rate) # Get the I0. No need to warn about it here, it will be done once # the image is written out. I0 = cspad_tbx.evt_pulse_energy(evt) if I0 is None: I0 = float("nan") self._I0.append(I0) # Get the FEE energy. Average the two readings before and after # attenuation separately. XXX What are the units? It look like # it could be mJ? fee_before = 0.5 * sum(evt.getFeeGasDet()[0:2]) if fee_before is None: fee_before = float("nan") self._fee_before.append(fee_before) fee_after = 0.5 * sum(evt.getFeeGasDet()[2:4]) if fee_after is None: fee_after = float("nan") self._fee_after.append(fee_after) # XXX Just a check: this is what xtcexplorer does: fee_energy = evt.get(xtc.TypeId.Type.Id_FEEGasDetEnergy) if fee_energy is not None: assert ( evt.getFeeGasDet()[0] == fee_energy.f_11_ENRC and evt.getFeeGasDet()[1] == fee_energy.f_12_ENRC and evt.getFeeGasDet()[2] == fee_energy.f_21_ENRC and evt.getFeeGasDet()[3] == fee_energy.f_22_ENRC ) """ # For Bill: expect 84240 data points for r0054 # # grep "^BILL_POINT" | cut -d' ' -f2,3,4,5,6 > t.dat # gnuplot> m=0.1 ; k=-0.01e-8; f(x) = k * x + m # gnuplot> fit f(x) "t.dat" using ($3):($5) via k,m if not hasattr(self, '_gmd_seqno'): self._gmd_seqno = 0 gmd = evt.get(key=xtc.TypeId.Type.Id_GMD) if gmd is None: return acq_apd = evt.getAcqValue('SxrEndstation-0|Acqiris-1', 0, env) if acq_apd is not None and acq_apd.waveform() is not None: w = acq_apd.waveform() baseline = numpy.mean(w[0:(w.shape[0] / 5)]) peak = numpy.min(w[(w.shape[0] / 5):w.shape[0]]) self._gmd_seqno += 1 print "BILL_POINT %d %s %s %s %s" % (self._gmd_seqno, repr(gmd.fBgValuePerSample), repr(gmd.fCorrectedSumPerPulse), repr(gmd.fRelativeEnergyPerPulse), repr(peak - baseline)) return """ """ # XXX Record injector motion--note that they cannot be added--see # Ray's email. injector_micos_xyz = cspad_tbx.env_pv3_get( env, ['SXR:EXP:MZM:%02d:ENCPOSITIONGET' % i for i in [1, 2, 3]]) if injector_micos_xyz is None: self.logger.error("No micos injector motor positions") injector_micos_xyz = (float('nan'), float('nan'), float('nan')) self._injector_micos_xyz.append(injector_micos_xyz) injector_rough_xyz = cspad_tbx.env_pv3_get( env, ['SXR:EXP:MMS:%02d.RBV' % i for i in [1, 2, 3]]) if injector_rough_xyz is None: self.logger.error("No rough injector motor positions") injector_rough_xyz = (float('nan'), float('nan'), float('nan')) self._injector_rough_xyz.append(injector_rough_xyz) # Injector power supplies XXX There is a third PSU, no? # # The -5kV supply # SXR:EXP:SHV:VHS6:CH0:VoltageMeasure # SXR:EXP:SHV:VHS6:CH0:CurrentMeasure # # The plus 5kV supply # SXR:EXP:SHV:VHS2:CH0:VoltageMeasure # SXR:EXP:SHV:VHS2:CH0:CurrentMeasure injector_plus_current = cspad_tbx.env_pv1_get( env, 'SXR:EXP:SHV:VHS6:CH0:CurrentMeasure') if injector_plus_current is None: self.logger.error("No plus-motor current") injector_plus_current = -1 self._injector_plus_current.append(injector_plus_current) injector_plus_voltage = cspad_tbx.env_pv1_get( env, 'SXR:EXP:SHV:VHS6:CH0:VoltageMeasure') if injector_plus_voltage is None: self.logger.error("No plus-motor voltage") injector_plus_voltage = -1 self._injector_plus_voltage.append(injector_plus_voltage) injector_minus_current = cspad_tbx.env_pv1_get( env, 'SXR:EXP:SHV:VHS2:CH0:CurrentMeasure') if injector_minus_current is None: self.logger.error("No minus-motor current") injector_minus_current = -1 self._injector_minus_current.append(injector_minus_current) injector_minus_voltage = cspad_tbx.env_pv1_get( env, 'SXR:EXP:SHV:VHS2:CH0:VoltageMeasure') if injector_minus_voltage is None: self.logger.error("No minus-motor voltage") injector_minus_voltage = -1 self._injector_minus_voltage.append(injector_minus_voltage) """ """ # The spectrometer motor positions are just used for sanity # checking. spectrometer_xyz = cspad_tbx.env_spectrometer_xyz_sxr(env) if spectrometer_xyz is None: self.logger.error("No spectrometer motor positions") spectrometer_xyz = (float('nan'), float('nan'), float('nan')) self._spectrometer_xyz.append(spectrometer_xyz) """ # Get the pulse energy after monochromator, and fall back on the # pre-monochromator energy if the former is absent. Record in # list for mean and stddev. XXX Verify that the wavelength after # the monochromator is updated at around 1 Hz. # # For the publication an offset and scale were calibrated. wavelength = cspad_tbx.env_wavelength_sxr(evt, env) if wavelength is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: energy = float("nan") else: energy = 12398.4187 / wavelength self._energy.append(energy) self._history_energy.push(time, energy) # XXX Not necessary?! """ # Laser shutters XXX need to sort out laser numbering XXX Laser # power stuff? XXX Position of polarizer/analyser shutters = cspad_tbx.env_laser_shutters(env) #print "Got shutters", shutters """ # Read out the diode traces from the via the Acqiris. XXX In any # case, the APD and the more sensitive Opto Diode in the monitor # tank (i.e. the transmission diode) should be anti-correlated, so # check it! The entire trace always covers 10 us. XXX Could this # be figured out from xtc.TypeId.Type.Id_AcqConfig? # # XXX This appears to be suboptimal: look at the # skewness-transform for the APD to sort this out. acq_apd = evt.getAcqValue("SxrEndstation-0|Acqiris-1", 0, env) acq_apd_integral = float("nan") if acq_apd is not None: waveform = acq_apd.waveform() if waveform is not None: # With a 40k-point trace, one should integrate from 18200 to # 18400. waveform = waveform.flatten() nmemb = len(waveform) // 200 if nmemb > 0: acq_apd_integral = acqiris_integrate(flex.double(waveform), 91 * nmemb, 100 * nmemb, nmemb) self._acq_apd_integral.append(acq_apd_integral) if evt.expNum() == 208: # Opto diode address for L632. acq_opto_diode = evt.getAcqValue("SxrEndstation-0|Acqiris-1", 1, env) elif evt.expNum() == 363: # Opto diode address for LB68. acq_opto_diode = evt.getAcqValue("SxrEndstation-0|Acqiris-2", 2, env) acq_opto_diode_integral = float("nan") if acq_opto_diode is not None: waveform = acq_opto_diode.waveform() if waveform is not None: # With a 40k-point trace, one should integrate from 16000 to # 24000. With a 20k-point trace, a suitable integration # region is bounded by 8000 and 12000. There is no need for # thresholding, because the integral of the Opto Diode will # not be used for hit finding. XXX What are the "misses" we # record on the Opto Diode? XXX The direct beam is completely # gone after it hits the sample, because soft X-rays. waveform = waveform.flatten() nmemb = len(waveform) // 5 if nmemb > 0: acq_opto_diode_integral = acqiris_integrate(flex.double(waveform), 2 * nmemb, 4 * nmemb, nmemb) self._acq_opto_diode_integral.append(acq_opto_diode_integral) # Sanity check: verify that the timestamps for the two Acqiris # traces are similar enough. if acq_apd is not None and acq_opto_diode is not None: assert ( len(acq_apd.timestamps()) == len(acq_opto_diode.timestamps()) and numpy.any(numpy.abs(acq_apd.timestamps() - acq_opto_diode.timestamps())) < 1e-6 ) # self.logger.info("DIODE INTEGRALS: %f %f %f" % (I0, acq_apd_integral, acq_opto_diode_integral)) """ import matplotlib.pyplot as plt hit_array_apd = apd_hitfind( flex.double(acq_apd.waveform()), len(acq_apd.waveform()) // 5) hit_array_opto_diode = apd_hitfind( flex.double(acq_opto_diode.waveform()), len(acq_opto_diode.waveform()) // 5) fig = plt.figure() ax = fig.add_subplot(111) #ax.plot( # range(len(acq_apd.timestamps())), acq_apd.waveform()) ax.plot( range(len(acq_opto_diode.timestamps())), acq_opto_diode.waveform()[0, :]) plt.show() fig = plt.figure() ax = fig.add_subplot(111) #ax.plot( # acq_apd.timestamps()[0:len(hit_array_apd)], hit_array) ax.plot( acq_opto_diode.timestamps()[0:len(hit_array_opto_diode)], hit_array) plt.show() """ # Determine whether the beam hit the sample, and register the # outcome. If not using any diodes for hit-finding, every shot is # assumed to be a hit. XXX Unfortunately, this crucial piece is # very unreliable. The threshold for the APD needs to be # verified--inspect all the histograms. XXX hitfind_flags is # probable better as a module parameter. # hitfind_flags = 0x3 hitfind_flags = 0 hit = False if not hitfind_flags: hit = True elif hitfind_flags & 0x1 and acq_apd_integral > 0.2: hit = True self._hit.append(hit) # Always proceed all the way through (even if some shots have # invalid values of e.g. I0) because images are precious. XXX # Must reset counters before returning! XXX What about skipping # all of the above if display is True? if self.cspad_img is not None: self._nframes += 1 """ # The spectrometer should not move! t = (self._spectrometer_xyz - self._spectrometer_xyz.mean()).rms_length() print "Spectrometer displacement", t # Fine/rough motor position deviations from the mean. See Ray's # email. t = (self._injector_micos_xyz - self._injector_micos_xyz.mean()).rms_length() print "Injector micos displacement", t t = (self._injector_rough_xyz - self._injector_rough_xyz.mean()).rms_length() print "Injector rough displacement", t # Injector motor position means and deviations if self._injector_plus_current.size() > 1: t = flex.mean_and_variance(self._injector_plus_current) print "Injector plus current mean %10e stddev %10e" % \ (t.mean(), t.unweighted_sample_standard_deviation()) if self._injector_plus_voltage.size() > 1: t = flex.mean_and_variance(self._injector_plus_voltage) print "Injector plus voltage mean %10e stddev %10e" % \ (t.mean(), t.unweighted_sample_standard_deviation()) if self._injector_minus_current.size() > 1: t = flex.mean_and_variance(self._injector_minus_current) print "Injector minus current mean %10e stddev %10e" % \ (t.mean(), t.unweighted_sample_standard_deviation()) if self._injector_minus_voltage.size() > 1: t = flex.mean_and_variance(self._injector_minus_voltage) print "Injector minus voltage mean %10e stddev %10e" % \ (t.mean(), t.unweighted_sample_standard_deviation()) """ # Energy statistics are collected from all shots, regardless of # whether they are hits or not. Since this statistic mentions # the frame number, it should be reported first. XXX The energy # should have a really small standard deviation. Check # self._energy.size() and self._history_energy.frequency() XXX # verify that it works for one data point. (energy_mean, energy_stddev, energy_nmemb, n) = self._filtered_stats( lambda x: not math.isnan(x) and x > 0, self._energy ) if n > 0: self.logger.warning("%d shots have undefined energy" % n) (I0_mean, I0_stddev, I0_nmemb, n) = self._filtered_stats(lambda x: not math.isnan(x), self._I0) if n > 0: self.logger.warning("%d shots have undefined I0" % n) self.logger.info( "Frame %d: E=%.3f+/-%.3f (N=%d) I0=%.0f+/-%.0f (N=%d)" % (self._nframes, energy_mean, energy_stddev, energy_nmemb, I0_mean, I0_stddev, I0_nmemb) ) # Sanity check: unless changed while integrating the frame, the # repetition rate should have a standard deviation of zero. dt = self._timestamp[-1] - self._timestamp[0] rr_mean = rr_observed = rr_stddev = 0 if dt > 0: rr_observed = (len(self._timestamp) - 1) / dt rr = filter(lambda x: not math.isnan(x) and x > 0, self._repetition_rate) if len(rr) > 1: rr_stats = flex.mean_and_variance(flex.double(rr)) rr_mean = rr_stats.mean() rr_stddev = rr_stats.unweighted_sample_standard_deviation() self.logger.info( "Repetition rate: %.3f Hz (observed), %.3f+/-%.3f Hz (expected)" % (rr_observed, rr_mean, rr_stddev) ) # Compare observed and configured exposure time. config = cspad_tbx.getConfig(self.address, env) exposure_time = 0 if config is not None and dt > 0 and len(self._timestamp) > 0: exposure_time = dt * (len(self._timestamp) + 1) / len(self._timestamp) self.logger.info( "Exposure time: %.3f s (observed), %.3f s (configured)" % (exposure_time, config.exposureTime()) ) # Compute the leading dead time, the time between starting the # readout of the previous frame and the arrival of the shot # immediately following it. This is an interesting statistic, # no matter what. XXX Maybe look at its distribution? dead_time = 0 if rr_observed > 0 and hasattr(self, "_previous_readout_time"): dead_time = self._timestamp[0] - self._previous_readout_time - 1 / rr_observed if math.isnan(dead_time): dead_time = 0 self.logger.info("Dead time: %.3f s" % dead_time) self._previous_readout_time = self._timestamp[-1] assert time == self._timestamp[-1] # XXX ZAP once one run survives it! # Flag blank images (i.e. images that had no hits), because # these may interesting for background subtraction. hits = self._hit.count(True) self.logger.info("Hit rate: %d/%d (%.2f%%)" % (hits, self._hit.size(), 100 * hits / self._hit.size())) if hits == 0: self.logger.info("Frame %d is blank" % self._nframes) # Get the normalisation factor by summing up I0 for all hits. # Invalid and non-positive values of I0 are treated as zeroes. # XXX Make this kind of summing a function of its own. I0 = sum(filter(lambda x: not math.isnan(x) and x > 0, self._I0.select(self._hit))) I0_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._I0)) fee_before_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._fee_before)) fee_after_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._fee_after)) # Register the template to the image and locate the regions of # interest based on the registration parameters. XXX Should # also give contrast: fit 2D-Gaussian to peak and report its # standard deviations and fit? if self._template is not None: gamma = lewis(self._template, self.cspad_img) p = flex.max_index(gamma) peak = ( p // gamma.focus()[1] - self._template.focus()[0] + 1, p % gamma.focus()[1] - self._template.focus()[1] + 1, ) # """ ### REFERENCE CHECK ### from os.path import dirname, isdir, join from scipy import io mat_dirname = dirname(cspad_tbx.pathsubst(self._mat_path, evt, env, frame_number=self._nframes)) if not isdir(mat_dirname): makedirs(mat_dirname) io.savemat( file_name=join(mat_dirname, "cross-check-%05d.mat" % self._nframes), mdict=dict( image=self.cspad_img.as_numpy_array(), template=self._template.as_numpy_array(), gamma=gamma.as_numpy_array(), peak=numpy.array(peak), ), appendmat=False, do_compression=True, oned_as="column", ) return ### REFERENCE CHECK ### # """ else: # Alternative: position everything with respect to the frame # origin. peak = (0, 0) # XXX Come up with a better way to handle the offsets! They # really do depend on the template, and should therefore be # packaged with it. self.logger.info("Template registration anchor point (%d, %d)" % (peak[0], peak[1])) roi = [] if evt.expNum() == 208: # Regions of interest for L632 (experiment number 208). XXX # Could perhaps migrate the template matching here instead? # The left, middle, and right manganese signals. XXX Extend the # rightmost ROI three pixels in upward direction (see runs 145 # and onwards, also note narrower slit)? roi.append((peak[0] + 59, peak[1] - 24, 12, 5)) roi.append((peak[0] + 61, peak[1] + 28, 12, 4)) roi.append((peak[0] + 61, peak[1] + 79, 12, 5)) # Two background regions between the manganese spots, with the # same total area as the signal. roi.append((peak[0] + 62, peak[1] + 1, 8, 8)) roi.append((peak[0] + 63, peak[1] + 51, 8, 8)) # The left and right direct reflections from the Si substrate # (i.e. the areas between the zone plates). These were the # features used for template registration. roi.append((peak[0], peak[1], 40, 10)) roi.append((peak[0], peak[1] + 50, 40, 9)) # Spot between the direct reflections. XXX What is this? roi.append((peak[0] + 1, peak[1] + 23, 22, 13)) # The horizontal slit, where the direct reflection occurs. This # is fixed. XXX Verify this! roi.append((22, 0, 41, 128)) # Background stripe, below the manganese spots. This is fixed # to the bottom of the detector. roi.append((104, 0, 20, 128)) elif evt.expNum() == 363: # Regions of interest for LB68 (experiment number 363). # 0-pixel are active, 255-pixel are inactive from scipy.misc import imread # Dec 5, 2013 (09:00 - 21:00): initial estimates from r0010 """ roi.append((peak[0] + 14, peak[1] + 138 + 23, 25, 50 - 25)) roi.append((peak[0] + 45, peak[1] + 138 + 23, 25, 50 - 25)) roi.append((peak[0] + 78, peak[1] + 137 + 23, 25, 50 - 25)) roi.append((peak[0] + 111, peak[1] + 137 + 23, 25, 50 - 25)) roi.append((peak[0] + 144, peak[1] + 137 + 23, 25, 50 - 25)) roi.append((peak[0] + 177, peak[1] + 136 + 23, 25, 50 - 25)) roi.append((peak[0] + 210, peak[1] + 136 + 23, 25, 50 - 25)) roi.append((peak[0] + 243, peak[1] + 136 + 23, 25, 50 - 25)) roi.append((peak[0] + 278, peak[1] + 135 + 23, 25, 50 - 25)) roi.append((peak[0] + 312, peak[1] + 135 + 23, 25, 50 - 25)) roi.append((peak[0] + 344, peak[1] + 135 + 23, 25, 50 - 25)) roi.append((peak[0] + 376, peak[1] + 135 + 23, 25, 50 - 25)) roi.append((peak[0] + 408, peak[1] + 135 + 23, 25, 50 - 25)) roi.append((peak[0] + 442, peak[1] + 135 + 23, 25, 50 - 25)) roi.append((peak[0] + 475, peak[1] + 135 + 23, 25, 50 - 25)) """ # Dec 6, 2013 (09:00 - 21:00): rough estimates """ roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg roi.append((peak[0] + 0, peak[1] + 135, 512, 25)) # oxygen roi.append((peak[0] + 0, peak[1] + 160, 512, 25)) # signal roi.append((peak[0] + 0, peak[1] + 300, 512, 130)) # zeroth order """ # Dec 7, 2013 (09:00 - 21:00): overlap between oxygen and # signal. Will loose some signal. """ roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg roi.append((peak[0] + 0, peak[1] + 135, 512, 50)) # oxygen roi.append((peak[0] + 0, peak[1] + 185, 512, 40)) # signal roi.append((peak[0] + 0, peak[1] + 270, 512, 170)) # zeroth order """ """ # Dec 7 2013 (09:00 - 21:00): binary masks stored in PNG # images. roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg roi.append((peak[0] + 0, peak[1] + 135, 512, 25)) # oxygen #roi_image = flex.float( # imread('/reg/neh/home1/hattne/myrelease/LB68-r0039-max-mask.png', # flatten=True)) #roi_image = flex.float( # imread('/reg/neh/home1/hattne/myrelease/LB68-r0039-std-mask.png', # flatten=True)) roi_image = flex.float( imread('/reg/neh/home1/hattne/myrelease/LB68-r0052-avg-mask.png', flatten=True)) roi_image = (255 - roi_image) #roi.append((0, 0, self.cspad_img.focus()[0], self.cspad_img.focus()[1])) roi.append(roi_image) roi.append((peak[0] + 0, peak[1] + 270, 512, 170)) # zeroth order """ # Dec 9, 2013 (09:00 - 21:00) # """ roi.append((peak[0] + 0, peak[1] + 25, 512, 25)) # bkg roi.append((peak[0] + 0, peak[1] + 135, 512, 25)) # oxygen # roi.append((peak[0] + 0, peak[1] + 160, 512, 25)) # signal roi_image = flex.float(imread("/reg/neh/home1/hattne/myrelease/LB68-r0067-max-mask.png", flatten=True)) roi.append(roi_image) roi.append((peak[0] + 0, peak[1] + 240, 512, 180)) # zeroth order # """ else: self.logger.error( "No regions of interest for %s (experiment number %d)" % (env.experiment(), evt.expNum()) ) # Clip the regions of interest to the actual image. If the ROI # does not overlap with the image at all, set its width and # height to zero. XXX Do the integration here as well? for i in range(len(roi)): if not isinstance(roi[i], tuple): continue r = roi[i] if ( r[0] + r[2] < 0 or r[0] >= self.cspad_img.focus()[0] or r[1] + r[3] < 0 or r[1] >= self.cspad_img.focus()[1] ): roi[i] = (r[0], r[1], 0, 0) continue r = roi[i] if r[0] < 0: roi[i] = (0, r[1], r[2] + r[0], r[3]) r = roi[i] if r[1] < 0: roi[i] = (r[0], 0, r[2], r[3] + r[1]) r = roi[i] if r[0] + r[2] > self.cspad_img.focus()[0]: roi[i] = (r[0], r[1], self.cspad_img.focus()[0] - r[0], r[3]) r = roi[i] if r[1] + r[3] > self.cspad_img.focus()[1]: roi[i] = (r[0], r[1], r[2], self.cspad_img.focus()[1] - r[1]) # Sum up intensities in all regions of interest, and keep track # of the actual number of pixels summed. The common_mode module # takes care of dark-subtraction. XXX Would like to estimate # sigma for spot, like in spotfinder/LABELIT. I = flex.double(len(roi)) I_nmemb = flex.int(len(roi)) for i in range(len(roi)): if isinstance(roi[i], flex.float): sel = roi[i].as_1d() < 128 I[i] = flex.sum(self.cspad_img.as_1d().select(sel)) I_nmemb[i] = sel.count(True) continue if roi[i][2] <= 0 or roi[i][3] <= 0: I[i] = 0 I_nmemb[i] = 0 else: I[i] = flex.sum( self.cspad_img.matrix_copy_block( i_row=roi[i][0], i_column=roi[i][1], n_rows=roi[i][2], n_columns=roi[i][3] ) ) I_nmemb[i] = roi[i][2] * roi[i][3] """ # Sanity check: white out the region of interest. self.cspad_img.matrix_paste_block_in_place( block=flex.double(flex.grid(roi[i][2], roi[i][3])), i_row=roi[i][0], i_column=roi[i][1]) """ acq_apd_sum = sum(filter(lambda x: not math.isnan(x) and x > 0, self._acq_apd_integral.select(self._hit))) acq_opto_diode_sum = sum( filter(lambda x: not math.isnan(x) and x > 0, self._acq_opto_diode_integral.select(self._hit)) ) acq_apd_sum_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._acq_apd_integral)) acq_opto_diode_sum_all = sum(filter(lambda x: not math.isnan(x) and x > 0, self._acq_opto_diode_integral)) # Append the data point to the stream: shots, hits, energy, and # I. XXX OrderedDict requires Python 2.7, could fall back on # regular Dict at the price of non-deterministic column order. from collections import OrderedDict csv_dict = OrderedDict( [ ("n_frames", self._hit.size()), ("n_hits", hits), ("I0", I0), ("I0_all", I0_all), ("fee_before_all", fee_before_all), ("fee_after_all", fee_after_all), ("energy_mean", energy_mean), ("acq_apd_sum", acq_apd_sum), ("acq_apd_sum_all", acq_apd_sum_all), ("acq_opto_diode_sum", acq_opto_diode_sum), ("acq_opto_diode_sum_all", acq_opto_diode_sum_all), ] ) for (i, item) in enumerate(zip(roi, I, I_nmemb)): key = "roi_" + ("bkg", "oxygen", "manganese", "zeroth_order")[i] csv_dict["%s_nmemb" % key] = item[2] if isinstance(item[0], tuple): csv_dict["%s_ss_start" % key] = item[0][0] csv_dict["%s_fs_start" % key] = item[0][1] csv_dict["%s_ss_size" % key] = item[0][2] csv_dict["%s_fs_size" % key] = item[0][3] else: csv_dict["%s_ss_start" % key] = 0 csv_dict["%s_fs_start" % key] = 0 csv_dict["%s_ss_size" % key] = item[0].focus()[0] csv_dict["%s_fs_size" % key] = item[0].focus()[1] csv_dict["%s_I" % key] = item[1] # XXX assert that keys match up with what's in the file already? # Or exploit the error-reporting mechanism already implemented? # Write the header. XXX How to control the order of the # columns? if not hasattr(self, "_csv"): from csv import DictWriter self._csv = DictWriter(self._stream_table, csv_dict.keys()) self._csv.writerow({key: key for key in csv_dict.keys()}) self._csv.writerow(csv_dict) # Output the non-normalised image and all other relevant data to # a binary MATLAB file. XXX What if scipy is not available? from os import makedirs, path from scipy import io mat_path = cspad_tbx.pathsubst(self._mat_path, evt, env, frame_number=self._nframes) if not path.isdir(path.dirname(mat_path)): makedirs(path.dirname(mat_path)) io.savemat( file_name=mat_path, mdict=dict( DATA=self.cspad_img.as_numpy_array(), DIODES=numpy.array((acq_apd_sum, acq_apd_sum_all, acq_opto_diode_sum, acq_opto_diode_sum_all)), ENERGY=energy_mean, HITS=numpy.array((hits, self._hit.size())), I0=numpy.array((I0, I0_all)), INTENSITIES=numpy.array(I), ROIS=numpy.array([r for r in roi if isinstance(r, tuple)]), ), appendmat=False, do_compression=True, oned_as="column", ) # Optionally update the image in the viewer. See mod_view. if self._display: from time import localtime, strftime # Copy over regions of interest to shared multiprocessing # array. XXX Flip to honour wxPython convention. for i in range(len(roi)): if not isinstance(roi[i], tuple): continue self._roi[4 * i + 0] = roi[i][1] self._roi[4 * i + 1] = roi[i][0] self._roi[4 * i + 2] = roi[i][3] self._roi[4 * i + 3] = roi[i][2] time_str = strftime("%H:%M:%S", localtime(evt.getTime().seconds())) title = "r%04d@%s: frame %d on %s" % (evt.run(), time_str, self._nframes, self.address) # XXX No distance in the Andor experiment. So don't bother # with the fictional beam center, distance, and saturation # value? See also mod_average.endjob() img_obj = ( dict( BEAM_CENTER=(0, 0), DATA=self.cspad_img, DETECTOR_ADDRESS=self.address, DISTANCE=10, # XXX Evil kludge to keep dxtbx happy! PIXEL_SIZE=13.5e-3, # XXX Hard-coded, again! SATURATED_VALUE=10000, TIME_TUPLE=cspad_tbx.evt_time(evt), WAVELENGTH=12398.4187 / energy, ), title, ) while not self._queue.empty(): if not self._proc.is_alive(): evt.setStatus(Event.Stop) return while True: try: self._queue.put(img_obj, timeout=1) break except Exception: # Queue.Full: pass self._reset_counters() return
def process_event(self, run, timestamp): """ Process a single event from a run @param run psana run object @param timestamp psana timestamp object """ ts = cspad_tbx.evt_timestamp( (timestamp.seconds(), timestamp.nanoseconds() / 1e6)) if ts is None: print "No timestamp, skipping shot" return if len(self.params_cache.debug.event_timestamp ) > 0 and ts not in self.params_cache.debug.event_timestamp: return if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events: if ts in self.known_events: if self.known_events[ts] not in ["stop", "done", "fail"]: if self.params_cache.debug.skip_bad_events: print "Skipping event %s: possibly caused an unknown exception previously" % ts return elif self.params_cache.debug.skip_processed_events: print "Skipping event %s: processed successfully previously" % ts return else: if self.params_cache.debug.skip_unprocessed_events: print "Skipping event %s: not processed previously" % ts return self.debug_start(ts) evt = run.event(timestamp) if evt.get("skip_event") or "skip_event" in [ key.key() for key in evt.keys() ]: print "Skipping event", ts self.debug_write("psana_skip", "skip") return print "Accepted", ts self.params = copy.deepcopy(self.params_cache) # the data needs to have already been processed and put into the event by psana if self.params.format.file_format == 'cbf': # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data( self.psana_det, evt, use_default=False, dark=True, common_mode=self.common_mode, apply_gain_mask=self.params.format.cbf.gain_mask_value is not None, gain_mask_value=self.params.format.cbf.gain_mask_value, per_pixel_gain=False) if data is None: print "No data" self.debug_write("no_data", "skip") return if self.params.format.cbf.override_distance is None: distance = cspad_tbx.env_distance( self.params.input.address, run.env(), self.params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" self.debug_write("no_distance", "skip") return else: distance = self.params.format.cbf.override_distance if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" self.debug_write("no_wavelength", "skip") return else: wavelength = 12398.4187 / self.params.format.cbf.override_energy if self.params.format.file_format == 'pickle': image_dict = evt.get(self.params.format.pickle.out_key) data = image_dict['DATA'] timestamp = t = ts s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[ 20:23] print "Processing shot", s if self.params.format.file_format == 'cbf': # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data( self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address) if self.params.input.reference_geometry is not None: from dxtbx.model import Detector # copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead. cspad_img._detector_instance = Detector.from_dict( self.reference_detector.to_dict()) cspad_img.sync_detector_to_cbf() elif self.params.format.file_format == 'pickle': from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory cspad_img = FormatPYunspecifiedStillInMemory(image_dict) cspad_img.timestamp = s if self.params.dispatch.dump_all: self.save_image( cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s)) self.cache_ranges(cspad_img, self.params) imgset = MemImageSet([cspad_img]) if self.params.dispatch.estimate_gain_only: from dials.command_line.estimate_gain import estimate_gain estimate_gain(imgset) return if not self.params.dispatch.find_spots: self.debug_write("data_loaded", "done") return datablock = DataBlockFactory.from_imageset(imgset)[0] # before calling DIALS for processing, set output paths according to the templates if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template: self.params.output.indexed_filename = os.path.join( self.params.output.output_dir, self.indexed_filename_template % ("idx-" + s)) if "%s" in self.refined_experiments_filename_template: self.params.output.refined_experiments_filename = os.path.join( self.params.output.output_dir, self.refined_experiments_filename_template % ("idx-" + s)) if "%s" in self.integrated_filename_template: self.params.output.integrated_filename = os.path.join( self.params.output.output_dir, self.integrated_filename_template % ("idx-" + s)) if "%s" in self.reindexedstrong_filename_template: self.params.output.reindexedstrong_filename = os.path.join( self.params.output.output_dir, self.reindexedstrong_filename_template % ("idx-" + s)) # Load a dials mask from the trusted range and psana mask from dials.util.masking import MaskGenerator generator = MaskGenerator(self.params.border_mask) mask = generator.generate(imgset) if self.params.format.file_format == "cbf": mask = tuple([a & b for a, b in zip(mask, self.dials_mask)]) if self.spotfinder_mask is None: self.params.spotfinder.lookup.mask = mask else: self.params.spotfinder.lookup.mask = tuple( [a & b for a, b in zip(mask, self.spotfinder_mask)]) if self.integration_mask is None: self.params.integration.lookup.mask = mask else: self.params.integration.lookup.mask = tuple( [a & b for a, b in zip(mask, self.integration_mask)]) self.debug_write("spotfind_start") try: observed = self.find_spots(datablock) except Exception, e: import traceback traceback.print_exc() print str(e), "event", timestamp self.debug_write("spotfinding_exception", "fail") return
def average(argv=None): if argv == None: argv = sys.argv[1:] try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") command_line = (libtbx.option_parser.option_parser(usage=""" %s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath] To write image pickles use -p, otherwise the program writes CSPAD CBFs. Writing CBFs requires the geometry to be already deployed. Examples: cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 Use one process on the current node to process all the events from run 25 of experiment cxi49812, using a detz_offset of 571. mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 As above, using 16 cores on the current node. bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812 As above, using the psanaq and 100 cores, putting the log in average.out and the output images in the folder cxi49812. """ % libtbx.env.dispatcher_name).option( None, "--as_pickle", "-p", action="store_true", default=False, dest="as_pickle", help="Write results as image pickle files instead of cbf files" ).option( None, "--raw_data", "-R", action="store_true", default=False, dest="raw_data", help= "Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)" ).option( None, "--background_pickle", "-B", default=None, dest="background_pickle", help="" ).option( None, "--config", "-c", type="string", default=None, dest="config", metavar="PATH", help="psana config file" ).option( None, "--experiment", "-x", type="string", default=None, dest="experiment", help="experiment name (eg cxi84914)" ).option( None, "--run", "-r", type="int", default=None, dest="run", help="run number" ).option( None, "--address", "-a", type="string", default="CxiDs2.0:Cspad.0", dest="address", help="detector address name (eg CxiDs2.0:Cspad.0)" ).option( None, "--detz_offset", "-d", type="float", default=None, dest="detz_offset", help= "offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)" ).option( None, "--outputdir", "-o", type="string", default=".", dest="outputdir", metavar="PATH", help="Optional path to output directory for output files" ).option( None, "--averagebase", "-A", type="string", default="{experiment!l}_avg-r{run:04d}", dest="averagepath", metavar="PATH", help= "Path to output average image without extension. String substitution allowed" ).option( None, "--stddevbase", "-S", type="string", default="{experiment!l}_stddev-r{run:04d}", dest="stddevpath", metavar="PATH", help= "Path to output standard deviation image without extension. String substitution allowed" ).option( None, "--maxbase", "-M", type="string", default="{experiment!l}_max-r{run:04d}", dest="maxpath", metavar="PATH", help= "Path to output maximum projection image without extension. String substitution allowed" ).option( None, "--numevents", "-n", type="int", default=None, dest="numevents", help="Maximum number of events to process. Default: all" ).option( None, "--skipevents", "-s", type="int", default=0, dest="skipevents", help="Number of events in the beginning of the run to skip. Default: 0" ).option( None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress" ).option( None, "--pickle-optical-metrology", "-m", action="store_true", default=False, dest="pickle_optical_metrology", help= "If writing pickle files, use the optical metrology in the experiment's calib directory" ).option( None, "--bin_size", "-b", type="int", default=None, dest="bin_size", help="Rayonix detector bin size" ).option( None, "--override_beam_x", "-X", type="float", default=None, dest="override_beam_x", help="Rayonix detector beam center x coordinate" ).option( None, "--override_beam_y", "-Y", type="float", default=None, dest="override_beam_y", help="Rayonix detector beam center y coordinate" ).option( None, "--calib_dir", "-C", type="string", default=None, dest="calib_dir", metavar="PATH", help="calibration directory" ).option( None, "--pickle_calib_dir", "-P", type="string", default=None, dest="pickle_calib_dir", metavar="PATH", help= "pickle calibration directory specification. Replaces --calib_dir functionality." ).option( None, "--xtc_dir", "-D", type="string", default=None, dest="xtc_dir", metavar="PATH", help="xtc stream directory" ).option( None, "--use_ffb", "-f", action="store_true", default=False, dest="use_ffb", help= "Use the fast feedback filesystem at LCLS. Only for the active experiment!" ).option( None, "--gain_mask_value", "-g", type="float", default=None, dest="gain_mask_value", help= "Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode." ).option( None, "--min", None, action="store_true", default=False, dest="do_minimum_projection", help="Output a minimum projection" ).option( None, "--minpath", None, type="string", default="{experiment!l}_min-r{run:04d}", dest="minpath", metavar="PATH", help= "Path to output minimum image without extension. String substitution allowed" )).process(args=argv) if len(command_line.args) > 0 or \ command_line.options.as_pickle is None or \ command_line.options.experiment is None or \ command_line.options.run is None or \ command_line.options.address is None or \ command_line.options.detz_offset is None or \ command_line.options.averagepath is None or \ command_line.options.stddevpath is None or \ command_line.options.maxpath is None or \ command_line.options.pickle_optical_metrology is None: command_line.parser.show_help() return # set this to sys.maxint to analyze all events if command_line.options.numevents is None: maxevents = sys.maxsize else: maxevents = command_line.options.numevents comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if command_line.options.config is not None: psana.setConfigFile(command_line.options.config) dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment, command_line.options.run) if command_line.options.xtc_dir is not None: if command_line.options.use_ffb: raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system") dataset_name += ":dir=%s" % command_line.options.xtc_dir elif command_line.options.use_ffb: # as ffb is only at SLAC, ok to hardcode /reg/d here dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % ( command_line.options.experiment[0:3], command_line.options.experiment) if command_line.options.calib_dir is not None: psana.setOption('psana.calib-dir', command_line.options.calib_dir) ds = psana.DataSource(dataset_name) address = command_line.options.address src = psana.Source('DetInfo(%s)' % address) nevent = np.array([0.]) if command_line.options.background_pickle is not None: background = easy_pickle.load( command_line.options.background_pickle)['DATA'].as_numpy_array() for run in ds.runs(): runnumber = run.run() if not command_line.options.as_pickle: psana_det = psana.Detector(address, ds.env()) # list of all events if command_line.options.skipevents > 0: print("Skipping first %d events" % command_line.options.skipevents) elif "Rayonix" in command_line.options.address: print("Skipping first image in the Rayonix detector" ) # Shuttering issue command_line.options.skipevents = 1 for i, evt in enumerate(run.events()): if i % size != rank: continue if i < command_line.options.skipevents: continue if i >= maxevents: break if i % 10 == 0: print('Rank', rank, 'processing event', i) #print "Event #",rank*mylength+i," has id:",evt.get(EventId) if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: data = evt.get(psana.Camera.FrameV1, src) if data is None: print("No data") continue data = data.data16().astype(np.float64) elif command_line.options.as_pickle: data = evt.get(psana.ndarray_float64_3, src, 'image0') else: # get numpy array, 32x185x388 from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data if command_line.options.raw_data: data = get_psana_corrected_data(psana_det, evt, use_default=False, dark=False, common_mode=None, apply_gain_mask=False, per_pixel_gain=False) else: if command_line.options.gain_mask_value is None: data = get_psana_corrected_data(psana_det, evt, use_default=True) else: data = get_psana_corrected_data( psana_det, evt, use_default=False, dark=True, common_mode=None, apply_gain_mask=True, gain_mask_value=command_line.options. gain_mask_value, per_pixel_gain=False) if data is None: print("No data") continue if command_line.options.background_pickle is not None: data -= background if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: distance = np.array([0.0]) wavelength = np.array([1.0]) else: d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset) if d is None: print("No distance, using distance", command_line.options.detz_offset) assert command_line.options.detz_offset is not None if 'distance' not in locals(): distance = np.array([command_line.options.detz_offset]) else: distance += command_line.options.detz_offset else: if 'distance' in locals(): distance += d else: distance = np.array([float(d)]) w = cspad_tbx.evt_wavelength(evt) if w is None: print("No wavelength") if 'wavelength' not in locals(): wavelength = np.array([1.0]) else: if 'wavelength' in locals(): wavelength += w else: wavelength = np.array([w]) t = cspad_tbx.evt_time(evt) if t is None: print("No timestamp, skipping shot") continue if 'timestamp' in locals(): timestamp += t[0] + (t[1] / 1000) else: timestamp = np.array([t[0] + (t[1] / 1000)]) if 'sum' in locals(): sum += data else: sum = np.array(data, copy=True) if 'sumsq' in locals(): sumsq += data * data else: sumsq = data * data if 'maximum' in locals(): maximum = np.maximum(maximum, data) else: maximum = np.array(data, copy=True) if command_line.options.do_minimum_projection: if 'minimum' in locals(): minimum = np.minimum(minimum, data) else: minimum = np.array(data, copy=True) nevent += 1 #sum the images across mpi cores if size > 1: print("Synchronizing rank", rank) totevent = np.zeros(nevent.shape) comm.Reduce(nevent, totevent) if rank == 0 and totevent[0] == 0: raise Sorry("No events found in the run") sumall = np.zeros(sum.shape).astype(sum.dtype) comm.Reduce(sum, sumall) sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype) comm.Reduce(sumsq, sumsqall) maxall = np.zeros(maximum.shape).astype(maximum.dtype) comm.Reduce(maximum, maxall, op=MPI.MAX) if command_line.options.do_minimum_projection: minall = np.zeros(maximum.shape).astype(minimum.dtype) comm.Reduce(minimum, minall, op=MPI.MIN) waveall = np.zeros(wavelength.shape).astype(wavelength.dtype) comm.Reduce(wavelength, waveall) distall = np.zeros(distance.shape).astype(distance.dtype) comm.Reduce(distance, distall) timeall = np.zeros(timestamp.shape).astype(timestamp.dtype) comm.Reduce(timestamp, timeall) if rank == 0: if size > 1: print("Synchronized") # Accumulating floating-point numbers introduces errors, # which may cause negative variances. Since a two-pass # approach is unacceptable, the standard deviation is # clamped at zero. mean = sumall / float(totevent[0]) variance = (sumsqall / float(totevent[0])) - (mean**2) variance[variance < 0] = 0 stddev = np.sqrt(variance) wavelength = waveall[0] / totevent[0] distance = distall[0] / totevent[0] pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value timestamp = timeall[0] / totevent[0] timestamp = (int(timestamp), timestamp % int(timestamp) * 1000) timestamp = cspad_tbx.evt_timestamp(timestamp) if command_line.options.as_pickle: extension = ".pickle" else: extension = ".cbf" dest_paths = [ cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env()) ] if command_line.options.do_minimum_projection: dest_paths.append( cspad_tbx.pathsubst(command_line.options.minpath + extension, evt, ds.env())) dest_paths = [ os.path.join(command_line.options.outputdir, path) for path in dest_paths ] if 'Rayonix' in command_line.options.address: all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) from xfel.cxi.cspad_ana import rayonix_tbx pixel_size = rayonix_tbx.get_rayonix_pixel_size( command_line.options.bin_size) beam_center = [ command_line.options.override_beam_x, command_line.options.override_beam_y ] active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]]) split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] for data, path in zip(all_data, dest_paths): print("Saving", path) d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=flex.double(data), distance=distance, pixel_size=pixel_size, saturated_value=rayonix_tbx.rayonix_saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: all_data = [mean, stddev, maxall] split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): d = cspad_tbx.dpack(address=old_style_address, data=flex.double(data), distance=distance, pixel_size=0.1, timestamp=timestamp, wavelength=wavelength) print("Saving", path) easy_pickle.dump(path, d) elif command_line.options.as_pickle: split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] xpp = 'xpp' in address.lower() if xpp: evt_time = cspad_tbx.evt_time( evt) # tuple of seconds, milliseconds timestamp = cspad_tbx.evt_timestamp( evt_time) # human readable format from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version( old_style_address, reverse_timestamp(timestamp)[0]) assert version_lookup is not None active_areas = xpp_active_areas[version_lookup]['active_areas'] beam_center = [1765 // 2, 1765 // 2] else: if command_line.options.pickle_calib_dir is not None: metro_path = command_line.options.pickle_calib_dir elif command_line.options.pickle_optical_metrology: from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path metro_path = get_calib_file_path(run.env(), address, run) else: metro_path = libtbx.env.find_in_repositories( "xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0") sections = parse_calib.calib2sections(metro_path) beam_center, active_areas = cspad_tbx.cbcaa( cspad_tbx.getConfig(address, ds.env()), sections) class fake_quad(object): def __init__(self, q, d): self.q = q self.d = d def quad(self): return self.q def data(self): return self.d if xpp: quads = [ fake_quad(i, mean[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [ fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [ fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [ fake_quad(i, minall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] minall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) minall = flex.double(minall.astype(np.float64)) else: quads = [ fake_quad(i, mean[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] mean = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [ fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] stddev = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [ fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] maxall = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [ fake_quad(i, minall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] minall = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) minall = flex.double(minall.astype(np.float64)) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print("Saving", path) d = cspad_tbx.dpack(active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=data, distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) else: # load a header only cspad cbf from the slac metrology from xfel.cftbx.detector import cspad_cbf_tbx import pycbf base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology( run, address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d" % run.run()) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print("Saving", path) cspad_img = cspad_cbf_tbx.format_object_from_data( base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=False) cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def run(self): """ Process all images assigned to this thread """ params, options = self.parser.parse_args(show_diff_phil=True) if params.input.experiment is None or \ params.input.run_num is None or \ params.input.address is None: raise Usage(self.usage) if params.format.file_format == "cbf": if params.format.cbf.detz_offset is None: raise Usage(self.usage) elif params.format.file_format == "pickle": if params.input.cfg is None: raise Usage(self.usage) else: raise Usage(self.usage) if not os.path.exists(params.output.output_dir): raise Sorry("Output path not found:" + params.output.output_dir) #Environment variable redirect for CBFLib temporary CBF_TMP_XYZ file output if params.format.file_format == "cbf": if params.output.tmp_output_dir is None: tmp_dir = os.path.join(params.output.output_dir, '.tmp') else: tmp_dir = os.path.join(params.output.tmp_output_dir, '.tmp') if not os.path.exists(tmp_dir): with show_mail_on_error(): try: os.makedirs(tmp_dir) # Can fail if running multiprocessed - that's OK if the folder was created except OSError as e: # In Python 2, a FileExistsError is just an OSError if e.errno != errno.EEXIST: # If this OSError is not a FileExistsError raise os.environ['CBF_TMP_DIR'] = tmp_dir # Save the paramters self.params = params self.options = options from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank( ) # each process in MPI has a unique id, 0-indexed size = comm.Get_size() # size: number of processes running in this job # set up psana if params.input.cfg is not None: psana.setConfigFile(params.input.cfg) if params.input.calib_dir is not None: psana.setOption('psana.calib-dir', params.input.calib_dir) dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment, params.input.run_num) if params.input.xtc_dir is not None: dataset_name = "exp=%s:run=%s:idx:dir=%s" % ( params.input.experiment, params.input.run_num, params.input.xtc_dir) ds = psana.DataSource(dataset_name) if params.format.file_format == "cbf": src = psana.Source('DetInfo(%s)' % params.input.address) psana_det = psana.Detector(params.input.address, ds.env()) # set this to sys.maxint to analyze all events if params.dispatch.max_events is None: max_events = sys.maxsize else: max_events = params.dispatch.max_events for run in ds.runs(): if params.format.file_format == "cbf": if params.format.cbf.mode == "cspad": # load a header only cspad cbf from the slac metrology base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology( run, params.input.address) if base_dxtbx is None: raise Sorry( "Couldn't load calibration file for run %d" % run.run()) elif params.format.cbf.mode == "rayonix": # load a header only rayonix cbf from the input parameters detector_size = rayonix_tbx.get_rayonix_detector_dimensions( ds.env()) base_dxtbx = rayonix_tbx.get_dxtbx_from_params( params.format.cbf.rayonix, detector_size) # list of all events times = run.times() if params.dispatch.selected_events: times = [ t for t in times if cspad_tbx.evt_timestamp((t.seconds(), t.nanoseconds() / 1e6)) in params.input.timestamp ] nevents = min(len(times), max_events) # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes mytimes = [ times[i] for i in range(nevents) if (i + rank) % size == 0 ] for i in range(len(mytimes)): evt = run.event(mytimes[i]) id = evt.get(psana.EventId) print("Event #", i, " has id:", id) timestamp = cspad_tbx.evt_timestamp( cspad_tbx.evt_time(evt)) # human readable format if timestamp is None: print("No timestamp, skipping shot") continue if evt.get("skip_event") or "skip_event" in [ key.key() for key in evt.keys() ]: print("Skipping event", timestamp) continue t = timestamp s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[ 17:19] + t[20:23] print("Processing shot", s) if params.format.file_format == "pickle": if evt.get("skip_event"): print("Skipping event", id) continue # the data needs to have already been processed and put into the event by psana data = evt.get(params.format.pickle.out_key) if data is None: print("No data") continue # set output paths according to the templates path = os.path.join(params.output.output_dir, "shot-" + s + ".pickle") print("Saving", path) easy_pickle.dump(path, data) elif params.format.file_format == "cbf": if params.format.cbf.mode == "cspad": # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data( psana_det, evt, use_default=False, dark=True, common_mode=None, apply_gain_mask=params.format.cbf.cspad. gain_mask_value is not None, gain_mask_value=params.format.cbf.cspad. gain_mask_value, per_pixel_gain=False) distance = cspad_tbx.env_distance( params.input.address, run.env(), params.format.cbf.detz_offset) elif params.format.cbf.mode == "rayonix": data = rayonix_tbx.get_data_from_psana_event( evt, params.input.address) distance = params.format.cbf.detz_offset if distance is None: print("No distance, skipping shot") continue if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print("No wavelength, skipping shot") continue else: wavelength = 12398.4187 / self.params.format.cbf.override_energy # stitch together the header, data and metadata into the final dxtbx format object if params.format.cbf.mode == "cspad": image = cspad_cbf_tbx.format_object_from_data( base_dxtbx, data, distance, wavelength, timestamp, params.input.address, round_to_int=False) elif params.format.cbf.mode == "rayonix": image = rayonix_tbx.format_object_from_data( base_dxtbx, data, distance, wavelength, timestamp, params.input.address) path = os.path.join(params.output.output_dir, "shot-" + s + ".cbf") print("Saving", path) # write the file import pycbf image._cbf_handle.write_widefile(path.encode(), pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0) run.end() ds.end()
def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ # Increase the event counter, even if this event is to be skipped. self.nshots += 1 if (evt.get("skip_event")): return sifoil = cspad_tbx.env_sifoil(env) if self.check_beam_status and sifoil is None: self.nfail += 1 self.logger.warning("event(): no Si-foil thickness, shot skipped") evt.put(skip_event_flag(), "skip_event") return if (self.sifoil is not None and self.sifoil != sifoil): self.logger.warning("event(): Si-foil changed mid-run: % 8i -> % 8d" % (self.sifoil, sifoil)) self.sifoil = sifoil if self.verbose: self.logger.info("Si-foil thickness: %i" %sifoil) self.evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds self.timestamp = cspad_tbx.evt_timestamp(self.evt_time) # human readable format if (self.timestamp is None): self.nfail += 1 self.logger.warning("event(): no timestamp, shot skipped") evt.put(skip_event_flag(), "skip_event") return if self.verbose: self.logger.info(self.timestamp) if self.override_energy is None: self.wavelength = cspad_tbx.evt_wavelength(evt, self.delta_k) if self.wavelength is None: if self.check_beam_status: self.nfail += 1 self.logger.warning("event(): no wavelength, shot skipped") evt.put(skip_event_flag(), "skip_event") return else: self.wavelength = 0 else: self.wavelength = 12398.4187/self.override_energy if self.verbose: self.logger.info("Wavelength: %.4f" %self.wavelength) self.pulse_length = cspad_tbx.evt_pulse_length(evt) if self.pulse_length is None: if self.check_beam_status: self.nfail += 1 self.logger.warning("event(): no pulse length, shot skipped") evt.put(skip_event_flag(), "skip_event") return else: self.pulse_length = 0 if self.verbose: self.logger.info("Pulse length: %s" %self.pulse_length) self.beam_charge = cspad_tbx.evt_beam_charge(evt) if self.beam_charge is None: if self.check_beam_status: self.nfail += 1 self.logger.warning("event(): no beam charge, shot skipped") evt.put(skip_event_flag(), "skip_event") return else: self.beam_charge = 0 if self.verbose: self.logger.info("Beam charge: %s" %self.beam_charge) self.injector_xyz = cspad_tbx.env_injector_xyz(env) #if self.injector_xyz is not None: #self.logger.info("injector_z: %i" %self.injector_xyz[2].value) self.laser_1_status.set_status(cspad_tbx.env_laser_status(env, laser_id=1), self.evt_time) self.laser_4_status.set_status(cspad_tbx.env_laser_status(env, laser_id=4), self.evt_time) self.laser_1_ms_since_change = self.laser_1_status.ms_since_last_status_change(self.evt_time) self.laser_4_ms_since_change = self.laser_4_status.ms_since_last_status_change(self.evt_time) if self.verbose: if self.laser_1_ms_since_change is not None: self.logger.info("ms since laser 1 status change: %i" %self.laser_1_ms_since_change) if self.laser_4_ms_since_change is not None: self.logger.info("ms since laser 4 status change: %i" %self.laser_4_ms_since_change) if self.laser_1_status is not None and self.laser_1_status.status is not None: self.logger.info("Laser 1 status: %i" %int(self.laser_1_status.status)) if self.laser_4_status is not None and self.laser_4_status.status is not None: self.logger.info("Laser 4 status: %i" %int(self.laser_4_status.status))
def __init__(self, dataset_name = None, detector_address = None, data_type = 'xtc', mask_path = None, mask_angles = None, mask_widths = None, backimg_path = None, backmsk_path = None, param_path = None, det_dist = None, det_pix = 0.075, beam_l = None, mask_thr = None, nQ = None, nPhi = None, dQ = 1, dPhi = 1, cent0 = None, r_max = None, dr = 10, dx = 5, dy = 5, r_0 = None, q_bound = None): """The fluctuation scattering class stores processing parameters, initiates mask and background data and retrieves 2D images from events. Processing options of the 2D images include: * Transform from cartesian to polar coordinates * Beam center refinement * Dynamic masking * Normalization % SAXS calculation * Particle sizing * Computation of in-frame 2-point angular auto-correlations using FFTs @param dataset_name Experiment name and run number @param detector_address Adress to back or front detector @param data_type Type of data file format (xtc, ffb, h5) @param mask_path Full path to static image mask @param mask_angles Center of angluar slices (deg) that should be masked out (due to jet streaks etc), [Ang1 Ang2 ...] @param mask_widths Width of angular slices (deg) that should be masked out (due to jet streaks etc), [delta1 delta2 ...] @param backimg_path Full path to background image @param backmsk_path Full path to background mask @param param_path Full path to file with pre-computed parameters (i.e beam center,particle nr,particle size) @param det_dist Override of detecor distance (in mm) @param det_pix Pixel size (in mm) @param beam_l Override of beam wavelength (in Angstrom) @param mask_thr Threshold for dynamic masking @param nQ Number of Q-bins to consider (in pixels) @param nPhi Number of Phi-bins to consider (in pixels) @param dQ Stepsize in Q (in pixels) @param dPhi Stepsize in Phi (in pixels) @param cent0 Initial beam center coordinates [xc,yc] @param r_max Maximum radial value to use for beamcenter refinement (in pixels) @param dr Stepsize in r (in pixels) @param dx Gridsize for beam center refinement in x, i.e xc+/-dx (in pixels) @param dy Gridsize for beam center refinement in y, i.e yc+/-dy (in pixles) @param r_0 Starting value for particle radius refinement [in Ang] @param q_bound Upper and Lower boundaries of q for Particle radius refinement [in Ang^-1] """ # Initialize parameters and configuration files once self.data_type = data_type self.dataset_name = dataset_name self.detector_address = detector_address if (self.data_type == 'xtc') or (self.data_type == 'ffb') : self.ds = DataSource(self.dataset_name) self.src = Detector(self.detector_address, self.ds.env()) if mask_path is None : # Create a binary mask of ones, default mask only works for xtc/ffb for run in self.ds.runs(): times = run.times() evt = run.event(times[0]) break mask_address = self.src.mask(run.run(),calib=True,status=True,edges=True,central=True,unbond=True,unbondnbrs=True) self.msk = self.src.image(evt,mask_address) else : self.msk = np.loadtxt(mask_path).astype(np.float64) if backimg_path is None : self.backimg = None else : self.backimg = np.loadtxt(backimg_path).astype(np.float64) if backmsk_path is None : self.backmsk = None else : self.backmsk = np.loadtxt(backmsk_path).astype(np.float64) if param_path is None : self.param = [] if det_dist is None : # Get detector distance from events for run in self.ds.runs(): self.det_dist = cspad_tbx.env_distance(self.detector_address, run.env(), 577) else : self.det_dist = det_dist self.det_pix = det_pix if beam_l is None : # Get wavelength from event, note it change slightly between events for run in self.ds.runs(): times = run.times() evt = run.event(times[0]) break self.beam_l = cspad_tbx.evt_wavelength(evt) else : self.beam_l = beam_l if mask_thr is None : # No dynamic masking self.thr = None else : self.thr = mask_thr if nQ is None : # Use image dimensions as a guide, leave room for offset beamC if self.msk.shape[0] > self.msk.shape[1] : self.nQ = int(self.msk.shape[1]/2)-20 else : self.nQ = int(self.msk.shape[0]/2)-20 else : self.nQ = nQ if (self.nQ % 10): # Ascert even number, speeds things up massively for FFT self.nQ = np.floor(self.nQ/10)*10 if (self.nQ % dQ): # Ascert clean divisor self.nQ = np.floor(self.nQ/dQ)*dQ if nPhi is None : # Estimate based on 2*pi*nQ self.nPhi = np.ceil(2*np.pi*self.nQ) else : self.nPhi = nPhi if (self.nPhi % 10): # Ascert even number, speeds things up massively for FFT self.nPhi = np.ceil(self.nPhi/10)*10 if (self.nPhi % dPhi): # Ascert clean divisor self.nPhi = np.ceil(self.nPhi/dPhi)*dPhi self.dQ = dQ self.dPhi = dPhi self.mask_angles = mask_angles self.mask_widths = mask_widths # Compute slices that should be masked in static mask if (self.mask_angles is not None) and (self.mask_widths is not None) : self.mask_angles = (self.mask_angles/360) * self.nPhi self.mask_widths = (self.mask_widths/360) * self.nPhi if (cent0 is None) or (sum(cent0) == 0): # Use center of gravity to estimate starting beamC self.cent0 = [int(round(self.msk.shape[1]/2)) , int(round(self.msk.shape[0]/2))] else : self.cent0 = cent0 self.cent = self.cent0 # Default center if r_max is None : # Default, Use half of nQ self.r_max = int(self.nQ/2) else : self.r_max = r_max if (self.r_max % dr): # Ascert clean divisor self.r_max = np.floor(self.r_max/dr)*dr self.dr = dr self.dx = dx self.dy = dy if r_0 is None : self.radius = 0 self.score = 0 self.r_0 = r_0 if q_bound is None or sum(q_bound)==0 : self.q_bound = [None,None] else : self.q_bound = [None,self.q_bound] # Compute q-spacing self.q = np.arange(0, self.nQ, self.dQ) self.q = self.q*self.det_pix/self.det_dist*4*np.pi/self.beam_l/2 # Compute Phi (Not accounting for curvature) self.phi = np.linspace(0, 2*np.pi, self.nPhi/self.dPhi,endpoint=False)