def _get_evt_det(self, evt): if self.acq_num == 1: acq1_src = psana.Source('DetInfo(SxrEndstation.0:Acqiris.1)') acq_det = evt.get(psana.Acqiris.DataDescV1, acq1_src) elif self.acq_num == 2: acq2_src = psana.Source('DetInfo(SxrEndstation.0:Acqiris.2)') acq_det = evt.get(psana.Acqiris.DataDescV1, acq2_src) return acq_det
def __init__(self): #Handle warnings warnings.filterwarnings('always',module='Utils',category=UserWarning) warnings.filterwarnings('ignore',module='Utils',category=RuntimeWarning, message="invalid value encountered in divide") #Some default values for the options self._experiment='amoc8114' #Experiment label self._darkreferencepath=[]; #Dark reference information self._lasingoffreferencepath=[]; #Dark reference information self._darkreference=[] self._lasingoffreference=[] self._nb=float('nan') #Number of bunches self._medianfilter=float('nan') #Number of neighbours for median filter self._snrfilter=float('nan') #Number of sigmas for the noise threshold self._roiwaistthres=float('nan') #Parameter for the roi location self._roiexpand=float('nan') #Parameter for the roi location self._islandsplitmethod='' #Method for island splitting self._islandsplitpar1=float('nan') self._islandsplitpar2=float('nan') self._currentevent=[] self._eventresultsstep1=[] self._eventresultsstep2=[] self._eventresultsstep3=[] self._env=[] self._globalcalibration=[] self._roixtcav=[] self._calpath='' #Different flags self._loadeddarkreference=False self._loadedlasingoffreference=False self._currenteventavailable=False self._currenteventprocessedstep1=False #Step one is pure processing of the trace, without lasing off referencing self._currenteventprocessedstep2=False #Step two is calculating physical units self._currenteventprocessedstep3=False #Step three is the processing of the profiles with respect to the reference self._envinfo=False #Camera and type for the xtcav images self.xtcav_camera = psana.Source('DetInfo(XrayTransportDiagnostic.0:Opal1000.0)') self.xtcav_type=psana.Camera.FrameV1 self._rawimage=[] #Ebeam type: it should actually be the version 5 which is the one that contains xtcav stuff self.ebeam_data=psana.Source('BldInfo(EBeam)') self._ebeam=[] #Gas detectors for the pulse energies self.gasdetector_data=psana.Source('BldInfo(FEEGasDetEnergy)') self._gasdetector=[]
def get_sharpness_vs_position(): ''' This is one function because I could not figure out how to get it to work as separate functions ''' ds = psana.DataSource("exp=SXR/sxri0214:run=139:idx") src = psana.Source("DetInfo(SxrBeamline.0:Opal1000.1)") epics = ds.env().epicsStore() m1_positions = [] sharp_vals = [] print ds.runs() for run in ds.runs(): #new #maybe there is only one run? It prints in a #way I don't understand print run times = run.times() #inxs = [] #indx = 0 for tm in times[0:120000]: print tm evt = run.event(tm) camera = evt.get(psana.Camera.FrameV1, src) projection = np.sum(camera.data16(), axis=1) a_val = working_algorithm.main(projection) sharp_vals.append(a_val) #if a_val < 100: # inxs.append(indx) position = epics.value('SXR:MON:MMS:05.RBV') m1_positions.append(position) #indx += 1 ar = np.array([sharp_vals, m1_positions]) print ar return ar
def look_at_spectrometer(): plt.figure(1) for run in range(282, 291): runstr = 'exp=diamcc14:run=%d' % run try: ds = psana.DataSource(runstr) except Exception, exp: print("unable to open %s" % runstr) print(exp) continue for evt in ds.events(): opal = evt.get( psana.Camera.FrameV1, psana.Source('DetInfo(XrayTransportDiagnostic.20:Opal1000.0)')) if None is opal: continue MX = 500 plt.imshow(np.minimum( MX, np.maximum(0, opal.data16().astype(np.float))), vmin=0, vmax=MX, interpolation='none') plt.show() plt.pause(.1) raw_input('hi')
def acqiris_data_dataext(event): return event['evt'].get( psana.Acqiris.DataDescV1, psana.Source(gp.monitor_params['PsanaParallelizationLayer'] ['digitizer_psana_source'])).data( gp.monitor_params['PsanaParallelizationLayer'] ['digitizer_psana_channel']).waveforms()
def get_rayonix_detector_dimensions(env): ''' Given a psana env object, find the detector dimensions @param env psana environment object ''' import psana cfgs = env.configStore() rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source('Rayonix')) return rayonix_cfg.width(), rayonix_cfg.height()
def psana_event_inspection(source): """Prints the structure of psana events. Takes a psana source string (e.g. exp=CXI/cxix....) and inspects the structure of the first event in the data, printing information about the the content of the event. Args: source (str): a psana source string (e.g. exp=CXI/cxix....). """ import psana def my_import(name): mod = __import__(name) components = name.split('.') for comp in components[1:]: mod = getattr(mod, comp) return mod def my_psana_from_string(name): components = name.split('.') mod = __import__(components[0]) for comp in components[1:]: mod = getattr(mod, comp) return mod print('\n\n') print('data source : %s' % source) print('\n') print('Opening dataset...') ds = psana.DataSource(source) print('\n') print('Epics pv Names (the confusing ones):') print(ds.env().epicsStore().pvNames()) print('\n') print('Epics aliases (the not so confusing ones):') print(ds.env().epicsStore().aliases()) print('\n') print('Event structure:') itr = ds.events() evt = itr.next() for k in evt.keys(): print('Type: %s Source: %s Alias: %s Key: %s') % ( k.type(), k.src(), k.alias(), k.key()) print('\n') for k in evt.keys(): print(k) beam = evt.get(psana.Bld.BldDataEBeamV7, psana.Source('BldInfo(EBeam)')) print('Photon energy: %f' % beam.ebeamPhotonEnergy())
def print_evrs(run_num): ds = psana.DataSource('exp=cxif7214:run=39') for evt in ds.events(): print[ e.eventCode() for e in evt.get( psana.EvrData.DataV3, psana.Source('DetInfo(NoDetector.0:Evr.0)')).fifoEvents() ] return
def data(self, evt): dl={} try: l3t = evt.get(psana.L3T.DataV2,psana.Source('')) if l3t is not None: dl['accept']=l3t.accept() except: pass return dl
def pump_laser_state_dataext(event): evr = event['evt'].get(psana.EvrData.DataV4, psana.Source('DetInfo(NoDetector.0:Evr.1)')) if not evr: evr = event['evt'].get(psana.EvrData.DataV4, psana.Source('DetInfo(NoDetector.0:Evr.2)')) if not evr.present(163): if evr.present( gp.monitor_params['RadialAveraging']['pump_laser_evr_code']): #pump laser on return 1 else: #pump laser off return 0 else: #dropped shot return 2
def psana_source(env, srcpar): """returns psana.Source(src) from other psana.Source brief src or alias. Parameters - srcpar : str - regular source or its alias, ex.: 'XppEndstation.0:Rayonix.0' or 'rayonix' - set_sub : bool - default=True - propagates source parameter to low level package """ #print 'type of srcpar: ', type(srcpar) src = srcpar if isinstance(srcpar, psana.Source) else psana.Source(srcpar) str_src = string_from_source(src) amap = env.aliasMap() psasrc = amap.src(str_src) source = src if amap.alias(psasrc) == '' else amap.src(str_src) if not isinstance(source, psana.Source): source = psana.Source(source) return source
def get_source(source_string, verbose=False): if verbose: print 'In "simplepsana.get_source()" function.' if psana is None: print 'ERROR: Function "simplepsana.getSoutrce" cannot be used without psana.' sys.exit() global _sourceDict if source_string not in _sourceDict: _sourceDict[source_string] = psana.Source(source_string) return _sourceDict[source_string]
def __init__(self, detname, name=None, saveTime=False): if name is None: self.name = detname else: self.name = name self.saveTime = saveTime defaultDetector.__init__(self, detname, self.name) cfg = self.det.env.configStore().get(psana.Generic1D.ConfigV0, psana.Source(detname)) self.wave8_shape = None if cfg is not None: self.wave8_shape = cfg.Length()
def __init__(self, detname, name=None, saveTime=False): if name is None: self.name = detname else: self.name = name self.saveTime = saveTime defaultDetector.__init__(self, detname, self.name) cfg = self.det.env.configStore().get(psana.Imp.ConfigV1, psana.Source(detname)) self.imp_shape = None if cfg is not None: self.imp_shape = cfg.numberOfSamples()
def _retrieve_cspad_evt_data(self, evt): """ Retrieve intensities collected on CsPad from input event. :param evt: psana event object :return data: list of quads that make up measured data """ data = list() cspad = evt.get(psana.CsPad.DataV2, psana.Source('DscCsPad')) for num in range(cspad.quads_shape()[0]): data.append(cspad.quads(num).data()) return data
def __init__(self, params): self.src = psana.Source(params.spectra_filter.detector_address) self.roi = params.spectra_filter.roi self.bg_roi = params.spectra_filter.background_roi if params.spectra_filter.background_path is None: self.dark_pickle = None else: self.dark_pickle = easy_pickle.load( params.spectra_filter.background_path)['DATA'].as_numpy_array( ) self.peak_range = params.spectra_filter.peak_range self.params = params
def __init__(self, image_file, **kwargs): super().__init__(image_file, locator_scope=rayonix_locator_scope, **kwargs) cfgs = self._ds.env().configStore() rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source("Rayonix")) if self.params.rayonix.bin_size is None: assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s() bin_size = rayonix_cfg.binning_f() else: bin_size = self.params.rayonix.bin_size self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(bin_size) self._image_size = rayonix_tbx.get_rayonix_detector_dimensions(self._ds.env())
def __init__(self, config, quiet=True): self._evt = None # EBeam self._EBeamSource = psana.Source('BldInfo(EBeam)') self._EBeamType = None self._phaseCavitySource = psana.Source('BldInfo(PhaseCavity)') # Evr self._EvrSource = psana.Source('DetInfo(NoDetector.0:Evr.0)') self._EvrType = psana.EvrData.DataV3 # photon beam #photonCalib = loadConfig(calibFile, quiet=quiet) #self._photonA = photonCalib.A #self._photonB = photonCalib.B self._photonA = config.lcls_photonEnergyA self._photonB = config.lcls_photonEnergyB self._feeSource = psana.Source('BldInfo(FEEGasDetEnergy)') self._idData = None
def test_detector_full_name(): ds = psana.DataSource( '/reg/g/psdm/detector/data_test/types/0007-NoDetector.0-Epix100a.0.xtc' ) env = ds.env() print 20 * '_', '\n%s:' % sys._getframe().f_code.co_name print 'src="Epix" :', detector_full_name( env, "Epix") print 'src=psana.Source("Epix")) :', detector_full_name( env, psana.Source('Epix')) print 'src="DetInfo(NoDetector.0:Epix100a.0)":', detector_full_name( env, 'DetInfo(NoDetector.0:Epix100a.0)') print 'for alias src="cs140_0" :', detector_full_name( env, 'cs140_0')
def __init__(self, params, run, detector, **kwargs): from xfel.cxi.cspad_ana import rayonix_tbx import psana FormatXTCSingleEvent.__init__(self, params, run, detector, **kwargs) self._cached_detector = {} cfgs = run.env().configStore() rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source("Rayonix")) if params.rayonix.bin_size is None: assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s() bin_size = rayonix_cfg.binning_f() else: bin_size = params.rayonix.bin_size self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(bin_size) self._image_size = rayonix_tbx.get_rayonix_detector_dimensions( run.env())
def getEventID(evt): try: # timestamp evtid = evt.get(psana.EventId) seconds = evtid.time()[0] nanoseconds = evtid.time()[1] fiducials = evtid.fiducials() # photon energy ebeam = evt.get(psana.Bld.BldDataEBeamV7, psana.Source('BldInfo(EBeam)')) photonEnergy = ebeam.ebeamPhotonEnergy() return seconds, nanoseconds, fiducials, photonEnergy except: pass return None, None, None, None
def __init__(self, image_file, **kwargs): super(FormatXTCRayonix, self).__init__( image_file, locator_scope=rayonix_locator_scope, **kwargs ) self._ds = FormatXTCRayonix._get_datasource(image_file, self.params) self._env = self._ds.env() self.populate_events() self.n_images = len(self.times) cfgs = self._ds.env().configStore() rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source("Rayonix")) if self.params.rayonix.bin_size is None: assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s() bin_size = rayonix_cfg.binning_f() else: bin_size = self.params.rayonix.bin_size self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(bin_size) self._image_size = rayonix_tbx.get_rayonix_detector_dimensions(self._ds.env())
def main(): args = parse_args() if args.run == 0: print 'Analyzing data from shared memory...' try: ds = psana.DataSource('shmem=1_1_psana_XCS.0:stop=no') except: raise IOError('Cannot find shared memory stream.') else: print 'Analyzing run: %d' % args.run ds = psana.DataSource('exp=sxra8513:run=%d' % args.run) # CHANGE THIS FOR NEW EXPT # this may also need to change for the new expt opal_src = psana.Source('DetInfo(SxrEndstation.0:Opal1000.1)') # iterate over events and extract peaks for i, evt in enumerate(ds.events()): # gets an "opal" object opal = evt.get(psana.Camera.FrameV1, opal_src) if opal: image = opal.data16().copy( ) # this is a numpy array of the opal image centers, widths = find_blobs(image, sigma_threshold=args.sigma) n_blobs = len(centers) print 'Shot %d :: found %d blobs :: %s' % (i, n_blobs, str(centers)) if args.view and (n_blobs > 0): draw_blobs(image, centers, widths) # if we reach the max number of shots, stop if i + 1 == args.num_max: print 'Reached max requested number of shots (%d)' % (i + 1, ) break return
def __init__(self, image_file, **kwargs): import psana assert (self.understand(image_file)) FormatXTC.__init__(self, image_file, locator_scope=rayonix_locator_scope, **kwargs) self._ds = FormatXTCRayonix._get_datasource(image_file, self.params) self._env = self._ds.env() self.populate_events() self.n_images = len(self.times) cfgs = self._ds.env().configStore() rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source('Rayonix')) if self.params.rayonix.bin_size is None: assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s() self._bin_size = rayonix_cfg.binning_f() else: self._bin_size = self.params.rayonix.bin_size self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(self._bin_size) self._image_size = rayonix_cfg.width(), rayonix_cfg.height()
def updatePhotonEnergy(self, arg): if arg == 'lcls': self.parent.ebeam = self.parent.evt.get( psana.Bld.BldDataEBeamV7, psana.Source('BldInfo(EBeam)')) if self.parent.ebeam: self.parent.photonEnergy = self.parent.ebeam.ebeamPhotonEnergy( ) else: try: wavelength = self.parent.epics.value( 'SIOC:SYS0:ML00:AO192') # nanometre h = 6.626070e-34 # J.m c = 2.99792458e8 # m/s joulesPerEv = 1.602176621e-19 # J/eV self.parent.photonEnergy = (h / joulesPerEv * c) / (wavelength * 1e-9) except: self.parent.photonEnergy = 0.0 self.parent.geom.p1.param( self.parent.geom.geom_grp, self.parent.geom.geom_photonEnergy_str).setValue( self.parent.photonEnergy)
else: cspad_data_shape = None # print(cspad_data_shape) comm.Barrier() cspad_data_shape = comm.bcast(cspad_data_shape, root=0) #### ds = psana.MPIDataSource('exp=' + expname + ':run=%s:smd'%run_num) # psana.DataSource('dir=/reg/d/ffb/xpp/xpph6615/xtc/:exp=xpph6615:run=%s:idx'%run) epics = ds.env().epicsStore() configStore = ds.env().configStore() total_events = 0 cspadDet = psana.Detector('jungfrau1M',ds.env()) ipm2_src = psana.Source('BldInfo(XPP-SB2-BMMON)') ipm3_src = psana.Source('BldInfo(XPP-SB3-BMMON)') evr_src = psana.Source("DetInfo(NoDetector.0:Evr.0)") def mpi_message(msg): if rank == 0: print(msg) """ Filters by skipping events that are outside of the desired range. Comment out what you do not want. """ def filter_events(evts): skipctr = 0 count = 0 for ev in evts: count += 1
def run(self): """ Process all images assigned to this thread """ params, options = self.parser.parse_args(show_diff_phil=True) if params.input.experiment is None or \ params.input.run_num is None or \ params.input.address is None: raise Usage(self.usage) if params.format.file_format == "cbf": if params.format.cbf.detz_offset is None: raise Usage(self.usage) elif params.format.file_format == "pickle": if params.format.pickle.cfg is None: raise Usage(self.usage) else: raise Usage(self.usage) if not os.path.exists(params.output.output_dir): raise Sorry("Output path not found:" + params.output.output_dir) # Save the paramters self.params = params self.options = options from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank( ) # each process in MPI has a unique id, 0-indexed size = comm.Get_size() # size: number of processes running in this job # set up psana if params.format.file_format == "pickle": psana.setConfigFile(params.format.pickle.cfg) dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment, params.input.run_num) ds = psana.DataSource(dataset_name) if params.format.file_format == "cbf": src = psana.Source('DetInfo(%s)' % params.input.address) psana_det = psana.Detector(params.input.address, ds.env()) # set this to sys.maxint to analyze all events if params.dispatch.max_events is None: max_events = sys.maxint else: max_events = params.dispatch.max_events for run in ds.runs(): if params.format.file_format == "cbf": # load a header only cspad cbf from the slac metrology base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology( run, params.input.address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d" % run.run()) # list of all events times = run.times() nevents = min(len(times), max_events) # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes mytimes = [ times[i] for i in xrange(nevents) if (i + rank) % size == 0 ] for i in xrange(len(mytimes)): evt = run.event(mytimes[i]) id = evt.get(psana.EventId) print "Event #", i, " has id:", id timestamp = cspad_tbx.evt_timestamp( cspad_tbx.evt_time(evt)) # human readable format if timestamp is None: print "No timestamp, skipping shot" continue t = timestamp s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[ 17:19] + t[20:23] print "Processing shot", s if params.format.file_format == "pickle": if evt.get("skip_event"): print "Skipping event", id continue # the data needs to have already been processed and put into the event by psana data = evt.get(params.format.pickle.out_key) if data is None: print "No data" continue # set output paths according to the templates path = os.path.join(params.output.output_dir, "shot-" + s + ".pickle") print "Saving", path easy_pickle.dump(path, data) elif params.format.file_format == "cbf": # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data( psana_det, evt, use_default=False, dark=True, common_mode=None, apply_gain_mask=params.format.cbf.gain_mask_value is not None, gain_mask_value=params.format.cbf.gain_mask_value, per_pixel_gain=False) distance = cspad_tbx.env_distance( params.input.address, run.env(), params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" continue if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" continue else: wavelength = 12398.4187 / self.params.format.cbf.override_energy # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data( base_dxtbx, data, distance, wavelength, timestamp, params.input.address) path = os.path.join(params.output.output_dir, "shot-" + s + ".cbf") print "Saving", path # write the file import pycbf cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0) run.end() ds.end()
f = h5py.File('/reg/neh/home2/tjlane/analysis/xppb0114/averages/r262_laser_avg.h5') vacuum = (np.array(f['/laser_on']) + np.array(f['/laser_off'])) / 2.0 f.close() # Define experiment, run. Shared memory syntax (for SXR): shmem=0_1_SXR.0:stop=no if args.run >= 0: print "\nReading run #: %d" % args.run ds = psana.DataSource('exp=xppb0114:run=%d' % args.run) elif args.run == -1: print "\nLocking on to shared memory..." ds = psana.DataSource('shmem=1_1_psana_XPP.0:stop=no') else: raise ValueError('`run` parameter must either be an int corresponding to a ' 'run, or the keyword "online"') cspad_src = psana.Source('DetInfo(XppGon.0:Cspad.0)') evr_src = psana.Source('DetInfo(NoDetector.0:Evr.0)') update_frequency = 100 # ------ END CONFIG --------- epics = ds.env().epicsStore() calib = ds.env().calibStore() laser_on = np.zeros((32, 185, 194*2)) laser_off = np.zeros((32, 185, 194*2)) shot_index = 0
def run(args): user_phil = [] for arg in args: if os.path.isfile(arg): try: user_phil.append(parse(file_name=arg)) except Exception as e: print(str(e)) raise Sorry("Couldn't parse phil file %s" % arg) else: try: user_phil.append(parse(arg)) except Exception as e: print(str(e)) raise Sorry("Couldn't parse argument %s" % arg) params = phil_scope.fetch(sources=user_phil).extract() # cxid9114, source fee: FeeHxSpectrometer.0:Opal1000.1, downstream: CxiDg3.0:Opal1000.0 # cxig3614, source fee: FeeHxSpectrometer.0:OrcaFl40.0 src = psana.Source(params.spectra_filter.detector_address) dataset_name = "exp=%s:run=%s:idx" % (params.experiment, params.runs) print("Dataset string:", dataset_name) ds = psana.DataSource(dataset_name) spf = spectra_filter(params) if params.selected_filter == None: filter_name = params.spectra_filter.filter[0].name else: filter_name = params.selected_filter rank = 0 size = 1 max_events = sys.maxsize for run in ds.runs(): print("starting run", run.run()) # list of all events times = run.times() if params.skip_events is not None: times = times[params.skip_events:] nevents = min(len(times), max_events) # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes mytimes = [times[i] for i in range(nevents) if (i + rank) % size == 0] for i, t in enumerate(mytimes): evt = run.event(t) accepted, data, spectrum, dc_offset, all_data, all_data_raw = spf.filter_event( evt, filter_name) if not accepted: continue print(cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)), "Publishing data for event", i) #header = "Event %d, m/f: %7.7f, f: %d"%(i, peak_max/flux, flux) header = "Event %d" % (i) if rank == 0: fee = Image(header, "FEE", data) # make a 2D plot publish.send("FEE", fee) # send to the display spectrumplot = XYPlot(header, 'summed 1D trace', list(range(data.shape[1])), spectrum) # make a 1D plot publish.send("SPECTRUM", spectrumplot) # send to the display fee = Image(header, "DC_OFFSET", dc_offset) # make a 2D plot publish.send("DC_OFFSET", fee) # send to the display fee = Image(header, "ALL_FEE", all_data) # make a 2D plot publish.send("ALL_FEE", fee) # send to the display fee = Image(header, "ALL_FEE_RAW", all_data_raw) # make a 2D plot publish.send("ALL_FEE_RAW", fee) # send to the display
def average(argv=None): if argv == None: argv = sys.argv[1:] try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") command_line = (libtbx.option_parser.option_parser(usage=""" %s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath] To write image pickles use -p, otherwise the program writes CSPAD CBFs. Writing CBFs requires the geometry to be already deployed. Examples: cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 Use one process on the current node to process all the events from run 25 of experiment cxi49812, using a detz_offset of 571. mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 As above, using 16 cores on the current node. bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812 As above, using the psanaq and 100 cores, putting the log in average.out and the output images in the folder cxi49812. """ % libtbx.env.dispatcher_name).option( None, "--as_pickle", "-p", action="store_true", default=False, dest="as_pickle", help="Write results as image pickle files instead of cbf files" ).option( None, "--raw_data", "-R", action="store_true", default=False, dest="raw_data", help= "Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)" ).option( None, "--background_pickle", "-B", default=None, dest="background_pickle", help="" ).option( None, "--config", "-c", type="string", default=None, dest="config", metavar="PATH", help="psana config file" ).option( None, "--experiment", "-x", type="string", default=None, dest="experiment", help="experiment name (eg cxi84914)" ).option( None, "--run", "-r", type="int", default=None, dest="run", help="run number" ).option( None, "--address", "-a", type="string", default="CxiDs2.0:Cspad.0", dest="address", help="detector address name (eg CxiDs2.0:Cspad.0)" ).option( None, "--detz_offset", "-d", type="float", default=None, dest="detz_offset", help= "offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)" ).option( None, "--outputdir", "-o", type="string", default=".", dest="outputdir", metavar="PATH", help="Optional path to output directory for output files" ).option( None, "--averagebase", "-A", type="string", default="{experiment!l}_avg-r{run:04d}", dest="averagepath", metavar="PATH", help= "Path to output average image without extension. String substitution allowed" ).option( None, "--stddevbase", "-S", type="string", default="{experiment!l}_stddev-r{run:04d}", dest="stddevpath", metavar="PATH", help= "Path to output standard deviation image without extension. String substitution allowed" ).option( None, "--maxbase", "-M", type="string", default="{experiment!l}_max-r{run:04d}", dest="maxpath", metavar="PATH", help= "Path to output maximum projection image without extension. String substitution allowed" ).option( None, "--numevents", "-n", type="int", default=None, dest="numevents", help="Maximum number of events to process. Default: all" ).option( None, "--skipevents", "-s", type="int", default=0, dest="skipevents", help="Number of events in the beginning of the run to skip. Default: 0" ).option( None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress" ).option( None, "--pickle-optical-metrology", "-m", action="store_true", default=False, dest="pickle_optical_metrology", help= "If writing pickle files, use the optical metrology in the experiment's calib directory" ).option( None, "--bin_size", "-b", type="int", default=None, dest="bin_size", help="Rayonix detector bin size" ).option( None, "--override_beam_x", "-X", type="float", default=None, dest="override_beam_x", help="Rayonix detector beam center x coordinate" ).option( None, "--override_beam_y", "-Y", type="float", default=None, dest="override_beam_y", help="Rayonix detector beam center y coordinate" ).option( None, "--calib_dir", "-C", type="string", default=None, dest="calib_dir", metavar="PATH", help="calibration directory" ).option( None, "--pickle_calib_dir", "-P", type="string", default=None, dest="pickle_calib_dir", metavar="PATH", help= "pickle calibration directory specification. Replaces --calib_dir functionality." ).option( None, "--xtc_dir", "-D", type="string", default=None, dest="xtc_dir", metavar="PATH", help="xtc stream directory" ).option( None, "--use_ffb", "-f", action="store_true", default=False, dest="use_ffb", help= "Use the fast feedback filesystem at LCLS. Only for the active experiment!" ).option( None, "--gain_mask_value", "-g", type="float", default=None, dest="gain_mask_value", help= "Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode." ).option( None, "--min", None, action="store_true", default=False, dest="do_minimum_projection", help="Output a minimum projection" ).option( None, "--minpath", None, type="string", default="{experiment!l}_min-r{run:04d}", dest="minpath", metavar="PATH", help= "Path to output minimum image without extension. String substitution allowed" )).process(args=argv) if len(command_line.args) > 0 or \ command_line.options.as_pickle is None or \ command_line.options.experiment is None or \ command_line.options.run is None or \ command_line.options.address is None or \ command_line.options.detz_offset is None or \ command_line.options.averagepath is None or \ command_line.options.stddevpath is None or \ command_line.options.maxpath is None or \ command_line.options.pickle_optical_metrology is None: command_line.parser.show_help() return # set this to sys.maxint to analyze all events if command_line.options.numevents is None: maxevents = sys.maxsize else: maxevents = command_line.options.numevents comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if command_line.options.config is not None: psana.setConfigFile(command_line.options.config) dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment, command_line.options.run) if command_line.options.xtc_dir is not None: if command_line.options.use_ffb: raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system") dataset_name += ":dir=%s" % command_line.options.xtc_dir elif command_line.options.use_ffb: # as ffb is only at SLAC, ok to hardcode /reg/d here dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % ( command_line.options.experiment[0:3], command_line.options.experiment) if command_line.options.calib_dir is not None: psana.setOption('psana.calib-dir', command_line.options.calib_dir) ds = psana.DataSource(dataset_name) address = command_line.options.address src = psana.Source('DetInfo(%s)' % address) nevent = np.array([0.]) if command_line.options.background_pickle is not None: background = easy_pickle.load( command_line.options.background_pickle)['DATA'].as_numpy_array() for run in ds.runs(): runnumber = run.run() if not command_line.options.as_pickle: psana_det = psana.Detector(address, ds.env()) # list of all events if command_line.options.skipevents > 0: print("Skipping first %d events" % command_line.options.skipevents) elif "Rayonix" in command_line.options.address: print("Skipping first image in the Rayonix detector" ) # Shuttering issue command_line.options.skipevents = 1 for i, evt in enumerate(run.events()): if i % size != rank: continue if i < command_line.options.skipevents: continue if i >= maxevents: break if i % 10 == 0: print('Rank', rank, 'processing event', i) #print "Event #",rank*mylength+i," has id:",evt.get(EventId) if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: data = evt.get(psana.Camera.FrameV1, src) if data is None: print("No data") continue data = data.data16().astype(np.float64) elif command_line.options.as_pickle: data = evt.get(psana.ndarray_float64_3, src, 'image0') else: # get numpy array, 32x185x388 from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data if command_line.options.raw_data: data = get_psana_corrected_data(psana_det, evt, use_default=False, dark=False, common_mode=None, apply_gain_mask=False, per_pixel_gain=False) else: if command_line.options.gain_mask_value is None: data = get_psana_corrected_data(psana_det, evt, use_default=True) else: data = get_psana_corrected_data( psana_det, evt, use_default=False, dark=True, common_mode=None, apply_gain_mask=True, gain_mask_value=command_line.options. gain_mask_value, per_pixel_gain=False) if data is None: print("No data") continue if command_line.options.background_pickle is not None: data -= background if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: distance = np.array([0.0]) wavelength = np.array([1.0]) else: d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset) if d is None: print("No distance, using distance", command_line.options.detz_offset) assert command_line.options.detz_offset is not None if 'distance' not in locals(): distance = np.array([command_line.options.detz_offset]) else: distance += command_line.options.detz_offset else: if 'distance' in locals(): distance += d else: distance = np.array([float(d)]) w = cspad_tbx.evt_wavelength(evt) if w is None: print("No wavelength") if 'wavelength' not in locals(): wavelength = np.array([1.0]) else: if 'wavelength' in locals(): wavelength += w else: wavelength = np.array([w]) t = cspad_tbx.evt_time(evt) if t is None: print("No timestamp, skipping shot") continue if 'timestamp' in locals(): timestamp += t[0] + (t[1] / 1000) else: timestamp = np.array([t[0] + (t[1] / 1000)]) if 'sum' in locals(): sum += data else: sum = np.array(data, copy=True) if 'sumsq' in locals(): sumsq += data * data else: sumsq = data * data if 'maximum' in locals(): maximum = np.maximum(maximum, data) else: maximum = np.array(data, copy=True) if command_line.options.do_minimum_projection: if 'minimum' in locals(): minimum = np.minimum(minimum, data) else: minimum = np.array(data, copy=True) nevent += 1 #sum the images across mpi cores if size > 1: print("Synchronizing rank", rank) totevent = np.zeros(nevent.shape) comm.Reduce(nevent, totevent) if rank == 0 and totevent[0] == 0: raise Sorry("No events found in the run") sumall = np.zeros(sum.shape).astype(sum.dtype) comm.Reduce(sum, sumall) sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype) comm.Reduce(sumsq, sumsqall) maxall = np.zeros(maximum.shape).astype(maximum.dtype) comm.Reduce(maximum, maxall, op=MPI.MAX) if command_line.options.do_minimum_projection: minall = np.zeros(maximum.shape).astype(minimum.dtype) comm.Reduce(minimum, minall, op=MPI.MIN) waveall = np.zeros(wavelength.shape).astype(wavelength.dtype) comm.Reduce(wavelength, waveall) distall = np.zeros(distance.shape).astype(distance.dtype) comm.Reduce(distance, distall) timeall = np.zeros(timestamp.shape).astype(timestamp.dtype) comm.Reduce(timestamp, timeall) if rank == 0: if size > 1: print("Synchronized") # Accumulating floating-point numbers introduces errors, # which may cause negative variances. Since a two-pass # approach is unacceptable, the standard deviation is # clamped at zero. mean = sumall / float(totevent[0]) variance = (sumsqall / float(totevent[0])) - (mean**2) variance[variance < 0] = 0 stddev = np.sqrt(variance) wavelength = waveall[0] / totevent[0] distance = distall[0] / totevent[0] pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value timestamp = timeall[0] / totevent[0] timestamp = (int(timestamp), timestamp % int(timestamp) * 1000) timestamp = cspad_tbx.evt_timestamp(timestamp) if command_line.options.as_pickle: extension = ".pickle" else: extension = ".cbf" dest_paths = [ cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env()) ] if command_line.options.do_minimum_projection: dest_paths.append( cspad_tbx.pathsubst(command_line.options.minpath + extension, evt, ds.env())) dest_paths = [ os.path.join(command_line.options.outputdir, path) for path in dest_paths ] if 'Rayonix' in command_line.options.address: all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) from xfel.cxi.cspad_ana import rayonix_tbx pixel_size = rayonix_tbx.get_rayonix_pixel_size( command_line.options.bin_size) beam_center = [ command_line.options.override_beam_x, command_line.options.override_beam_y ] active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]]) split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] for data, path in zip(all_data, dest_paths): print("Saving", path) d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=flex.double(data), distance=distance, pixel_size=pixel_size, saturated_value=rayonix_tbx.rayonix_saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: all_data = [mean, stddev, maxall] split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): d = cspad_tbx.dpack(address=old_style_address, data=flex.double(data), distance=distance, pixel_size=0.1, timestamp=timestamp, wavelength=wavelength) print("Saving", path) easy_pickle.dump(path, d) elif command_line.options.as_pickle: split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] xpp = 'xpp' in address.lower() if xpp: evt_time = cspad_tbx.evt_time( evt) # tuple of seconds, milliseconds timestamp = cspad_tbx.evt_timestamp( evt_time) # human readable format from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version( old_style_address, reverse_timestamp(timestamp)[0]) assert version_lookup is not None active_areas = xpp_active_areas[version_lookup]['active_areas'] beam_center = [1765 // 2, 1765 // 2] else: if command_line.options.pickle_calib_dir is not None: metro_path = command_line.options.pickle_calib_dir elif command_line.options.pickle_optical_metrology: from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path metro_path = get_calib_file_path(run.env(), address, run) else: metro_path = libtbx.env.find_in_repositories( "xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0") sections = parse_calib.calib2sections(metro_path) beam_center, active_areas = cspad_tbx.cbcaa( cspad_tbx.getConfig(address, ds.env()), sections) class fake_quad(object): def __init__(self, q, d): self.q = q self.d = d def quad(self): return self.q def data(self): return self.d if xpp: quads = [ fake_quad(i, mean[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [ fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [ fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [ fake_quad(i, minall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] minall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) minall = flex.double(minall.astype(np.float64)) else: quads = [ fake_quad(i, mean[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] mean = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [ fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] stddev = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [ fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] maxall = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [ fake_quad(i, minall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] minall = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) minall = flex.double(minall.astype(np.float64)) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print("Saving", path) d = cspad_tbx.dpack(active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=data, distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) else: # load a header only cspad cbf from the slac metrology from xfel.cftbx.detector import cspad_cbf_tbx import pycbf base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology( run, address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d" % run.run()) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print("Saving", path) cspad_img = cspad_cbf_tbx.format_object_from_data( base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=False) cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)