def _getCalibrationValues(nev, evt, xtcavroipars, xtcavcalibpars): """ Internal method. Sets calibration parameters for image processing Returns: roi: region of interest in image global_calibration: global parameters of xtcav machine saturation_value: value at which image is saturated and no longer valid first_image: index of first valid shot in run """ roi_xtcav = xtup.getXTCAVImageROI(xtcavroipars, evt) #logger.debug('roi_xtcav: %s' % str(roi_xtcav)) global_calibration = xtup.getGlobalXTCAVCalibration( xtcavcalibpars, evt) #logger.debug('global_calibration: %s' % str(global_calibration)) saturation_value = xtup.getCameraSaturationValue(xtcavcalibpars, evt) #logger.debug('saturation_value: %s' % str(saturation_value)) logger.info('Event %2d CalibrationValues: roi_xtcav: %s\t global_calibration: %s\t saturation_value: %s'%\ (nev, str(roi_xtcav), str(global_calibration), str(saturation_value))) resp = (roi_xtcav, global_calibration, saturation_value) return None if None in resp else resp
def setDetectors(run, camera=None, ebeam=None, gasdetector=None, eventid=None, xtcavpars=None): """ sets detector and data objects """ o = Empty() o._camera = camera if camera is not None else run.Detector( cons.DETNAME) # 'xtcav' o._ebeam = ebeam if ebeam is not None else run.Detector( cons.EBEAM) # 'ebeam' o._gasdetector = gasdetector if gasdetector is not None else run.Detector( cons.GAS_DETECTOR) # 'gasdetector' o._eventid = eventid if eventid is not None else run.Detector( cons.EVENTID) # 'eventid' o._xtcavpars = xtcavpars if xtcavpars is not None else run.Detector( cons.XTCAVPARS) # 'xtcavpars' o._camraw = xtup.get_attribute(o._camera, 'raw') o._valsebm = xtup.get_attribute(o._ebeam, 'valsebm') o._valsgd = xtup.get_attribute(o._gasdetector, 'valsgd') o._valseid = xtup.get_attribute(o._eventid, 'valseid') o._valsxtp = xtup.get_attribute(o._xtcavpars, 'valsxtp') if None in (o._camraw, o._valsebm, o._valsgd, o._valseid, o._valsxtp): sys.error( 'FATAL ERROR IN THE DETECTOR INTERFACE: MISSING ATTRIBUTE MUST BE IMPLEMENTED' ) return o
def _setCalibrations(self, evt): """ Method that sets the xtcav calibration values for a given run. """ # DONE in __init__ #if not self._camera: self._setDetectorDataObjects() #if not self._darkreference: self._loadDarkReference() #if not self._lasingoffreference: self._loadLasingOffReference() self._roixtcav = xtup.getXTCAVImageROI(self._valsxtp, evt) logger.debug('_roixtcav: %s' % str(self._roixtcav)) self._global_calibration = xtup.getGlobalXTCAVCalibration( self._valsxtp, evt) logger.debug('_global_calibration: %s' % str(self._global_calibration)) self._saturation_value = xtup.getCameraSaturationValue( self._valsxtp, evt) logger.debug('_saturation_value: %d' % self._saturation_value) if self._roixtcav and self._global_calibration and self._saturation_value: #Only reason to do this is to allow us to use same 'processImage' function #across lasing on/off shots self.parameters = LasingOnParameters(\ self.num_bunches, self.snr_filter,\ self.roi_expand, self.roi_fraction,\ self.island_split_method, self.island_split_par1,\ self.island_split_par2) self._calibrationsset = True
def data_xtcavpars(valsxtp, evt): """ ROI_SIZE_X_names = ['XTCAV_ROI_sizeX', 'ROI_X_Length', 'OTRS:DMP1:695:SizeX'] ROI_SIZE_Y_names = ['XTCAV_ROI_sizeY', 'ROI_Y_Length', 'OTRS:DMP1:695:SizeY'] ROI_START_X_names = ['XTCAV_ROI_startX', 'ROI_X_Offset', 'OTRS:DMP1:695:MinX'] ROI_START_Y_names = ['XTCAV_ROI_startY', 'ROI_Y_Offset', 'OTRS:DMP1:695:MinY'] UM_PER_PIX_names = ['XTCAV_calib_umPerPx','OTRS:DMP1:695:RESOLUTION'] STR_STRENGTH_names = ['XTCAV_strength_par_S','Streak_Strength','OTRS:DMP1:695:TCAL_X'] RF_AMP_CALIB_names = ['XTCAV_Amp_Des_calib_MV','XTCAV_Cal_Amp','SIOC:SYS0:ML01:AO214'] RF_PHASE_CALIB_names = ['XTCAV_Phas_Des_calib_deg','XTCAV_Cal_Phase','SIOC:SYS0:ML01:AO215'] DUMP_E_names = ['XTCAV_Beam_energy_dump_GeV','Dump_Energy','REFS:DMP1:400:EDES'] DUMP_DISP_names = ['XTCAV_calib_disp_posToEnergy','Dump_Disp','SIOC:SYS0:ML01:AO216'] o.XTCAV_calib_umPerPx = valsxtp.XTCAV_calib_umPerPx(evt) o.XTCAV_strength_par_S = valsxtp.XTCAV_strength_par_S(evt) ... a = valsxtp.getattr(valsxtp, 'OTRS:DMP1:695:SizeX', None) o.OTRS_DMP1_695_SizeX = None if a is None else a(evt) """ o = Empty() for lstname, names in cons.xtcav_varname.items(): for name in names: a = xtup.get_attribute(valsxtp, name) if a is not None: value = a(evt) o.setattr(name.replace(':', '_'), value) #convert name like 'UM_PER_PIX_names' to 'umperpix' varname = lstname.rstrip('_names').replace('_', '').lower() if value is not None: o.setattr(varname, value) return o
def _loadDarkReference(self): """ Loads the dark reference from file or DB. """ self._darkreference = None if self.dark_reference_path: self._darkreference = DarkBackgroundReference.load( self.dark_reference_path) logger.info('Using file ' + self.dark_reference_path.split('/')[-1] + ' for dark reference') if self._darkreference is None: #dark_data, dark_meta = self._camera.calibconst.get('xtcav_pedestals') dark_data, dark_meta = xtup.get_calibconst(self._camera, 'xtcav_pedestals', cons.DETNAME, self.run.expt, self.run.runnum) self._darkreference = xtu.xtcav_calib_object_from_dict(dark_data) logger.debug('==== dark_meta:\n%s' % str(dark_meta)) logger.debug('==== dir(_darkreference):\n%s' % str(dir(self._darkreference))) logger.debug('==== _darkreference.ROI:\n%s' % str(self._darkreference.ROI)) logger.debug( info_ndarr(self._darkreference.image, '==== darkreference.image:')) logger.info('Using dark reference from DB')
def _loadLasingOffReference(self): """ Loads the lasing off reference parameters from file or DB or set them to default. """ self._lasingoffreference = None # if self.lasingoff_ref_path: # self._lasingoffreference = LasingOffReference.load(self.lasingoff_ref_path) # logger.info('Using lasing off reference from file %s'%self.lasingoff_ref_path.split('/')[-1]) # self._setLasingOffReferenceParameters() # return # if self._lasingoffreference is None: #lofr_data, lofr_meta = self._camera.calibconst.get('xtcav_lasingoff') lofr_data, lofr_meta = xtup.get_calibconst(self._camera, 'xtcav_lasingoff', cons.DETNAME, self.run.expt, self.run.runnum) self._lasingoffreference = xtu.xtcav_calib_object_from_dict(lofr_data) logger.debug('==== lofr_meta:\n%s' % str(lofr_meta)) logger.debug('==== dir(_lasingoffreference):\n%s' % str(dir(self._lasingoffreference))) logger.debug('==== _lasingoffreference.parameters:\n%s' % str(self._lasingoffreference.parameters)) logger.debug('==== _lasingoffreference.averaged_profiles:\n%s' % str(self._lasingoffreference.averaged_profiles)) logger.info('Using lasing off reference from DB') self._setLasingOffReferenceParameters() if not self._lasingoffreference: logger.warning( 'Lasing off reference for run %d not found, using default values' % self._currentevent.run()) self._setDefaultProcessingParameters() return
def getLasingOffShot(lon, fname_loff): results=lon._pulse_characterization loff = lon._lasingoffreference ibunch = 0 group = results.groupnum[ibunch] profs = loff.averaged_profiles ds = DataSource(files=fname_loff) run = next(ds.runs()) print('XXXX dir(run)', dir(run)) print('XXXX max_events', run.max_events) print('XXXX esm', run.esm) print('XXXX timestamp', run.timestamp) print('XXXX dm', run.dm) print('XXXX smd_dm', run.smd_dm) #==================== sys.exit('TEST EXIT') #==================== camera = run.Detector(cons.DETNAME) camraw = xtup.get_attribute(camera,'raw') #LCLS1 stuff does not work here #times = run.times() #time = profs.eventTime[ibunch][group] #fid = profs.eventFid[ibunch][group] #et = EventTime(int(time),int(fid)) #evt_loff = run.event(et) if camraw is None: print('No lasing off image found for unixtime',time,'and fiducials',fid) print('Found lasing off shot in run',loff.parameters.run) return camraw(evt_loff)
def __init__(self, args): #self.args = args #fname = getattr(args, 'fname', '/reg/g/psdm/detector/data2_test/xtc/data-amox23616-r0104-e000400-xtcav-v2.xtc2') experiment = getattr(args, 'experiment', 'amox23616') run_number = getattr(args, 'run_number', 104) max_shots = getattr(args, 'max_shots', 400) validity_range = getattr(args, 'validity_range', None) save_to_file = getattr(args, 'save_to_file', True) calibration_path = getattr(args, 'calibration_path', '') start_image = getattr(args, 'start_image', 0) self.image = None self.ROI = None self.n = 0 self.parameters = DarkBackgroundParameters( experiment=experiment, max_shots=max_shots, run_number=run_number, validity_range=validity_range, calibration_path=calibration_path) """ After setting all the parameters, this method has to be called to generate the dark reference and save it in the proper location. """ logger.info('dark background reference') #logger.info('\t Data file: %s' % fname) logger.info('\t Experiment: %s' % self.parameters.experiment) logger.info('\t Run: %s' % self.parameters.run_number) logger.info('\t Valid shots to process: %d' % self.parameters.max_shots) logger.info('\t Detector name: %s' % cons.DETNAME) #Loading the dataset from the "dark" run, this way of working should be compatible with both xtc and hdf5 files #ds=DataSource(files=fname) ds = DataSource(exp=self.parameters.experiment, run=self.parameters.run_number) run = next(ds.runs()) logger.info('\t RunInfo expt: %s runnum: %d\n' % (run.expt, run.runnum)) #Camera and type for the xtcav images camera = run.Detector(cons.DETNAME) #ebeam = run.Detector(cons.EBEAM) #eventid = run.Detector(cons.EVENTID) #gasdetector = run.Detector(cons.GAS_DETECTOR) xtcavroipars = xtup.get_roi_parameters(run) #Stores for environment variables #configStore=dataSource.env().configStore() #epicsStore=dataSource.env().epicsStore() print('\n', 100 * '_', '\n') #camraw = xtup.get_attribute(camera, 'raw') #valsxtp = xtup.get_attribute(xtcavpars, 'valsxtp') #valsebm = xtup.get_attribute(ebeam, 'valsebm') #valseid = xtup.get_attribute(eventid, 'valseid') #valsgd = xtup.get_attribute(gasdetector, 'valsgd') #if camraw is None : # valsebm, eventid, valsgd) : # sys.error('FATAL ERROR IN THE DETECTOR INTERFACE: MISSING ATTRIBUTE MUST BE IMPLEMENTED') accumulator_xtcav = None n = 0 #Counter for the total number of xtcav images processed for nev, evt in enumerate(run.events()): #print('Event %03d'%nev, end='') img = camera.raw.value(evt) if img is None: continue if self.ROI is None: if not self._getImageROI(nev, evt, xtcavroipars): continue accumulator_xtcav = np.zeros((self.ROI.yN, self.ROI.xN), dtype=np.float64) accumulator_xtcav += img n += 1 if n % 5 == 0: sys.stdout.write('\r%.1f %% done, %d / %d' % (float(n) / self.parameters.max_shots * 100, n, self.parameters.max_shots)) sys.stdout.flush() if n >= self.parameters.max_shots: break #At the end of the program the total accumulator is saved sys.stdout.write('\nMaximum number of images processed\n') self.image = accumulator_xtcav / n if not self.parameters.validity_range or not type( self.parameters.validity_range) == tuple: self.parameters = self.parameters._replace( validity_range=(self.parameters.run_number, 9999)) #'end')) elif len(self.parameters.validity_range) == 1: self.parameters = self.parameters._replace( validity_range=(self.parameters.validity_range[0], 9999)) #'end')) logger.info(info_ndarr(self.image, 'averaged raw:')) logger.info('self.parameters: %s' % str(self.parameters)) if save_to_file: #cp = CalibrationPaths(dataSource.env(), self.parameters.calibration_path) #fname = cp.newCalFileName(cons.DB_FILE_NAME, self.parameters.validity_range[0], self.parameters.validity_range[1]) fname = 'cons-%s-%04d-xtcav-pedestals.data' % (run.expt, run.runnum ) #, cons.DETNAME) self.save(fname)
def _getImageROI(self, nev, evt, xtcavpars): self.ROI = xtup.getXTCAVImageROI(xtcavpars, evt) logger.info('\t Event %2d ImageROI: %s' % (nev, str(self.ROI))) return self.ROI is not None
def processEvent(self, evt): """ Args: evt (psana event): relevant event to retrieve information from Returns: True: All the input form detectors necessary for a good reconstruction are present in the event. False: The information from some detectors is missing for that event. It may still be possible to get information. """ self._currentevent = evt self._pulse_characterization = None self._image_profile = None self._processed_image = None if not self._calibrationsset: self._setCalibrations(evt) if not self._calibrationsset: logger.warning('CALIBRATION IS NOT SET YET..., try next event') return False #Obtain the shot to shot parameters necessary for the retrieval of the x and y axis in time and energy units shot_to_shot = xtup.getShotToShotParameters(evt, self._valsebm, self._valsgd, self._valseid) logger.debug('shot_to_shot: %s' % str(shot_to_shot)) if not shot_to_shot.valid: logger.warning('shot_to_shot info is not valid') return False self._rawimage = self._camraw(evt) logger.debug(info_ndarr(self._rawimage, 'camera raw:')) if self._rawimage is None: logger.warning('Could not retrieve image') return False self._image_profile, self._processed_image = xtu.processImage(\ self._rawimage,\ self.parameters,\ self._darkreference,\ self._global_calibration,\ self._saturation_value,\ self._roixtcav,\ shot_to_shot) logger.debug('After xtu.processImage: _image_profile:\n%s' % xtu.info_xtcav_object(self._image_profile)) logger.debug('After xtu.processImage: _processed_image:\n%s' % info_ndarr(self._processed_image)) if not self._image_profile: logger.warning('Cannot create image profile') return False if not self._lasingoffreference: logger.warning( 'Cannot perform analysis without lasing off reference') return False #Using all the available data, perform the retrieval for that given shot self._pulse_characterization = xtu.processLasingSingleShot( self._image_profile, self._lasingoffreference.averaged_profiles) logger.debug( 'After xtu.processLasingSingleShot: _pulse_characterization:\n%s', xtu.info_xtcav_object(self._pulse_characterization)) if not self._pulse_characterization: return False return True
def __init__(self, args): """ """ #self.args = args fname = getattr( args, 'fname', '/reg/g/psdm/detector/data2_test/xtc/data-amox23616-r0131-e000200-xtcav-v2.xtc2' ) experiment = getattr(args, 'experiment', 'amox23616') run_number = getattr(args, 'run_number', 131) max_shots = getattr(args, 'max_shots', 401) #Maximum number of shots to process validity_range = getattr(args, 'validity_range', None) save_to_file = getattr(args, 'save_to_file', True) calibration_path = getattr(args, 'calibration_path', '') start_image = getattr(args, 'start_image', 0) dark_reference_path = getattr(args, 'dark_reference_path', None) #Dark reference information num_bunches = getattr(args, 'num_bunches', 1) #Number of bunches num_groups = getattr(args, 'num_groups', None) #Number of profiles to average together snr_filter = getattr(args, 'snr_filter', 10) #Number of sigmas for the noise threshold roi_expand = getattr(args, 'roi_expand', 1) #Parameter for the roi location roi_fraction = getattr(args, 'roi_fraction', cons.ROI_PIXEL_FRACTION) island_split_method = getattr( args, 'island_split_method', cons.DEFAULT_SPLIT_METHOD) #Method for island splitting island_split_par1 = getattr( args, 'island_split_par1', 3.0 ) #Ratio between number of pixels between largest and second largest groups when calling scipy.label island_split_par2 = getattr( args, 'island_split_par2', 5. ) #Ratio between number of pixels between second/third largest groups when calling scipy.label PLOT_IMAGE = getattr(args, 'plot_image', False) if PLOT_IMAGE: self.fig, self.axim, self.axcb = gr.fig_img_cbar_axes(fig=None,\ win_axim=(0.05, 0.05, 0.87, 0.93),\ win_axcb=(0.923, 0.05, 0.02, 0.93)) #, **kwargs) #if type(run_number) == int: # run_number = str(run_number) self.parameters = LasingOffParameters( experiment=experiment, max_shots=max_shots, run_number=run_number, start_image=start_image, validity_range=validity_range, dark_reference_path=dark_reference_path, num_bunches=num_bunches, num_groups=num_groups, snr_filter=snr_filter, roi_expand=roi_expand, roi_fraction=roi_fraction, island_split_method=island_split_method, island_split_par2=island_split_par2, island_split_par1=island_split_par1, calibration_path=calibration_path, fname=fname, version=1) if rank == 0: print('Lasing off reference') print('\t File name: %s' % self.parameters.fname) print('\t Experiment: %s' % self.parameters.experiment) print('\t Runs: %s' % self.parameters.run_number) print('\t Number of bunches: %d' % self.parameters.num_bunches) print('\t Valid shots to process: %d' % self.parameters.max_shots) print('\t Dark reference run: %s' % self.parameters.dark_reference_path) #Loading the data, this way of working should be compatible with both xtc and hdf5 files #ds = psana.DataSource("exp=%s:run=%s:idx" % (self.parameters.experiment, self.parameters.run_number)) ds = DataSource(files=fname) run = next(ds.runs()) # run = ds.runs().next() #env = SimulatorEnvironment() # ds.env() #Camera for the xtcav images, Ebeam type, eventid, gas detectors camera = run.Detector(cons.DETNAME) # psana.Detector(cons.DETNAME) ebeam = run.Detector( cons.EBEAM) #SimulatorEBeam() # psana.Detector(cons.EBEAM) eventid = run.Detector( cons.EVENTID) #SimulatorEventId() # evt.get(psana.EventId) gasdetector = run.Detector( cons.GAS_DETECTOR ) #SimulatorGasDetector() # psana.Detector(cons.GAS_DETECTOR) xtcavpars = run.Detector(cons.XTCAVPARS) # Empty list for the statistics obtained from each image, the shot to shot properties, # and the ROI of each image (although this ROI is initially the same for each shot, # it becomes different when the image is cropped around the trace) list_image_profiles = [] #dark_background = self._getDarkBackground(env) dark_data, dark_meta = xtup.get_calibconst(camera, 'xtcav_pedestals', cons.DETNAME, experiment, run_number) logger.debug('==== dark_meta:\n%s' % str(dark_meta)) dark_background = xtu.xtcav_calib_object_from_dict(dark_data) logger.debug('==== dir(dark_background):\n%s' % str(dir(dark_background))) logger.debug('==== dark_background.ROI:\n%s' % str(dark_background.ROI)) logger.debug( info_ndarr(dark_background.image, '==== dark_background.image:')) print('\n', 100 * '_', '\n') camraw = xtup.get_attribute(camera, 'raw') valsxtp = xtup.get_attribute(xtcavpars, 'valsxtp') valsebm = xtup.get_attribute(ebeam, 'valsebm') valseid = xtup.get_attribute(eventid, 'valseid') valsgd = xtup.get_attribute(gasdetector, 'valsgd') if None in (camraw, valsxtp, valsebm, eventid, valsgd): sys.error( 'FATAL ERROR IN THE DETECTOR INTERFACE: MISSING ATTRIBUTE MUST BE IMPLEMENTED' ) #times = run.times() #image_numbers = xtup.divideImageTasks(first_event, len(times), rank, size) roi_xtcav, global_calibration, saturation_value = None, None, None num_processed = 0 #Counter for the total number of xtcav images processed within the run for nev, evt in enumerate(run.events()): #logger.info('Event %03d'%nev) img = camraw(evt) if img is None: continue if roi_xtcav is None: # get calibration values needed to process images. resp = self._getCalibrationValues(nev, evt, camraw, valsxtp) if resp is None: continue roi_xtcav, global_calibration, saturation_value = resp #Obtain the shot to shot parameters necessary for the retrieval of the x and y axis in time and energy units shot_to_shot = xtup.getShotToShotParameters( evt, valsebm, valsgd, valseid) #logger.debug('shot_to_shot: %s' % str(shot_to_shot)) if not shot_to_shot.valid: continue image_profile, _ = xtu.processImage(img, self.parameters, dark_background, global_calibration, saturation_value, roi_xtcav, shot_to_shot) #logger.debug(info_ndarr(image_profile, 'LasingOffReference image_profile')) if not image_profile: continue #Append only image profile, omit processed image list_image_profiles.append(image_profile) num_processed += 1 self._printProgressStatements(num_processed) if num_processed >= np.ceil( self.parameters.max_shots / float(size)): break if PLOT_IMAGE: nda = img mean, std = nda.mean(), nda.std() aran = (mean - 3 * std, mean + 5 * std) self.axim.clear() self.axcb.clear() imsh = gr.imshow(self.axim, nda, amp_range=aran, extent=None, interpolation='nearest',\ aspect='auto', origin='upper', orientation='horizontal', cmap='inferno') cbar = gr.colorbar(self.fig, imsh, self.axcb, orientation='vertical', amp_range=aran) gr.set_win_title(self.fig, 'Event: %d' % nev) gr.draw_fig(self.fig) gr.show(mode='non-hold') # here gather all shots in one core, add all lists #image_profiles = comm.gather(list_image_profiles, root=0) image_profiles = list_image_profiles if rank != 0: return sys.stdout.write('\n') # Flatten gathered arrays #image_profiles = [item for sublist in image_profiles for item in sublist] #for i,ipf in enumerate(image_profiles) : # print('XXX image_profiles %d:\n %s'%(i,str(ipf))) #Since there are 12 cores it is possible that there are more references than needed. In that case we discard some if len(image_profiles) > self.parameters.max_shots: image_profiles = image_profiles[0:self.parameters.max_shots] #At the end, all the reference profiles are converted to Physical units, grouped and averaged together averaged_profiles = xtu.averageXTCAVProfilesGroups( image_profiles, self.parameters.num_groups) self.averaged_profiles, num_groups = averaged_profiles self.n = num_processed self.parameters = self.parameters._replace(num_groups=num_groups) logger.debug('self.parameters.validity_range: %s type: %s' % (self.parameters.validity_range, type(self.parameters.validity_range))) logger.debug( 'self.parameters.run_number: %d type: %s' % (self.parameters.run_number, type(self.parameters.run_number))) # Set validity range, replace 'end' -> 9999 othervise save does not work... if not self.parameters.validity_range or not type( self.parameters.validity_range) == tuple: self.parameters = self.parameters._replace( validity_range=(self.parameters.run_number, 9999)) # IT WAS 'end')) elif len(self.parameters.validity_range) == 1: self.parameters = self.parameters._replace( validity_range=(self.parameters.validity_range[0], 9999)) # 'end')) #===================== #sys.exit('TEST EXIT') #===================== if save_to_file: #cp = CalibrationPaths(env, self.parameters.calibration_path) #file = cp.newCalFileName(cons.LOR_FILE_NAME, self.parameters.validity_range[0], self.parameters.validity_range[1]) fname = 'cons-%s-%04d-xtcav-lasingoff.data' % (run.expt, run.runnum ) # , cons.DETNAME) self.save(fname)
def _getImageROI(self, nev, evt, camraw, valsxtp): self.ROI = xtup.getXTCAVImageROI(valsxtp, evt) logger.info('\t Event %2d ImageROI: %s' % (nev, str(self.ROI))) return self.ROI is not None