def getNoe(args): runStr = "%04d" % args.run ds = psana.DataSource("exp=" + args.exp + ":run=" + runStr + ':idx') run = ds.runs().next() times = run.times() # check if the user requested specific number of events if args.noe == -1: numJobs = len(times) else: if args.noe <= len(times): numJobs = args.noe else: numJobs = len(times) return numJobs
def test_detector_full_name(): ds = psana.DataSource( '/reg/g/psdm/detector/data_test/types/0007-NoDetector.0-Epix100a.0.xtc' ) env = ds.env() print 20 * '_', '\n%s:' % sys._getframe().f_code.co_name print 'src="Epix" :', detector_full_name( env, "Epix") print 'src=psana.Source("Epix")) :', detector_full_name( env, psana.Source('Epix')) print 'src="DetInfo(NoDetector.0:Epix100a.0)":', detector_full_name( env, 'DetInfo(NoDetector.0:Epix100a.0)') print 'for alias src="cs140_0" :', detector_full_name( env, 'cs140_0')
def look_at_keys(): for run in range(282, 291): runstr = 'exp=diamcc14:run=%d' % run try: ds = psana.DataSource(runstr) except Exception, exp: print("unable to open %s" % runstr) print(exp) continue epics, data = count_data(ds, limit=0) print("========= %s ========" % runstr) print("-- epics --") pprint(dict(epics)) print("-- data --") pprint(dict(data))
def setupExperiment(self): access = 'exp=' + str(self.experimentName) + ':run=' + str(self.runNumber) + ':idx' if 'ffb' in self.access.lower(): access += ':dir=/reg/d/ffb/' + self.experimentName[:3] + \ '/' + self.experimentName + '/xtc' self.ds = psana.DataSource(access) self.run = self.ds.runs().next() self.times = self.run.times() self.eventTotal = len(self.times) self.env = self.ds.env() self.evt = self.run.event(self.times[0]) self.det = psana.Detector(str(self.detInfo), self.env) self.det.do_reshape_2d_to_3d(flag=True) self.getDetInfoList() self.detAlias = self.getDetectorAlias(str(self.detInfo)) self.updateClen() # Get epics variable, clen
def stack2image(stack=None, experimentName=None, runNumber=None, detInfo=None, eventInd=0): ds = psana.DataSource('exp=' + str(experimentName) + ':run=' + str(runNumber) + ':idx') run = ds.runs().next() times = run.times() env = ds.env() evt = run.event(times[0]) det = psana.Detector(str(detInfo), env) evt = run.event(times[eventInd]) image = det.image(evt, stack) return image
def issue_2022_06_17(): """Default common_mode parameters """ import psana import numpy as np expname = 'uedcom103' runnum = 419 ds = psana.DataSource(exp=expname,run=runnum,detectors=['epixquad','epicsinfo']) myrun = next(ds.runs()) det = myrun.Detector('epixquad') #print(det.calibconst.keys(),det.calibconst) print(det.calibconst.keys()) #cmpars, meta = det.calibconst['common_mode'] cmpars = det.raw._common_mode() print('cmpars',cmpars)
def _get_datasource(image_file, params): """ Construct a psana data source object given the locator parameters """ if params.calib_dir is not None: psana.setOption("psana.calib-dir", params.calib_dir) if params.data_source is None: if (params.experiment is None or params.run is None or params.mode is None or len(params.run) == 0): return False img = "exp=%s:run=%s:%s" % ( params.experiment, ",".join(["%d" % r for r in params.run]), params.mode, ) else: img = params.data_source return psana.DataSource(img)
def __init__(self, exp, run, detector_name, codes=None): """ exp: experiment string run: run number detector_name: detecrtor string """ assert (has_psana) self.run_str = run self.exp = exp self.codes = codes self.ds = psana.DataSource("exp=%s:run=%d:idx" % (exp, run)) self.run = self.ds.runs().next() self.times = self.run.times() self.N_events = self.N = len(self.times) self.event_info_string = "Start" self.detnames = [d for sl in psana.DetNames() for d in sl] self.env = self.ds.env() assert (detector_name in self.detnames) self.code_dets = [ psana.Detector(d, self.env) for d in self.detnames if d.startswith("evr") ] self.detector_name = detector_name self.Detector = psana.Detector(self.detector_name, self.env) self.gas_reader = psana.Detector("FEEGasDetEnergy", self.env) self.spectrometer = psana.Detector("FeeSpec-bin", self.env) # Get image shape.. I = None i = 0 while I is None: ev = self.run.event(self.times[i]) if ev is None: i += 1 continue I = self.Detector.image(ev) i += 1 self.img_sh = I.shape self.empty_img = np.zeros(self.img_sh) self.gain_map = self.Detector.gain_mask(run) > 1
def runmaster(args): exprun = "exp="+str(args.exp)+":run="+str(args.run) ds = psana.DataSource(exprun+':idx') run = ds.runs().next() det = psana.Detector(args.det) times = run.times() evt = None counter = 0 while evt is None: evt = run.event(times[counter]) counter += 1 max_img = None mean_img = None median_img = None ## counter is a monitor to record how many images have been process in total ## nClients is the number of clients to process the data counter = 0. nClients = comm_size-1 while nClients > 0: # Remove client if the run ended md = mpidata() md.recv() if max_img is None: max_img = md.max_img mean_img = md.mean_img median_img = md.median_img else: max_img = np.maximum(max_img, md.max_img) mean_img = (mean_img * counter + md.mean_img * md.small.counter)*1.0/(counter + md.small.counter) counter = counter + md.small.counter nClients -= 1 print "##### Total number of images: %s/%s " % (str(counter), str(args.noe)) params = experiparams(experimentName=args.exp, runNumber=args.run, outDir=os.path.realpath(args.outDir)) np.save(params.cxiDir+"/"+args.exp+"_"+str(args.run).zfill(4)+"_max_assem.npy", det.image(evt, max_img)) np.save(params.cxiDir+"/"+args.exp+"_"+str(args.run).zfill(4)+"_max.npy", max_img) np.save(params.cxiDir+"/"+args.exp+"_"+str(args.run).zfill(4)+"_mean_assem.npy", det.image(evt, mean_img/counter)) np.save(params.cxiDir+"/"+args.exp+"_"+str(args.run).zfill(4)+"_mean.npy", mean_img/counter) print "## saving the average powder pattern to: %s" % params.cxiDir
def getNoe(args): runStr = "%04d" % args.run access = "exp=" + args.exp + ":run=" + runStr + ':idx' if 'ffb' in args.access.lower(): access += ':dir=/reg/d/ffb/' + args.exp[:3] + '/' + args.exp + '/xtc' print "findPeaks: ", access ds = psana.DataSource(access) run = next(ds.runs()) times = run.times() # check if the user requested specific number of events if args.noe == -1: numJobs = len(times) else: if args.noe <= len(times): numJobs = args.noe else: numJobs = len(times) return numJobs
def psanaCenter(experimentName, runNumber, detectorName): """ load current center from psana """ ds = psana.DataSource("exp=" + experimentName + ":run=" + str(runNumber) + ':idx') run = ds.runs().next() det = psana.Detector(detectorName) times = run.times() evt = None counter = 0 while evt is None: evt = run.event(times[counter]) # guarantee we have a valid event counter += 1 psanaCx, psanaCy = det.point_indexes(evt, pxy_um=(0, 0)) pixelSize = det.pixel_size(evt) print("Current centre along row,centre along column: ", psanaCx, psanaCy) return psanaCx, psanaCy
def connectToDataSource(args, config, verbose=False): # If online if not args.offline: # make the shared memory string dataSource = 'shmem=AMO.0:stop=no' else: dataSource = config.offlineSource #config.makeTofConfigList(online=False) if verbose: print config if verbose: # check the host name host = platform.node() print 'rank {} (on {}) connecting to datasource: {}'.format( rank, host, dataSource) return psana.DataSource(dataSource)
def setup_exp(exp_name, run_num, det_name, mask_calib_on=True, mask_status_on=True, mask_edges_on=True, mask_central_on=True, mask_unbond_on=True, mask_unbondnrs_on=True): """ :param exp_name: :param run_num: :param mask_calib_on: :param mask_status_on: :param mask_edges_on: :param mask_central_on: :param mask_unbond_on: :param mask_unbondnrs_on: :return: """ ds = psana.DataSource('exp={}:run{}:idx'.format(exp_name, run_num)) run = ds.runs().next() times = run.times() env = ds.env() det = psana.Detector(det_name, env) et = psana.EventTime(int(ts[0]), fid[0]) evt = run.event(et) example = det.image(evt) shape = example.shape # get mask and save mask mask_stack = det.mask(evt, calib=mask_calib_on, status=mask_status_on, edges=mask_edges_on, central=mask_central_on, unbond=mask_unbond_on, unbondnbrs=mask_unbondnrs_on) mask_2d = det.image(evt, mask_stack) return
def main2(): import psana ds = psana.DataSource("exp=cxid9114:run=62") events = ds.events() det = psana.Detector('CxiDs2.0:Cspad.0') dark = det.pedestals(62) gain_map = det.gain_mask(62) == 1 mask = mask_utils.mask_small_regions(gain_map) mask2 = np.load("mask/details_mask.npy") mask *= mask2 start = 0 all_ylow = [] all_yhigh = [] for i in range(1200): if i < 1000: continue ev = events.next() if ev is None: continue if i < start: continue raw = det.raw(ev) if raw is None: continue data = raw - dark # new_data = correct_panels( data, gain_map, mask, plot=True) xlow, ylow, xhigh, yhigh, new_data = get_gain_dists( data, gain_map, mask) all_ylow.append(ylow) all_yhigh.append(yhigh) #plt.figure() #plt.imshow( new_data[0], vmin=-10,vmax=50,cmap='gnuplot') #plt.show() print i np.savez("all_shot_hists", ylow=all_ylow, yhigh=all_yhigh, xlow=xlow, xhigh=xhigh)
def getNoe(args): if facility == 'LCLS': runStr = "%04d" % args.run access = "exp=" + args.exp + ":run=" + runStr + ':idx' if 'ffb' in args.access.lower(): access += ':dir=/reg/d/ffb/' + args.exp[:3] + '/' + args.exp + '/xtc' ds = psana.DataSource(access) run = ds.runs().next() times = run.times() numJobs = len(times) elif facility == 'PAL': _temp = args.dir + '/' + args.exp[: 3] + '/' + args.exp + '/data/run' + str( args.run).zfill(4) + '/*.h5' numJobs = len(glob.glob(_temp)) # check if the user requested specific number of events if args.noe > -1 and args.noe <= numJobs: numJobs = args.noe return numJobs
def get_det_suggest(self): if not isinstance(self.expName,str): return [] if not isinstance(self.runNumber,int): return [] if self.ds is None: return [] myAreaDetectors = [] import Detector.PyDetector import Detector.AreaDetector ds = psana.DataSource("exp="+self.expName+":run="+str(self.runNumber)+':idx') for k in psana.DetNames(): try: if Detector.PyDetector.dettype(str(k[0]), self.env) == Detector.AreaDetector.AreaDetector: myAreaDetectors.append(k) except Exception as err: continue detInfoList = list(set(myAreaDetectors)) return detInfoList
def full_api_example(experiment, run_number, detector_address, params_file, event_num): """ Demo using the cctbx/lcls api @param experiment LCLS experiment string @param run_number Run number @param params_file cctbx/DIALS parameter file for processing @param event_num Index for specific event to process """ output_tag = '%s_run%d' % (experiment, run_number) print("Getting datasource") ds = psana.DataSource('exp=%s:run=%d' % (experiment, run_number)) processor = CctbxPsanaEventProcessor( params_file, output_tag ) # note, logfile already initialized in this demo, so don't do it twice for run in ds.runs(): print("Getting detector") det = psana.Detector(detector_address) processor.setup_run(run, det) for event_id, event in enumerate(ds.events()): print(event_id) if event_num is not None and event_id != event_num: continue tag = '%s_%s' % (output_tag, str(event_id)) experiments = processor.experiments_from_event(event) processor.tag = tag processor.setup_filenames(tag) try: processor.pre_process(experiments) observed = processor.find_spots(experiments) experiments, indexed = processor.index(experiments, observed) experiments, indexed = processor.refine(experiments, indexed) integrated = processor.integrate(experiments, indexed) print("Integrated %d spots on %d lattices" % (len(integrated), len(experiments))) except Exception as e: print("Couldn't process event %d" % event_id, str(e)) break break processor.finalize()
def algorithm(self): #self.push(kwargs) exp, runnum, det = self.getExpRunDet() ds = psana.DataSource('exp=%s:run=%d:idx'%(exp,runnum)) d = psana.Detector(det) d.do_reshape_2d_to_3d(flag=True) run = ds.runs().next() times = run.times() env = ds.env() mask = d.mask(runnum,calib=True,status=True,edges=True,central=True,unbond=True,unbondnbrs=True) numEvents = len(times) for j in range(numEvents): evt = run.event(times[j]) try: nda = d.calib(evt) * mask except TypeError: nda = d.calib(evt) if (nda is not None): print(exp, runnum, det, j) print("Sum is", np.sum(nda)) break
def expdet(experimentName,runNumber,detectorName): try: ds = psana.DataSource("exp="+experimentName+":run="+str(runNumber)+':idx') except: return None,None run = ds.runs().next() env = ds.env() times = run.times() det = psana.Detector(detectorName) epics = ds.env().epicsStore() counter = 0 evt = None while evt is None: evt = run.event(times[counter]) counter += 1 if counter > 1000: break if counter > len(times)-2: break return evt,det
def image2stack(image=None, experimentName=None, runNumber=None, detInfo=None, eventInd=0): ds = psana.DataSource('exp=' + str(experimentName) + ':run=' + str(runNumber) + ':idx') run = ds.runs().next() times = run.times() env = ds.env() evt = run.event(times[0]) det = psana.Detector(str(detInfo), env) evt = run.event(times[eventInd]) stack = det.ndarray_from_image(par=evt, image=image, pix_scale_size_um=None, xy0_off_pix=None) return stack
def main(): # Parse user input params = parse_input_arguments(sys.argv) radius = params['radius'] exp = params['exp'] run = params['run'] detname = params['det'] comm = MPI.COMM_WORLD rank = comm.Get_rank() numWorkers = comm.size print rank dsname = 'exp='+exp+':run='+str(run)+':idx' ds = psana.DataSource(dsname) run = ds.runs().next() times = run.times() env = ds.env() numEvents = len(times) evt = run.event(times[0]) # setup detector det = psana.Detector(detname, env) ipx, ipy = det.point_indexes(evt, pxy_um=(0,0)) myJobs = getMyUnfairShare(numEvents, numWorkers, rank) avg_error_arr = average_error_per_pixel(myJobs, ipx, ipy, radius, run, times, det) if rank == 0: start_time = time.time() for i in range(1, numWorkers): avg_error_arr = np.append(avg_error_arr, comm.recv(source=i)) np.save('Errors_amo86615_pnccdBack.npy', avg_error_arr) print 'Total time is: ' + str(time.time() - start_time) + 'seconds.' else: comm.send(avg_error_arr, dest=0)
def makeDataSourceAndSmallData(experimentNameAndRun, h5FileName, ttDevice, ttCode, shared_memory): global ttAnalyze smldata = "None" small_hdf5_dir = "hdf5" os.system("mkdir " + small_hdf5_dir) if (ttDevice is not None): print("setting up time tool.") print("Device = " + ttDevice) print("bykick code = " + str(ttCode)) ttOptions = TimeTool.AnalyzeOptions(get_key=ttDevice, eventcode_nobeam=ttCode) ttAnalyze = TimeTool.PyAnalyze(ttOptions) print("loading experiment data using standard small data") myDataSource = psana.MPIDataSource(experimentNameAndRun, module=ttAnalyze) print("defining small data") if (h5FileName != "None"): smldata = myDataSource.small_data(small_hdf5_dir + "/" + h5FileName, gather_interval=10) else: print("loading mpi data source") if (shared_memory): myDataSource = psana.DataSource('shmem=psana.0:stop=no') else: myDataSource = psana.MPIDataSource(experimentNameAndRun) print("defining small data. hook in place ") if (h5FileName != "None"): smldata = myDataSource.small_data(small_hdf5_dir + "/" + h5FileName, gather_interval=10) return (myDataSource, smldata)
def main3(): import psana ds = psana.DataSource("exp=cxid9114:run=62") events = ds.events() det = psana.Detector('CxiDs2.0:Cspad.0') dark = det.pedestals(62) gain_map = det.gain_mask(62) == 1 plt.imshow(gain_map[0]) plt.show() mask = mask_utils.mask_small_regions(gain_map) mask2 = np.load("details_mask.npy") mask *= mask2 start = 0 for i in range(100): ev = events.next() if ev is None: continue if i < start: continue data = det.calib(ev, cmpars=(5, 0, 0, 0, 0)) #data = det.calib(ev, cmpars=(1,25,25,100,1)) if data is None: continue plt.imshow(gain_map[0]) plt.show() xlow, ylow, xhigh, yhigh, new_data = get_gain_dists( data, gain_map, mask) low_g0, low_g1, fit_low = fit_utils.fit_low_gain_dist(xlow, ylow, plot=1) high_g0, high_g1, fit_high = fit_utils.fit_high_gain_dist(xhigh, yhigh, plot=1) plt.figure() plt.imshow(data[0], vmin=-10, vmax=50, cmap='gnuplot') plt.show()
def loadPsanaMask(experimentName, runNumber, detectorName): ds = psana.DataSource("exp=" + experimentName + ":run=" + str(runNumber) + ':idx') det = psana.Detector(detectorName) det.do_reshape_2d_to_3d(flag=True) run = ds.runs().next() times = run.times() evt = None counter = 0 while evt is None: evt = run.event(times[counter]) counter += 1 unassem_img = det.mask(runNumber, calib=True, status=True, edges=True, central=True, unbond=True, unbondnbrs=True) assem_img = det.image(evt, unassem_img) return unassem_img, assem_img
def _get_datasource(image_file, params): """Construct a psana data source object given the locator parameters""" if params.calib_dir is not None: psana.setOption("psana.calib-dir", params.calib_dir) if params.data_source is None: if (params.experiment is None or params.run is None or params.mode is None or len(params.run) == 0): return False img = "exp=%s:run=%s:%s" % ( params.experiment, ",".join(["%d" % r for r in params.run]), params.mode, ) if params.use_ffb: # as ffb is only at SLAC, ok to hardcode /reg/d here img += ":dir=/reg/d/ffb/%s/%s/xtc" % ( params.experiment[0:3], params.experiment, ) else: img = params.data_source return psana.DataSource(img)
def issue_2022_04_06(): """O'Grady, Paul Christopher <*****@*****.**> Wed 4/6/2022 6:01 PM Hi Mikhail, I started looking at this, but I think you will be more efficient than me. since you are more expert on calibration. The script below has a det.raw.calib that always returns 0. I think it might be because all pixels are marked bad in pixel_status with values of 42 (the corresponding pedestal run is 397 I believe). A later run 463 (which uses pedestal run 420) is fine. So it feels like there is some issue generating the constants with run 397? Would you have time to see what is going wrong with the run 397 pedestals (I think)? If you don?t have time I will continue looking. Thanks! chris """ import psana import numpy as np expname = 'uedcom103' runnum = 419 ds = psana.DataSource(exp=expname,run=runnum,detectors=['epixquad','epicsinfo']) myrun = next(ds.runs()) det = myrun.Detector('epixquad') print(det.calibconst.keys(),det.calibconst) for nevent,evt in enumerate(myrun.events()): print(det.raw.calib(evt)) break
def make_data_binary_file(dsname = 'exp=xpptut15:run=54',\ dname = 'CxiDs2.0:Cspad.0',\ nevs = 1000000,\ nskip = 0,\ do_calib = False,\ ofname = 'data.bin',\ verbos = True) : ds = psana.DataSource(dsname) det = psana.Detector(dname) #run = ds.runs().next() nev_min = nskip nev_max = nskip + nevs f = open(ofname, 'wb') myrun = next(ds.runs()) for nevt, evt in enumerate(myrun.events()): if nevt % 100 == 0: print 'Event %d' % nevt if nevt < nev_min: continue if nevt >= nev_max: break data = det.calib(evt) if do_calib else\ det.raw(evt) if data is None: continue if do_calib: data = data.astype(np.int16) if verbos: ent = entropy(data) msg = '%4d: data entropy=%.3f' % (nevt, ent) print_ndarr(data, name=msg, first=0, last=10) #data.tofile(f) f.write(data.tobytes()) f.close() print '%d datasets saved file %s' % (nevt, ofname)
def __init__(self, exp='cxip10016', run='1', gain_fnam=None, mask_fnam=None): """ By default gain = 1 and the mask is obtained from psana. For example: gain_fnam = '/reg/data/ana04/cxi/cxilt1417/scratch/gain/gain_prelim.npy' mask_fnam = '/reg/data/ana04/cxi/cxilt1417/scratch/masks/better_mask.npy' """ # # Initialize variables # print "Debug: loading psanabb" self.dsname_smd = 'exp=' + exp + ':run=' + run + ':smd' self.ds_smd = psana.MPIDataSource(self.dsname_smd) self.env_smd = self.ds_smd.env() self.evt = self.ds_smd.events().next() print "Debug: loading psanabb idx events" self.dsname_idx = 'exp=' + exp + ':run=' + run + ':idx' self.ds_idx = psana.DataSource(self.dsname_idx) self.env_idx = self.ds_idx.env() # # get event times # self.run = self.ds_idx.runs().next() self.times = self.run.times() self.nevents = len(self.times) print "Debug: loading cspad info" # load information self.loadCspad(gain_fnam=gain_fnam, mask_fnam=mask_fnam) self.wavelength = self.get_wavelength(self.evt) self.energy = self.get_photon_beam_energy(self.evt)
from pylab import * import psana dsource = psana.DataSource("exp=sxr10116:run=24") psana.DetNames() exitSlitOpal = psana.Detector("EXS_OPAL") getEnergy = psana.Detector("SIOC:SYS0:ML00:AO627") enumeratedEvents = enumerate(dsource.events()) eventNumber, myEvent = next(enumeratedEvents) myImage = exitSlitOpal.image(myEvent) myEnergy = getEnergy(myEvent) monoExitSlitYagSpectrum = sum(myImage, axis=1) myEnergyList = array([myEnergy]) for eventNumber, myEvent in enumeratedEvents: #myImage = exitSlitOpal.image(myEvent) myEnergy = getEnergy(myEvent) #monoExitSlitYagSpectrum = vstack([monoExitSlitYagSpectrum,sum(myImage,axis=1)]) myEnergyList = append(myEnergyList, myEnergy) #print eventNumber
drop=(xon==0) drop[0]=0 #e cannot get the pre-shot for this one. predrop=np.append(drop[1:],[0]).astype(bool) offfids = allfids[drop] #this look ok offtimes = alltimes[drop] preofffids = allfids[predrop] #this really does not. preofftimes = alltimes[predrop] #if expname=='xcsx35617': # Ped_epix10k = getPed_epix10k_xcsx35617(int(run)) #elif expname=='xcslt8717': # Ped_epix10k = getPed_epix10k_xcslt8717(int(run)) dsenv = (psana.DataSource('exp=%s:run=%d:smd'%(expname, int(run)))).env() runIdx = psana.DataSource('exp=%s:run=%d:idx'%(expname, int(run))).runs().next() #the ghost fitting should actually make a great pedestal! if expname=='xcslt8717': epixDet = DetObject('epix10ka2m', dsenv, int(run), common_mode=80) epixDetPre = DetObject('epix10ka2m', dsenv, int(run), common_mode=80) #epixDet = DetObject('epix10ka2m', dsenv, int(run), common_mode=81) #epixDetPre = DetObject('epix10ka2m', dsenv, int(run), common_mode=81) #if Ped_epix10k is not None: # epixDet.setPed(Ped_epix10k) # epixDetPre.setPed(Ped_epix10k) elif expname=='xcslt4017': #I'm not so sure this is what we want here. Maybe we did used method 81. epixDet = DetObject('epix10ka2m', dsenv, int(run), common_mode=80) epixDetPre = DetObject('epix10ka2m', dsenv, int(run), common_mode=80) else: