class CrystalFindingAnt: def __init__(self, host): self.host = host self.outdir = '/reg/d/psdm/cxi/cxitut13/res/autosfx/output' self.goodLikelihood = .003 #Minimum number of peaks to bev/reg/d/psdm/cxi/cxitut13/res/autosfx/output found to calculate likelihood self.goodNumPeaks = 10 #Minimum number of events to be found considered a good run self.minCrystals = 2 #List of good experiments with all runs evaluated - This ensures that an experiment will not have all # of its runs evaluated a second time self.goodList = [] #Condition to switch from random crawling to running through each experiment self.lastGood = False #Number of good runs saved by Queen, or num found by an Ant self.numSaved = 0 #Time when Ant is initialized self.startTime = time.time() self.context = zmq.Context() if self.host == "": self.socket = self.context.socket(zmq.REP) self.socket.bind("tcp://*:5556") else: self.socket = self.context.socket(zmq.REQ) self.socket.connect("tcp://%s:5556" % self.host) def calculate_likelihood(self, qPeaks): nPeaks = int(qPeaks.shape[1]) selfD = distance.cdist(qPeaks.transpose(), qPeaks.transpose(), 'euclidean') sortedSelfD = np.sort(selfD) closestNeighborDist = sortedSelfD[:, 1] meanClosestNeighborDist = np.median(closestNeighborDist) closestPeaks = [None] * nPeaks coords = qPeaks.transpose() pairsFound = 0. for ii in range(nPeaks): index = np.where(selfD[ii, :] == closestNeighborDist[ii]) closestPeaks[ii] = coords[list(index[0]), :].copy() p = coords[ii, :] flip = 2 * p - closestPeaks[ii] d = distance.cdist(coords, flip, 'euclidean') sigma = closestNeighborDist[ii] / 4. mu = 0. bins = d vals = np.exp(-(bins - mu) ** 2 / (2. * sigma ** 2)) weight = np.sum(vals) pairsFound += weight pairsFound = pairsFound / 2. pairsFoundPerSpot = pairsFound / float(nPeaks) return [meanClosestNeighborDist, pairsFoundPerSpot] def likelihood(self, peaks): maxPeaks = 2048 # Likelihood numPeaksFound = peaks.shape[0] if numPeaksFound >= self.goodNumPeaks and \ numPeaksFound <= maxPeaks: cenX = self.iX[np.array(peaks[:, 0], dtype=np.int64), np.array(peaks[:, 1], dtype=np.int64), np.array(peaks[:, 2], dtype=np.int64)] + 0.5 cenY = self.iY[np.array(peaks[:, 0], dtype=np.int64), np.array(peaks[:, 1], dtype=np.int64), np.array(peaks[:, 2], dtype=np.int64)] + 0.5 x = cenX - self.ipx # center[0] y = cenY - self.ipy # center[1] pixSize = float(self.det.pixel_size(self.evt)) self.detectorDistance = np.mean(self.det.coords_z(self.evt)) * 1e-6 # metres detdis = float(self.detectorDistance) z = detdis / pixSize * np.ones(x.shape) # pixels ebeamDet = psana.Detector('EBeam') ebeam = ebeamDet.get(self.evt) try: photonEnergy = ebeam.ebeamPhotonEnergy() except: photonEnergy = 1 wavelength = 12.407002 / float(photonEnergy) # Angstrom norm = np.sqrt(x ** 2 + y ** 2 + z ** 2) qPeaks = (np.array([x, y, z]) / norm - np.array([[0.], [0.], [1.]])) / wavelength [meanClosestNeighborDist, pairsFoundPerSpot] = self.calculate_likelihood(qPeaks) else: pairsFoundPerSpot = 0.0 return pairsFoundPerSpot def respond2Clients(self): message = self.socket.recv() mode,exp,runnum,detname,numFound = message.split(",") runnum = int(runnum) if mode == "check": status = self.checkStatus(exp,runnum,detname) elif mode == "update": status = self.updateStatus(exp,runnum,detname,int(numFound)) self.socket.send(str(status)) def checkStatus(self, exp, runnum, detname): if self.host == "": outfile = os.path.join(self.outdir, exp + '.npz') try: npzfile = np.load(outfile) ind = np.where(npzfile['runs'] == runnum)[0] if len(ind) == 0: return False else: return True except: return False else: # ask master for status self.socket.send("check,"+exp+","+str(runnum)+","+detname+","+"-1") message = self.socket.recv() if message == "True": return True else: return False def updateStatus(self, exp, runnum, detname, num): if self.host == "": outfile = os.path.join(self.outdir, exp + '.npz') todo = os.path.join(self.outdir, 'todo.txt') try: npzfile = np.load(outfile) runs = npzfile['runs'] ind = np.where(runs == runnum)[0] # Does the run exist? if len(ind) == 0: # insert run ind = np.where(runs < runnum)[0] if len(ind) == 0: runs = np.insert(runs, 0, runnum) else: runs = np.insert(runs, ind[-1]+1, runnum) np.savez(outfile, runs=runs, detname=detname) else: print "This exp:run exists" except: runs = np.array([runnum],) np.savez(outfile, runs=runs, detname=detname) self.numSaved+=1 print"Good runs found: %d"%self.numSaved timeSince = time.time() - self.startTime with open(todo, "a") as f: msg = exp + " " + str(runnum) + " " + detname + " " + str(num) + " " + str(timeSince) + "\n" f.write(msg) else: # ask master to update self.socket.send("update," + exp + "," + str(runnum) + "," + detname + "," + str(num)) message = self.socket.recv() if message == "True": return True else: return False def run(self): """ Runs the peakfinder, adds values to the database, trains peaknet, and reports model to the master Arguments: alg -- the peakfinding algorithm kwargs -- peakfinding parameters, host and server name, client name """ evaluateAllRuns = False while True: if self.host == "": # respond to clients self.respond2Clients() else: print("Next...") # randomly choose experiment + run if not evaluateAllRuns: print("Randomly fetching run") self.exp, self.runnum, self.detname = randExpRunDet() else: try: print("Fecthing next run in experiment") self.exp, self.runnum, self.detname = nextExpRunDet(self.goodExp, self.runList[0]) if self.exp is None: self.runList.pop(0) continue except: evaluateAllRuns = False continue if not self.checkStatus(self.exp, self.runnum, self.detname): print "trying: exp %s, run %s, det %s"%(self.exp,self.runnum,self.detname) try: #temp self.ds = safeDataSource(self.exp, self.runnum) except: #temp continue #temp self.run = self.ds.runs().next() self.times = self.run.times() #Start temp code if self.detname is None: continue #End temp code self.det = psana.Detector(self.detname) self.det.do_reshape_2d_to_3d(flag=True) try: self.iX = np.array(self.det.indexes_x(self.run), dtype=np.int64) self.iY = np.array(self.det.indexes_y(self.run), dtype=np.int64) self.ipx, self.ipy = self.det.point_indexes(self.run, pxy_um=(0, 0)) self.alg = PyAlgos() self.alg.set_peak_selection_pars(npix_min=2, npix_max=30, amax_thr=300, atot_thr=600, son_min=10) mask = self.det.mask(self.runnum, calib=True, status=True, edges=True, central=True, unbond=True, unbondnbrs=True) samples = np.linspace(0, len(self.times), num=100, endpoint=False, retstep=False, dtype='int') offset = np.floor(np.random.uniform(0, len(self.times)-samples[-1])).astype('int') mysamples = samples + offset numCrystals = 0 for self.eventNum in mysamples: self.evt = self.run.event(self.times[self.eventNum]) calib = self.det.calib(self.evt) if calib is not None: peaks = self.alg.peak_finder_v3r3(calib, rank=3, r0=3, dr=2, nsigm=10, mask=mask.astype(np.uint16)) if self.likelihood(peaks) >= self.goodLikelihood: numCrystals += 1 if numCrystals >= self.minCrystals: self.numSaved +=1 self.updateStatus(self.exp, self.runnum, self.detname, self.numSaved) self.lastGood = True break except: print "Could not analyse this run" #If an experiment has not had all of its runs evaluated yet # and if the last randomly selected run in this experiment was good # then all the runs in this experiment should be evaluated if (self.exp not in self.goodList) and self.lastGood: self.goodExp = self.exp #Save the name of this experiment self.goodRun = self.runnum #Save the run that has already been evaluated self.lastGood = False #Reset the condition that the last run was "good" self.goodList.append(self.goodExp) #Add this experiment name to the list of experiments that have had all runs evaluated self.runList = returnRunList(self.goodExp, self.goodRun) #save list of all runs in this good exp evaluateAllRuns = True #rerun loop with new algorithm that evaluates each run in an experiment continue if evaluateAllRuns: #If the loop is currently evaluating all of the runs in an experiment if(len(self.runList) > 1): self.runList.pop(0) #Remove runs from the list of runs each time they are evaluated else: self.runList.pop(0)#Remove runs until the list is completely empty evaluateAllRuns = False #Stop evaluated all the runs of an experiment, go back to random fetching
env = ds.env() numEvents = len(times) mask = d.mask(runnum, calib=True, status=True, edges=True, central=True, unbond=True, unbondnbrs=True) for j in range(numEvents): print("Event #:", j + 1) evt = run.event(times[j]) try: nda = d.calib(evt) * mask except TypeError: nda = d.calib(evt) if (nda is not None): print(nda.shape[1]) peaks = alg.peak_finder_v3r3(nda, rank=3, r0=3, dr=2, nsigm=5) numPeaksFound = len(peaks) if (numPeaksFound >= 15): for x in range(nda.shape[1]): for y in range(nda.shape[2]): if nda[0][x][y] > 5000: print(exp, runnum, j + 1, x, y) pixelList.append([exp, runnum, j + 1, x, y]) print("done") pprint(pixelList)
class PeakFinder: def __init__(self, exp, run, detname, evt, detector, algorithm, hitParam_alg_npix_min, hitParam_alg_npix_max, hitParam_alg_amax_thr, hitParam_alg_atot_thr, hitParam_alg_son_min, streakMask_on, streakMask_sigma, streakMask_width, userMask_path, psanaMask_on, psanaMask_calib, psanaMask_status, psanaMask_edges, psanaMask_central, psanaMask_unbond, psanaMask_unbondnrs, generousMask=0, medianFilterOn=0, medianRank=5, radialFilterOn=0, distance=0.0, windows=None, **kwargs): self.exp = exp self.run = run self.detname = detname self.det = detector self.algorithm = algorithm self.maxRes = 0 self.npix_min = hitParam_alg_npix_min self.npix_max = hitParam_alg_npix_max self.amax_thr = hitParam_alg_amax_thr self.atot_thr = hitParam_alg_atot_thr self.son_min = hitParam_alg_son_min self.streakMask_on = str2bool(streakMask_on) self.streakMask_sigma = streakMask_sigma self.streakMask_width = streakMask_width self.userMask_path = userMask_path self.psanaMask_on = str2bool(psanaMask_on) self.psanaMask_calib = str2bool(psanaMask_calib) self.psanaMask_status = str2bool(psanaMask_status) self.psanaMask_edges = str2bool(psanaMask_edges) self.psanaMask_central = str2bool(psanaMask_central) self.psanaMask_unbond = str2bool(psanaMask_unbond) self.psanaMask_unbondnrs = str2bool(psanaMask_unbondnrs) self.generousMaskOn = generousMask self.medianFilterOn = medianFilterOn self.medianRank = medianRank self.radialFilterOn = radialFilterOn self.distance = distance self.windows = windows self.userMask = None self.psanaMask = None self.streakMask = None self.userPsanaMask = None self.combinedMask = None self.generousMask = None # Make user mask if self.userMask_path is not None: self.userMask = np.load(self.userMask_path) if facility == 'LCLS': # Make psana mask if self.psanaMask_on: self.psanaMask = detector.mask( evt, calib=self.psanaMask_calib, status=self.psanaMask_status, edges=self.psanaMask_edges, central=self.psanaMask_central, unbond=self.psanaMask_unbond, unbondnbrs=self.psanaMask_unbondnrs) if self.generousMaskOn: self.psanaMask = self.generousBadPixel(self.psanaMask) # Combine userMask and psanaMask self.userPsanaMask = np.ones_like(self.det.calib(evt), dtype=np.int16) if self.userMask is not None: self.userPsanaMask *= self.userMask if self.psanaMask is not None: self.userPsanaMask *= self.psanaMask elif facility == 'PAL': self.userPsanaMask = self.userMask # Powder of hits and misses self.powderHits = None self.powderMisses = None # set algorithm specific parameters if algorithm == 1: self.hitParam_alg1_thr_low = kwargs["alg1_thr_low"] self.hitParam_alg1_thr_high = kwargs["alg1_thr_high"] self.hitParam_alg1_rank = int(kwargs["alg1_rank"]) self.hitParam_alg1_radius = int(kwargs["alg1_radius"]) self.hitParam_alg1_dr = kwargs["alg1_dr"] elif algorithm == 2: self.hitParam_alg1_thr_low = kwargs["alg1_thr_low"] self.hitParam_alg1_thr_high = kwargs["alg1_thr_high"] self.hitParam_alg1_rank = int(kwargs["alg1_rank"]) self.hitParam_alg1_radius = int(kwargs["alg1_radius"]) self.hitParam_alg1_dr = kwargs["alg1_dr"] elif algorithm == 3: self.hitParam_alg3_rank = kwargs["alg3_rank"] self.hitParam_alg3_r0 = int(kwargs["alg3_r0"]) self.hitParam_alg3_dr = kwargs["alg3_dr"] elif algorithm == 4: self.hitParam_alg4_thr_low = kwargs["alg4_thr_low"] self.hitParam_alg4_thr_high = kwargs["alg4_thr_high"] self.hitParam_alg4_rank = int(kwargs["alg4_rank"]) self.hitParam_alg4_r0 = int(kwargs["alg4_r0"]) self.hitParam_alg4_dr = kwargs["alg4_dr"] if facility == 'LCLS': self.access = kwargs["access"] if self.algorithm == 1: self.alg = PyAlgos(mask=None, pbits=0) self.peakRadius = int(self.hitParam_alg1_radius) self.alg.set_peak_selection_pars(npix_min=self.npix_min, npix_max=self.npix_max, \ amax_thr=self.amax_thr, atot_thr=self.atot_thr, \ son_min=self.son_min) #self.alg = myskbeam.DropletA(self.peakRadius, self.peakRadius+self.hitParam_alg1_dr) #PyAlgos(windows=self.windows, mask=None, pbits=0) elif self.algorithm == 2: # set peak-selector parameters: self.alg = PyAlgos(mask=None, pbits=0) self.peakRadius = int(self.hitParam_alg1_radius) self.alg.set_peak_selection_pars(npix_min=self.npix_min, npix_max=self.npix_max, \ amax_thr=self.amax_thr, atot_thr=self.atot_thr, \ son_min=self.son_min) elif facility == 'PAL': self.peakRadius = int(self.hitParam_alg1_radius) self.alg = myskbeam.DropletA(self.peakRadius, self.hitParam_alg1_dr) if facility == 'LCLS': self.StreakMask = myskbeam.StreakMask(self.det, evt, width=self.streakMask_width, sigma=self.streakMask_sigma) self.cx, self.cy = self.det.point_indexes(evt, pxy_um=(0, 0)) self.iX = np.array(self.det.indexes_x(evt), dtype=np.int64) self.iY = np.array(self.det.indexes_y(evt), dtype=np.int64) if len(self.iX.shape) == 2: self.iX = np.expand_dims(self.iX, axis=0) self.iY = np.expand_dims(self.iY, axis=0) # Initialize radial background subtraction self.setupExperiment() if self.radialFilterOn: self.setupRadialBackground() self.updatePolarizationFactor() elif facility == 'PAL': self.geom = kwargs["geom"] with open(self.geom, 'r') as f: lines = f.readlines() for i, line in enumerate(lines): if 'p0/max_ss' in line: dim0 = int(line.split('=')[-1]) + 1 elif 'p0/max_fs' in line: dim1 = int(line.split('=')[-1]) + 1 elif 'p0/corner_x' in line: self.cy = -1 * float(line.split('=')[-1]) elif 'p0/corner_y' in line: self.cx = float(line.split('=')[-1]) self.iy = np.tile(np.arange(dim0), [dim1, 1]) self.ix = np.transpose(self.iy) self.iX = np.array(self.ix, dtype=np.int64) self.iY = np.array(self.iy, dtype=np.int64) def generousBadPixel(self, unassemMask, n=10): generousBadPixelMask = unassemMask (numAsic, numFs, numSs) = unassemMask.shape for i in range(numAsic): for a in range(numSs): numBadPixels = len(np.where(unassemMask[i, :, a] == 0)[0]) if numBadPixels >= n: generousBadPixelMask[i, :, a] = 0 return generousBadPixelMask def setupExperiment(self): access = 'exp=' + str(self.exp) + ':run=' + str(self.run) + ':idx' if 'ffb' in self.access.lower(): access += ':dir=/reg/d/ffb/' + self.exp[:3] + '/' + self.exp + '/xtc' self.ds = psana.DataSource(access) self.run = self.ds.runs().next() self.times = self.run.times() self.eventTotal = len(self.times) self.env = self.ds.env() self.evt = self.run.event(self.times[0]) self.det = psana.Detector(str(self.detname), self.env) self.det.do_reshape_2d_to_3d(flag=True) def setupRadialBackground(self): self.geo = self.det.geometry( self.run ) # self.geo = GeometryAccess(self.parent.geom.calibPath+'/'+self.parent.geom.calibFile) self.xarr, self.yarr, self.zarr = self.geo.get_pixel_coords() self.ix = self.det.indexes_x(self.evt) self.iy = self.det.indexes_y(self.evt) if self.ix is None: self.iy = np.tile(np.arange(self.userMask.shape[1]), [self.userMask.shape[2], 1]) self.ix = np.transpose(self.iy) self.iX = np.array(self.ix, dtype=np.int64) self.iY = np.array(self.iy, dtype=np.int64) if len(self.iX.shape) == 2: self.iX = np.expand_dims(self.iX, axis=0) self.iY = np.expand_dims(self.iY, axis=0) self.mask = self.geo.get_pixel_mask( mbits=0377 ) # mask for 2x1 edges, two central columns, and unbound pixels with their neighbours self.rb = RadialBkgd(self.xarr, self.yarr, mask=self.mask, radedges=None, nradbins=100, phiedges=(0, 360), nphibins=1) def updatePolarizationFactor(self): self.pf = polarization_factor(self.rb.pixel_rad(), self.rb.pixel_phi(), self.distance * 1e6) # convert to um def findPeaks(self, calib, evt, thr_high=None, thr_low=None): if facility == 'LCLS': if self.streakMask_on: # make new streak mask self.streakMask = self.StreakMask.getStreakMaskCalib(evt) # Apply background correction if self.medianFilterOn: calib -= median_filter_ndarr(calib, self.medianRank) if self.radialFilterOn: self.pf.shape = calib.shape # FIXME: shape is 1d calib = self.rb.subtract_bkgd(calib * self.pf) calib.shape = self.userPsanaMask.shape # FIXME: shape is 1d if self.streakMask is not None: self.combinedMask = self.userPsanaMask * self.streakMask else: self.combinedMask = self.userPsanaMask # set new mask #self.alg.set_mask(self.combinedMask) # This doesn't work reliably elif facility == 'PAL': self.combinedMask = self.userMask # set algorithm specific parameters if self.algorithm == 1: if facility == 'LCLS': #print "param: ", self.npix_min, self.npix_max, self.atot_thr, self.son_min, thr_low, thr_high, np.sum(self.combinedMask) # v1 - aka Droplet Finder - two-threshold peak-finding algorithm in restricted region # around pixel with maximal intensity. if thr_high is None: # use gui input self.peakRadius = int(self.hitParam_alg1_radius) self.peaks = self.alg.peak_finder_v4r3( calib, thr_low=self.hitParam_alg1_thr_low, thr_high=self.hitParam_alg1_thr_high, rank=self.hitParam_alg1_rank, r0=self.hitParam_alg1_radius, dr=self.hitParam_alg1_dr, mask=self.combinedMask.astype(np.uint16)) # self.peaks = self.alg.peak_finder_v4r2(calib, # thr_low=self.hitParam_alg1_thr_low, # thr_high=self.hitParam_alg1_thr_high, # rank=self.hitParam_alg1_rank, # r0=self.hitParam_alg1_radius, # dr=self.hitParam_alg1_dr) else: self.peaks = self.alg.findPeaks(calib, npix_min=self.npix_min, npix_max=self.npix_max, atot_thr=self.atot_thr, son_min=self.son_min, thr_low=thr_low, thr_high=thr_high, mask=self.combinedMask) # self.peaks = self.alg.peak_finder_v4r2(calib, # thr_low=thr_low, # thr_high=thr_high, # rank=self.hitParam_alg1_rank, # r0=self.hitParam_alg1_radius, # dr=self.hitParam_alg1_dr) elif facility == 'PAL': self.peakRadius = int(self.hitParam_alg1_radius) _calib = np.zeros((1, calib.shape[0], calib.shape[1])) _calib[0, :, :] = calib if self.combinedMask is None: _mask = None else: _mask = self.combinedMask.astype(np.uint16) self.peaks = self.alg.findPeaks( _calib, npix_min=self.npix_min, npix_max=self.npix_max, son_min=self.son_min, thr_low=self.hitParam_alg1_thr_low, thr_high=self.hitParam_alg1_thr_high, atot_thr=self.atot_thr, r0=self.peakRadius, dr=int(self.hitParam_alg1_dr), mask=_mask) elif self.algorithm == 2: if facility == 'LCLS': #print "param: ", self.npix_min, self.npix_max, self.atot_thr, self.son_min, thr_low, thr_high, np.sum(self.combinedMask) # v1 - aka Droplet Finder - two-threshold peak-finding algorithm in restricted region # around pixel with maximal intensity. self.peakRadius = int(self.hitParam_alg1_radius) self.peaks = self.alg.peak_finder_v3r3( calib, rank=int(self.hitParam_alg1_rank), r0=self.peakRadius, dr=self.hitParam_alg1_dr, nsigm=self.son_min, mask=self.combinedMask.astype(np.uint16)) elif self.algorithm == 3: self.peaks = self.alg.peak_finder_v3(calib, rank=self.hitParam_alg3_rank, r0=self.hitParam_alg3_r0, dr=self.hitParam_alg3_dr) elif self.algorithm == 4: # v4 - aka iDroplet Finder - two-threshold peak-finding algorithm in restricted region # around pixel with maximal intensity. self.peaks = self.alg.peak_finder_v4(calib, thr_low=self.hitParam_alg4_thr_low, thr_high=self.hitParam_alg4_thr_high, \ rank=self.hitParam_alg4_rank, r0=self.hitParam_alg4_r0, dr=self.hitParam_alg4_dr) self.numPeaksFound = self.peaks.shape[0] if self.numPeaksFound > 0: if facility == 'LCLS': cenX = self.iX[np.array(self.peaks[:, 0], dtype=np.int64), np.array(self.peaks[:, 1], dtype=np.int64), np.array(self.peaks[:, 2], dtype=np.int64)] + 0.5 cenY = self.iY[np.array(self.peaks[:, 0], dtype=np.int64), np.array(self.peaks[:, 1], dtype=np.int64), np.array(self.peaks[:, 2], dtype=np.int64)] + 0.5 elif facility == 'PAL': cenX = self.iX[np.array(self.peaks[:, 1], dtype=np.int64), np.array(self.peaks[:, 2], dtype=np.int64)] + 0.5 cenY = self.iY[np.array(self.peaks[:, 1], dtype=np.int64), np.array(self.peaks[:, 2], dtype=np.int64)] + 0.5 self.maxRes = getMaxRes(cenX, cenY, self.cx, self.cy) else: self.maxRes = 0 if self.numPeaksFound >= 15: if self.powderHits is None: self.powderHits = calib else: self.powderHits = np.maximum(self.powderHits, calib) else: if self.powderMisses is None: self.powderMisses = calib else: self.powderMisses = np.maximum(self.powderMisses, calib) if self.powderHits is None: self.powderHits = np.zeros_like(calib) if self.powderMisses is None: self.powderMisses = np.zeros_like(calib)
class CrystalFindingAnt: def __init__(self, host): self.host = host self.outdir = '/reg/d/psdm/cxi/cxitut13/res/autosfx/output' self.goodLikelihood = .003 #Minimum number of peaks to be found to calculate likelihood self.goodNumPeaks = 10 #Minimum number of events to be found considered a good run self.minCrystals = 2 self.context = zmq.Context() if self.host == "": self.socket = self.context.socket(zmq.REP) self.socket.bind("tcp://*:5556") else: self.socket = self.context.socket(zmq.REQ) self.socket.connect("tcp://%s:5556" % self.host) def calculate_likelihood(self, qPeaks): nPeaks = int(qPeaks.shape[1]) selfD = distance.cdist(qPeaks.transpose(), qPeaks.transpose(), 'euclidean') sortedSelfD = np.sort(selfD) closestNeighborDist = sortedSelfD[:, 1] meanClosestNeighborDist = np.median(closestNeighborDist) closestPeaks = [None] * nPeaks coords = qPeaks.transpose() pairsFound = 0. for ii in range(nPeaks): index = np.where(selfD[ii, :] == closestNeighborDist[ii]) closestPeaks[ii] = coords[list(index[0]), :].copy() p = coords[ii, :] flip = 2 * p - closestPeaks[ii] d = distance.cdist(coords, flip, 'euclidean') sigma = closestNeighborDist[ii] / 4. mu = 0. bins = d vals = np.exp(-(bins - mu)**2 / (2. * sigma**2)) weight = np.sum(vals) pairsFound += weight pairsFound = pairsFound / 2. pairsFoundPerSpot = pairsFound / float(nPeaks) return [meanClosestNeighborDist, pairsFoundPerSpot] def likelihood(self, peaks): maxPeaks = 2048 # Likelihood numPeaksFound = peaks.shape[0] if numPeaksFound >= self.goodNumPeaks and \ numPeaksFound <= maxPeaks: cenX = self.iX[np.array(peaks[:, 0], dtype=np.int64), np.array(peaks[:, 1], dtype=np.int64), np.array(peaks[:, 2], dtype=np.int64)] + 0.5 cenY = self.iY[np.array(peaks[:, 0], dtype=np.int64), np.array(peaks[:, 1], dtype=np.int64), np.array(peaks[:, 2], dtype=np.int64)] + 0.5 x = cenX - self.ipx # center[0] y = cenY - self.ipy # center[1] pixSize = float(self.det.pixel_size(self.evt)) self.detectorDistance = np.mean(self.det.coords_z( self.evt)) * 1e-6 # metres detdis = float(self.detectorDistance) z = detdis / pixSize * np.ones(x.shape) # pixels ebeamDet = psana.Detector('EBeam') ebeam = ebeamDet.get(self.evt) try: photonEnergy = ebeam.ebeamPhotonEnergy() except: photonEnergy = 1 wavelength = 12.407002 / float(photonEnergy) # Angstrom norm = np.sqrt(x**2 + y**2 + z**2) qPeaks = (np.array([x, y, z]) / norm - np.array([[0.], [0.], [1.]])) / wavelength [meanClosestNeighborDist, pairsFoundPerSpot] = self.calculate_likelihood(qPeaks) else: pairsFoundPerSpot = 0.0 return pairsFoundPerSpot def respond2Clients(self): message = self.socket.recv() mode, exp, runnum, detname = message.split(",") runnum = int(runnum) if mode == "check": status = self.checkStatus(exp, runnum, detname) elif mode == "update": status = self.updateStatus(exp, runnum, detname) self.socket.send(str(status)) def checkStatus(self, exp, runnum, detname): if self.host == "": outfile = os.path.join(self.outdir, exp + '.npz') try: npzfile = np.load(outfile) ind = np.where(npzfile['runs'] == runnum)[0] if len(ind) == 0: return False else: return True except: return False else: # ask master for status self.socket.send("check," + exp + "," + str(runnum) + "," + detname) message = self.socket.recv() if message == "True": return True else: return False def updateStatus(self, exp, runnum, detname): if self.host == "": outfile = os.path.join(self.outdir, exp + '.npz') todo = os.path.join(self.outdir, 'todo.txt') try: npzfile = np.load(outfile) runs = npzfile['runs'] ind = np.where(runs == runnum)[0] # Does the run exist? if len(ind) == 0: print "Found one!" # insert run ind = np.where(runs < runnum)[0] if len(ind) == 0: runs = np.insert(runs, 0, runnum) else: runs = np.insert(runs, ind[-1] + 1, runnum) np.savez(outfile, runs=runs, detname=detname) else: print "This exp:run exists" except: runs = np.array([runnum], ) np.savez(outfile, runs=runs, detname=detname) print "Create new file" with open(todo, "a") as f: msg = exp + " " + str(runnum) + " " + detname + "\n" f.write(msg) else: # ask master to update self.socket.send("update," + exp + "," + str(runnum) + "," + detname) message = self.socket.recv() if message == "True": return True else: return False def run(self): """ Runs the peakfinder, adds values to the database, trains peaknet, and reports model to the master Arguments: alg -- the peakfinding algorithm kwargs -- peakfinding parameters, host and server name, client name """ while True: if self.host == "": # respond to clients self.respond2Clients() else: # randomly choose experiment + run self.exp, self.runnum, self.detname = randExpRunDet() print "exp run det: ", self.exp, self.runnum, self.detname if not self.checkStatus(self.exp, self.runnum, self.detname): self.ds = safeDataSource(self.exp, self.runnum) self.run = self.ds.runs().next() self.times = self.run.times() self.det = psana.Detector(self.detname) self.det.do_reshape_2d_to_3d(flag=True) try: self.iX = np.array(self.det.indexes_x(self.run), dtype=np.int64) self.iY = np.array(self.det.indexes_y(self.run), dtype=np.int64) self.ipx, self.ipy = self.det.point_indexes(self.run, pxy_um=(0, 0)) self.alg = PyAlgos() self.alg.set_peak_selection_pars(npix_min=2, npix_max=30, amax_thr=300, atot_thr=600, son_min=10) mask = self.det.mask(self.runnum, calib=True, status=True, edges=True, central=True, unbond=True, unbondnbrs=True) samples = np.linspace(0, len(self.times), num=100, endpoint=False, retstep=False, dtype='int') offset = np.floor( np.random.uniform(0, len(self.times) - samples[-1])).astype('int') mysamples = samples + offset numCrystals = 0 for self.eventNum in mysamples: self.evt = self.run.event( self.times[self.eventNum]) calib = self.det.calib(self.evt) if calib is not None: peaks = self.alg.peak_finder_v3r3( calib, rank=3, r0=3, dr=2, nsigm=10, mask=mask.astype(np.uint16)) if self.likelihood( peaks) >= self.goodLikelihood: numCrystals += 1 if numCrystals >= self.minCrystals: self.updateStatus(self.exp, self.runnum, self.detname) break except: print "Could not analyse this run"
r0 = 3 dr = 2 winR = r0 + dr + rank gSeg = 0 for t in range(len(times)): # loop through all event in a run # if (t % 10) == 0 : # print t if (t == 100): break print "Event=", t evt = run.event(times[t]) calib = det.calib(evt) #print calib.shape peaks = alg.peak_finder_v3r3(calib, rank=3, r0=r0, dr=dr, nsigm=10, mask=peakMask) roi = ROI.copy() #np.ones_like(calib,dtype='uint8') #print hdr # numPeaks=0 # print type(len(peaks)) peakLen = np.array([len(peaks), len(peaks)], np.int64) peaksF.write(bytearray(peakLen)) print(peakLen[0]) # print "Number of peaks:"+str(len(peaks)) for i, peak in enumerate(peaks): # numPeaks=numPeaks+1 _seg, _row, _col, _npix, _amax, _atot, _rcgrav, _ccgrav, _rsig, _csig, _rmin, _rmax, _cmin, _cmax, _bkgd, _noise, _son = peak # print fmt % (_seg, _row, _col, _npix, _atot) _seg = int(_seg)
class PeakFinding(object): def __init__(self, parent = None): self.parent = parent ## Dock: Peak finder self.dock = Dock("Peak Finder", size=(1, 1)) self.win = ParameterTree() self.dock.addWidget(self.win) #self.w11 = pg.LayoutWidget() #self.generatePowderBtn = QtGui.QPushButton('Generate Powder') #self.launchBtn = QtGui.QPushButton('Launch peak finder') #self.w11.addWidget(self.launchBtn, row=0,col=0) #self.w11.addWidget(self.generatePowderBtn, row=0, col=0) #self.dock.addWidget(self.w11) self.userUpdate = None self.doingUpdate = False # Peak finding self.hitParam_grp = 'Peak finder' self.hitParam_showPeaks_str = 'Show peaks found' self.hitParam_autoPeaks_str = 'Auto peak finder' self.hitParam_algorithm_str = 'Algorithm' self.hitParam_parameters_str = 'Parameters' # algorithm 0 self.hitParam_algorithm0_str = 'None' # algorithm 1 self.hitParam_alg1_npix_min_str = 'npix_min' self.hitParam_alg1_npix_max_str = 'npix_max' self.hitParam_alg1_amax_thr_str = 'amax_thr' self.hitParam_alg1_atot_thr_str = 'atot_thr' self.hitParam_alg1_son_min_str = 'son_min' self.hitParam_algorithm1_str = 'Droplet (a.k.a v4r3)' self.hitParam_alg1_thr_low_str = 'thr_low' self.hitParam_alg1_thr_high_str = 'thr_high' self.hitParam_alg1_rank_str = 'rank' self.hitParam_alg1_radius_str = 'radius' self.hitParam_alg1_dr_str = 'dr' # algorithm 2 self.hitParam_alg2_npix_min_str = 'npix_min' self.hitParam_alg2_npix_max_str = 'npix_max' self.hitParam_alg2_amax_thr_str = 'amax_thr' self.hitParam_alg2_atot_thr_str = 'atot_thr' self.hitParam_alg2_son_min_str = 'son_min' self.hitParam_algorithm2_str = 'Adaptive (a.k.a v3r3)' self.hitParam_alg2_thr_str = 'thr' self.hitParam_alg2_r0_str = 'r0' self.hitParam_alg2_dr_str = 'dr' # algorithm 3 self.hitParam_alg3_npix_min_str = 'npix_min' self.hitParam_alg3_npix_max_str = 'npix_max' self.hitParam_alg3_amax_thr_str = 'amax_thr' self.hitParam_alg3_atot_thr_str = 'atot_thr' self.hitParam_alg3_son_min_str = 'son_min' self.hitParam_algorithm3_str = 'szPeak' self.hitParam_alg3_thr_str = 'thr' self.hitParam_alg3_r0_str = 'r0' self.hitParam_alg3_dr_str = 'dr' self.hitParam_outDir_str = 'Output directory' self.hitParam_runs_str = 'Run(s)' self.hitParam_queue_str = 'queue' self.hitParam_cpu_str = 'CPUs' self.hitParam_psanaq_str = 'psanaq' self.hitParam_psnehq_str = 'psnehq' self.hitParam_psfehq_str = 'psfehq' self.hitParam_psnehprioq_str = 'psnehprioq' self.hitParam_psfehprioq_str = 'psfehprioq' self.hitParam_psnehhiprioq_str = 'psnehhiprioq' self.hitParam_psfehhiprioq_str = 'psfehhiprioq' self.hitParam_psdebugq_str = 'psdebugq' self.hitParam_psanagpuq_str = 'psanagpuq' self.hitParam_psanaidleq_str = 'psanaidleq' self.hitParam_noQueue_str = 'N/A' self.hitParam_noe_str = 'Number of events to process' self.hitParam_threshold_str = 'Indexable number of peaks' self.hitParam_launch_str = 'Launch peak finder' #self.hitParam_launchAuto_str = 'Launch auto peak finder' self.hitParam_extra_str = 'Extra parameters' self.save_minPeaks_str = 'Minimum number of peaks' self.save_maxPeaks_str = 'Maximum number of peaks' self.save_minRes_str = 'Minimum resolution (pixels)' self.save_sample_str = 'Sample name' self.tag_str = '.cxi tag' if self.parent.mode == "sfx": self.showPeaks = True else: self.showPeaks = False self.turnOnAutoPeaks = False self.ind = None self.pairsFoundPerSpot = 0 self.peaks = None self.numPeaksFound = 0 self.algorithm = 2 self.algInitDone = False self.peaksMaxRes = 0 self.classify = False self.tag = '' self.hitParam_alg1_npix_min = 2. self.hitParam_alg1_npix_max = 30. self.hitParam_alg1_amax_thr = 300. self.hitParam_alg1_atot_thr = 600. self.hitParam_alg1_son_min = 10. self.hitParam_alg1_thr_low = 0. self.hitParam_alg1_thr_high = 0. self.hitParam_alg1_rank = 3 self.hitParam_alg1_radius = 3 self.hitParam_alg1_dr = 2 self.hitParam_outDir = self.parent.psocakeDir self.hitParam_outDir_overridden = False self.hitParam_runs = '' self.hitParam_queue = self.hitParam_psanaq_str self.hitParam_cpus = 24 self.hitParam_noe = -1 self.hitParam_threshold = 15 # usually crystals with less than 15 peaks are not indexable self.minPeaks = 10 self.maxPeaks = 2048 self.minRes = -1 self.sample = 'sample' self.profile = 0 self.hitParam_extra = '' self.params = [ {'name': self.hitParam_grp, 'type': 'group', 'children': [ {'name': self.hitParam_showPeaks_str, 'type': 'bool', 'value': self.showPeaks, 'tip': "Show peaks found shot-to-shot"}, #{'name': self.hitParam_autoPeaks_str, 'type': 'bool', 'value': self.turnOnAutoPeaks, # 'tip': "Automatically find peaks"}, {'name': self.hitParam_algorithm_str, 'type': 'list', 'values': {self.hitParam_algorithm2_str: 2, self.hitParam_algorithm1_str: 1, self.hitParam_algorithm0_str: 0}, 'value': self.algorithm}, {'name': self.hitParam_parameters_str, 'visible': True, 'expanded': False, 'type': 'str', 'value': "", 'readonly': True, 'children': [ {'name': self.hitParam_alg1_npix_min_str, 'type': 'float', 'value': self.hitParam_alg1_npix_min, 'tip': "Only keep the peak if number of pixels above thr_low is above this value"}, {'name': self.hitParam_alg1_npix_max_str, 'type': 'float', 'value': self.hitParam_alg1_npix_max, 'tip': "Only keep the peak if number of pixels above thr_low is below this value"}, {'name': self.hitParam_alg1_amax_thr_str, 'type': 'float', 'decimals': 7, 'value': self.hitParam_alg1_amax_thr, 'tip': "Only keep the peak if max value is above this value"}, {'name': self.hitParam_alg1_atot_thr_str, 'type': 'float', 'decimals': 7, 'value': self.hitParam_alg1_atot_thr, 'tip': "Only keep the peak if integral inside region of interest is above this value"}, {'name': self.hitParam_alg1_son_min_str, 'type': 'float', 'value': self.hitParam_alg1_son_min, 'tip': "Only keep the peak if signal-over-noise is above this value"}, {'name': self.hitParam_alg1_thr_low_str, 'type': 'float', 'decimals': 7, 'value': self.hitParam_alg1_thr_low, 'tip': "Grow a seed peak if above this value"}, {'name': self.hitParam_alg1_thr_high_str, 'type': 'float', 'decimals': 7, 'value': self.hitParam_alg1_thr_high, 'tip': "Start a seed peak if above this value"}, {'name': self.hitParam_alg1_rank_str, 'type': 'int', 'value': self.hitParam_alg1_rank, 'tip': "region of integration is a square, (2r+1)x(2r+1)"}, {'name': self.hitParam_alg1_radius_str, 'type': 'int', 'value': self.hitParam_alg1_radius, 'tip': "region inside the region of interest"}, {'name': self.hitParam_alg1_dr_str, 'type': 'float', 'value': self.hitParam_alg1_dr, 'tip': "background region outside the region of interest"}, ]}, {'name': self.save_minPeaks_str, 'type': 'int', 'decimals': 7, 'value': self.minPeaks, 'tip': "Index only if there are more Bragg peaks found"}, {'name': self.save_maxPeaks_str, 'type': 'int', 'decimals': 7, 'value': self.maxPeaks, 'tip': "Index only if there are less Bragg peaks found"}, {'name': self.save_minRes_str, 'type': 'int', 'decimals': 7, 'value': self.minRes, 'tip': "Index only if Bragg peak resolution is at least this"}, {'name': self.save_sample_str, 'type': 'str', 'value': self.sample, 'tip': "Sample name saved inside cxi"}, {'name': self.tag_str, 'type': 'str', 'value': self.tag, 'tip': "attach tag to cxi, e.g. cxitut13_0010_tag.cxi"}, {'name': self.hitParam_outDir_str, 'type': 'str', 'value': self.hitParam_outDir}, {'name': self.hitParam_runs_str, 'type': 'str', 'value': self.hitParam_runs}, {'name': self.hitParam_queue_str, 'type': 'list', 'values': {self.hitParam_psfehhiprioq_str: 'psfehhiprioq', self.hitParam_psnehhiprioq_str: 'psnehhiprioq', self.hitParam_psfehprioq_str: 'psfehprioq', self.hitParam_psnehprioq_str: 'psnehprioq', self.hitParam_psfehq_str: 'psfehq', self.hitParam_psnehq_str: 'psnehq', self.hitParam_psanaq_str: 'psanaq', self.hitParam_psdebugq_str: 'psdebugq', self.hitParam_psanagpuq_str: 'psanagpuq', self.hitParam_psanaidleq_str: 'psanaidleq'}, 'value': self.hitParam_queue, 'tip': "Choose queue"}, {'name': self.hitParam_cpu_str, 'type': 'int', 'decimals': 7, 'value': self.hitParam_cpus}, {'name': self.hitParam_noe_str, 'type': 'int', 'decimals': 7, 'value': self.hitParam_noe, 'tip': "number of events to process, default=-1 means process all events"}, {'name': self.hitParam_extra_str, 'type': 'str', 'value': self.hitParam_extra, 'tip': "Extra peak finding flags"}, {'name': self.hitParam_launch_str, 'type': 'action'}, #{'name': self.hitParam_launchAuto_str, 'type': 'action'}, ]}, ] self.p3 = Parameter.create(name='paramsPeakFinder', type='group', \ children=self.params, expanded=True) self.win.setParameters(self.p3, showTop=False) self.p3.sigTreeStateChanged.connect(self.change) #self.parent.connect(self.launchBtn, QtCore.SIGNAL("clicked()"), self.findPeaks) def digestRunList(self, runList): runsToDo = [] if not runList: print("Run(s) is empty. Please type in the run number(s).") return runsToDo runLists = str(runList).split(",") for list in runLists: temp = list.split(":") if len(temp) == 2: for i in np.arange(int(temp[0]),int(temp[1])+1): runsToDo.append(i) elif len(temp) == 1: runsToDo.append(int(temp[0])) return runsToDo def updateParam(self): if self.userUpdate is None: if self.parent.psocakeRunDir is not None: peakParamFname = self.parent.psocakeRunDir + '/peakParam' if self.tag: peakParamFname += '_' + self.tag peakParamFname += '.json' if os.path.exists(peakParamFname): with open(peakParamFname) as infile: d = json.load(infile) if d[self.hitParam_algorithm_str] == 1 or d[self.hitParam_algorithm_str] == 2: # Update variables try: self.hitParam_alg1_npix_min = d[self.hitParam_alg1_npix_min_str] self.hitParam_alg1_npix_max = d[self.hitParam_alg1_npix_max_str] self.hitParam_alg1_amax_thr = d[self.hitParam_alg1_amax_thr_str] self.hitParam_alg1_atot_thr = d[self.hitParam_alg1_atot_thr_str] self.hitParam_alg1_son_min = d[self.hitParam_alg1_son_min_str] self.hitParam_alg1_thr_low = d[self.hitParam_alg1_thr_low_str] self.hitParam_alg1_thr_high = d[self.hitParam_alg1_thr_high_str] self.hitParam_alg1_rank = int(d[self.hitParam_alg1_rank_str]) self.hitParam_alg1_radius = int(d[self.hitParam_alg1_radius_str]) self.hitParam_alg1_dr = d[self.hitParam_alg1_dr_str] # Update GUI self.doingUpdate = True #self.p3.param(self.hitParam_grp, self.hitParam_algorithm_str).setValue(self.algorithm) self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_npix_min_str).setValue( self.hitParam_alg1_npix_min) self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_npix_max_str).setValue( self.hitParam_alg1_npix_max) self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_amax_thr_str).setValue( self.hitParam_alg1_amax_thr) self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_atot_thr_str).setValue( self.hitParam_alg1_atot_thr) self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_son_min_str).setValue( self.hitParam_alg1_son_min) self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_thr_low_str).setValue( self.hitParam_alg1_thr_low) self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_thr_high_str).setValue( self.hitParam_alg1_thr_high) self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_rank_str).setValue( self.hitParam_alg1_rank) self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_radius_str).setValue( self.hitParam_alg1_radius) self.doingUpdate = False self.p3.param(self.hitParam_grp, self.hitParam_parameters_str, self.hitParam_alg1_dr_str).setValue( self.hitParam_alg1_dr) except: pass def writeStatus(self, fname, d): json.dump(d, open(fname, 'w')) # Launch peak finding def findPeaks(self): self.parent.autoPeakFinding = self.turnOnAutoPeaks self.parent.thread.append(LaunchPeakFinder.LaunchPeakFinder(self.parent)) # send parent parameters with self self.parent.thread[self.parent.threadCounter].launch(self.parent.experimentName, self.parent.detInfo) self.parent.threadCounter+=1 # Save peak finding parameters runsToDo = self.digestRunList(self.hitParam_runs) for run in runsToDo: peakParamFname = self.parent.psocakeDir+'/r'+str(run).zfill(4)+'/peakParam' if self.tag: peakParamFname += '_'+self.tag peakParamFname += '.json' d = {self.hitParam_algorithm_str: self.algorithm, self.hitParam_alg1_npix_min_str: self.hitParam_alg1_npix_min, self.hitParam_alg1_npix_max_str: self.hitParam_alg1_npix_max, self.hitParam_alg1_amax_thr_str: self.hitParam_alg1_amax_thr, self.hitParam_alg1_atot_thr_str: self.hitParam_alg1_atot_thr, self.hitParam_alg1_son_min_str: self.hitParam_alg1_son_min, self.hitParam_alg1_thr_low_str: self.hitParam_alg1_thr_low, self.hitParam_alg1_thr_high_str: self.hitParam_alg1_thr_high, self.hitParam_alg1_rank_str: self.hitParam_alg1_rank, self.hitParam_alg1_radius_str: self.hitParam_alg1_radius, self.hitParam_alg1_dr_str: self.hitParam_alg1_dr} if not os.path.exists(self.parent.psocakeDir+'/r'+str(run).zfill(4)): os.mkdir(self.parent.psocakeDir+'/r'+str(run).zfill(4)) self.writeStatus(peakParamFname, d) # If anything changes in the parameter tree, print a message def change(self, panel, changes): for param, change, data in changes: path = panel.childPath(param) if self.parent.args.v >= 1: print(' path: %s' % path) print(' change: %s' % change) print(' data: %s' % str(data)) print(' ----------') self.paramUpdate(path, change, data) ############################## # Mandatory parameter update # ############################## def paramUpdate(self, path, change, data): if path[0] == self.hitParam_grp: if path[1] == self.hitParam_algorithm_str: self.algInitDone = False self.updateAlgorithm(data) elif path[1] == self.hitParam_showPeaks_str: self.showPeaks = data if self.showPeaks and self.doingUpdate is False: self.updateClassification() if not self.showPeaks: print("no need to index") self.peaks = None self.numPeaksFound = 0 self.drawPeaks() self.parent.index.updateIndex() elif path[1] == self.hitParam_autoPeaks_str: ########################### self.turnOnAutoPeaks = data if self.showPeaks and self.doingUpdate is False: self.updateClassification() ########################### elif path[1] == self.hitParam_outDir_str: self.hitParam_outDir = data self.hitParam_outDir_overridden = True elif path[1] == self.hitParam_runs_str: self.hitParam_runs = data elif path[1] == self.hitParam_queue_str: self.hitParam_queue = data elif path[1] == self.hitParam_cpu_str: self.hitParam_cpus = data elif path[1] == self.hitParam_noe_str: self.hitParam_noe = data elif path[1] == self.hitParam_threshold_str: self.hitParam_threshold = data elif path[1] == self.hitParam_launch_str: self.findPeaks() elif path[1] == self.save_minPeaks_str: self.minPeaks = data elif path[1] == self.save_maxPeaks_str: self.maxPeaks = data elif path[1] == self.save_minRes_str: self.minRes = data elif path[1] == self.save_sample_str: self.sample = data elif path[1] == self.tag_str: self.updateTag(data) elif path[1] == self.hitParam_extra_str: self.hitParam_extra = data elif path[2] == self.hitParam_alg1_npix_min_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_npix_min = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() elif path[2] == self.hitParam_alg1_npix_max_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_npix_max = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() elif path[2] == self.hitParam_alg1_amax_thr_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_amax_thr = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() elif path[2] == self.hitParam_alg1_atot_thr_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_atot_thr = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() elif path[2] == self.hitParam_alg1_son_min_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_son_min = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() elif path[2] == self.hitParam_alg1_thr_low_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_thr_low = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() elif path[2] == self.hitParam_alg1_thr_high_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_thr_high = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() elif path[2] == self.hitParam_alg1_rank_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_rank = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() elif path[2] == self.hitParam_alg1_radius_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_radius = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() elif path[2] == self.hitParam_alg1_dr_str and path[1] == self.hitParam_parameters_str: self.hitParam_alg1_dr = data self.algInitDone = False self.userUpdate = True if self.showPeaks and self.doingUpdate is False: self.updateClassification() def updateAlgorithm(self, data): self.algorithm = data self.algInitDone = False self.updateClassification() if self.parent.args.v >= 1: print("##### Done updateAlgorithm: ", self.algorithm) def updateTag(self, data): if self.tag: fname = self.parent.small.quantifier_filename.split("_"+self.tag+".cxi")[0] else: fname = self.parent.small.quantifier_filename.split(".cxi")[0] if data: fname += "_" + data + ".cxi" else: fname += ".cxi" self.tag = data self.parent.small.pSmall.param(self.parent.small.quantifier_grp, self.parent.small.quantifier_filename_str).setValue(fname) self.parent.small.reloadQuantifier() def saveCheetahFormat(self, arg): dim0, dim1 = utils.getCheetahDim(self.parent.detInfo) if dim0 > 0: maxNumPeaks = 2048 if self.parent.index.hiddenCXI is not None and self.peaks.shape[0] >= self.minPeaks: myHdf5 = h5py.File(self.parent.index.hiddenCXI, 'w') grpName = "/entry_1/result_1" dset_nPeaks = "/nPeaks" dset_posX = "/peakXPosRaw" dset_posY = "/peakYPosRaw" dset_atot = "/peakTotalIntensity" if grpName in myHdf5: del myHdf5[grpName] myHdf5.create_group(grpName) myHdf5.create_dataset(grpName + dset_nPeaks, (1,), dtype='int') myHdf5.create_dataset(grpName + dset_posX, (1, maxNumPeaks), dtype='float32', chunks=(1, maxNumPeaks)) myHdf5.create_dataset(grpName + dset_posY, (1, maxNumPeaks), dtype='float32', chunks=(1, maxNumPeaks)) myHdf5.create_dataset(grpName + dset_atot, (1, maxNumPeaks), dtype='float32', chunks=(1, maxNumPeaks)) myHdf5.create_dataset("/LCLS/detector_1/EncoderValue", (1,), dtype=float) myHdf5.create_dataset("/LCLS/photon_energy_eV", (1,), dtype=float) dset = myHdf5.create_dataset("/entry_1/data_1/data", (1, dim0, dim1), dtype=float) dsetM = myHdf5.create_dataset("/entry_1/data_1/mask", (dim0, dim1), dtype='int') # Convert calib image to cheetah image img = utils.pct(self.parent.detInfo, self.parent.calib) mask = utils.pct(self.parent.detInfo, self.parent.mk.combinedMask) dset[0, :, :] = img dsetM[:, :] = mask peaks = self.peaks.copy() nPeaks = peaks.shape[0] if nPeaks > maxNumPeaks: peaks = peaks[:maxNumPeaks] nPeaks = maxNumPeaks segs = peaks[:, 0] rows = peaks[:, 1] cols = peaks[:, 2] atots = peaks[:, 5] cheetahRows, cheetahCols = utils.convert_peaks_to_cheetah(self.parent.detInfo, segs, rows, cols) myHdf5[grpName + dset_posX][0, :nPeaks] = cheetahCols myHdf5[grpName + dset_posY][0, :nPeaks] = cheetahRows myHdf5[grpName + dset_atot][0, :nPeaks] = atots #for i, peak in enumerate(peaks): # seg, row, col, npix, amax, atot, rcent, ccent, rsigma, csigma, rmin, rmax, cmin, cmax, bkgd, rms, son = peak[0:17] # cheetahRow, cheetahCol = utils.convert_peaks_to_cheetah(self.parent.detInfo, seg, row, col) # myHdf5[grpName + dset_posX][0, i] = cheetahCol # myHdf5[grpName + dset_posY][0, i] = cheetahRow # myHdf5[grpName + dset_atot][0, i] = atot myHdf5[grpName + dset_nPeaks][0] = nPeaks if self.parent.args.v >= 1: print("hiddenCXI clen (mm): ", self.parent.clen * 1000.) myHdf5["/LCLS/detector_1/EncoderValue"][0] = self.parent.clen * 1000. # mm myHdf5["/LCLS/photon_energy_eV"][0] = self.parent.photonEnergy myHdf5.close() def updateClassification(self): if self.parent.calib is not None: if self.parent.mk.streakMaskOn: self.parent.mk.initMask() self.parent.mk.streakMask = self.parent.mk.StreakMask.getStreakMaskCalib(self.parent.evt) if self.parent.mk.streakMask is None: self.parent.mk.streakMaskAssem = None else: self.parent.mk.streakMaskAssem = self.parent.det.image(self.parent.evt, self.parent.mk.streakMask) self.algInitDone = False self.parent.mk.displayMask() # update combined mask self.parent.mk.combinedMask = np.ones_like(self.parent.calib) if self.parent.mk.streakMask is not None and self.parent.mk.streakMaskOn is True: self.parent.mk.combinedMask *= self.parent.mk.streakMask if self.parent.mk.userMask is not None and self.parent.mk.userMaskOn is True: self.parent.mk.combinedMask *= self.parent.mk.userMask if self.parent.mk.psanaMask is not None and self.parent.mk.psanaMaskOn is True: self.parent.mk.combinedMask *= self.parent.mk.psanaMask # Peak output (0-16): # 0 seg # 1 row # 2 col # 3 npix: no. of pixels in the ROI intensities above threshold # 4 amp_max: max intensity # 5 amp_tot: sum of intensities # 6,7: row_cgrav: center of mass # 8,9: row_sigma # 10,11,12,13: minimum bounding box # 14: background # 15: noise # 16: signal over noise if self.algorithm == 0: # No peak algorithm self.peaks = None self.drawPeaks() else: # Only initialize the hit finder algorithm once if self.algInitDone is False: self.windows = None self.alg = [] # set peak-selector parameters: if self.algorithm == 1: self.alg = PyAlgos(mask=None, pbits=0) self.peakRadius = int(self.hitParam_alg1_radius) self.alg.set_peak_selection_pars(npix_min=self.hitParam_alg1_npix_min, npix_max=self.hitParam_alg1_npix_max, \ amax_thr=self.hitParam_alg1_amax_thr, atot_thr=self.hitParam_alg1_atot_thr, \ son_min=self.hitParam_alg1_son_min) elif self.algorithm == 2: self.alg = PyAlgos(mask=None, pbits=0) self.peakRadius = int(self.hitParam_alg1_radius) self.alg.set_peak_selection_pars(npix_min=self.hitParam_alg1_npix_min, npix_max=self.hitParam_alg1_npix_max, \ amax_thr=self.hitParam_alg1_amax_thr, atot_thr=self.hitParam_alg1_atot_thr, \ son_min=self.hitParam_alg1_son_min) elif self.algorithm == 3: self.alg = PyAlgos(mask=None, pbits=0) self.peakRadius = int(self.hitParam_alg1_radius) self.alg.set_peak_selection_pars(npix_min=self.hitParam_alg1_npix_min, npix_max=self.hitParam_alg1_npix_max, \ amax_thr=self.hitParam_alg1_amax_thr, atot_thr=self.hitParam_alg1_atot_thr, \ son_min=self.hitParam_alg1_son_min) ix = self.parent.det.indexes_x(self.parent.evt) iy = self.parent.det.indexes_y(self.parent.evt) self.iX = np.array(ix, dtype=np.int64) self.iY = np.array(iy, dtype=np.int64) self.algInitDone = True self.parent.calib = self.parent.calib * 1.0 # Neccessary when int is returned if self.algorithm == 1: # v1 - aka Droplet Finder - two-threshold peak-finding algorithm in restricted region # around pixel with maximal intensity. if not self.turnOnAutoPeaks: self.peakRadius = int(self.hitParam_alg1_radius) self.peaks = self.alg.peak_finder_v4r3(self.parent.calib, thr_low=self.hitParam_alg1_thr_low, thr_high=self.hitParam_alg1_thr_high, rank=int(self.hitParam_alg1_rank), r0=self.peakRadius, dr=self.hitParam_alg1_dr, mask=self.parent.mk.combinedMask.astype(np.uint16)) else: ################################ # Determine thr_high and thr_low if self.ind is None: with pg.BusyCursor(): powderSumFname = self.parent.psocakeRunDir + '/' + \ self.parent.experimentName + '_' + \ str(self.parent.runNumber).zfill(4) + '_' + \ self.parent.detInfo + '_mean.npy' if not os.path.exists(powderSumFname): # Generate powder cmd = "mpirun -n 6 generatePowder exp=" + self.parent.experimentName + \ ":run=" + str(self.parent.runNumber) + " -d " + self.parent.detInfo + \ " -o " + self.parent.psocakeRunDir cmd += " -n " + str(256) cmd += " --random" print("Running on local machine: ", cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = process.communicate() # Copy as background.npy import shutil shutil.copyfile(powderSumFname, self.parent.psocakeRunDir + '/background.npy') # Read in powder pattern and calculate pixel indices powderSum = np.load(powderSumFname) powderSum1D = powderSum.ravel() cy, cx = self.parent.det.indexes_xy(self.parent.evt) #ipx, ipy = self.parent.det.point_indexes(self.parent.evt, pxy_um=(0, 0)) try: ipy, ipx = self.parent.det.point_indexes(self.parent.evt, pxy_um=(0, 0), pix_scale_size_um=None, xy0_off_pix=None, cframe=gu.CFRAME_PSANA, fract=True) except AttributeError: ipx, ipy = self.parent.det.point_indexes(self.parent.evt, pxy_um=(0, 0)) r = np.sqrt((cx - ipx) ** 2 + (cy - ipy) ** 2).ravel().astype(int) startR = 0 endR = np.max(r) profile = np.zeros(endR - startR, ) for i, val in enumerate(np.arange(startR, endR)): ind = np.where(r == val)[0].astype(int) if len(ind) > 0: profile[i] = np.mean(powderSum1D[ind]) myThreshInd = np.argmax(profile) print("###################################################") print("Solution scattering radius (pixels): ", myThreshInd) print("###################################################") thickness = 10 indLo = np.where(r >= myThreshInd - thickness / 2.)[0].astype(int) indHi = np.where(r <= myThreshInd + thickness / 2.)[0].astype(int) self.ind = np.intersect1d(indLo, indHi) ix = self.parent.det.indexes_x(self.parent.evt) iy = self.parent.det.indexes_y(self.parent.evt) self.iX = np.array(ix, dtype=np.int64) self.iY = np.array(iy, dtype=np.int64) calib1D = self.parent.calib.ravel() mean = np.mean(calib1D[self.ind]) spread = np.std(calib1D[self.ind]) highSigma = 3.5 lowSigma = 2.5 thr_high = int(mean + highSigma * spread + 50) thr_low = int(mean + lowSigma * spread + 50) self.peaks = self.alg.findPeaks(self.parent.calib, npix_min=self.hitParam_alg1_npix_min, npix_max=self.hitParam_alg1_npix_max, atot_thr=0, son_min=self.hitParam_alg1_son_min, thr_low=thr_low, thr_high=thr_high, mask=self.parent.mk.combinedMask.astype(np.uint16)) elif self.algorithm == 2: # v2 - aka Adaptive peak finder self.peaks = self.alg.peak_finder_v3r3(self.parent.calib, rank=int(self.hitParam_alg1_rank), r0=self.peakRadius, dr=self.hitParam_alg1_dr, nsigm=self.hitParam_alg1_son_min, mask=self.parent.mk.combinedMask.astype(np.uint16))#)#thr=self.hitParam_alg2_thr, r0=self.peakRadius, dr=self.hitParam_alg2_dr) elif self.algorithm == 3: # perform binning here binr = 2 binc = 2 downCalib = sm.block_reduce(self.parent.calib, block_size=(1, binr, binc), func=np.sum) downWeight = sm.block_reduce(self.parent.mk.combinedMask, block_size=(1, binr, binc), func=np.sum) warr = np.zeros_like(downCalib, dtype='float32') ind = np.where(downWeight > 0) warr[ind] = downCalib[ind] / downWeight[ind] upCalib = utils.upsample(warr, self.parent.calib.shape, binr, binc) self.peaks = self.alg.peak_finder_v3r3(upCalib, rank=int(self.hitParam_alg1_rank), r0=self.peakRadius, dr=self.hitParam_alg1_dr, nsigm=self.hitParam_alg1_son_min, mask=self.parent.mk.combinedMask.astype(np.uint16)) self.numPeaksFound = self.peaks.shape[0] if self.numPeaksFound > self.minPeaks and self.numPeaksFound < self.maxPeaks:# and self.turnOnAutoPeaks: cenX = self.iX[np.array(self.peaks[:, 0], dtype=np.int64), np.array(self.peaks[:, 1], dtype=np.int64), np.array(self.peaks[:, 2], dtype=np.int64)] + 0.5 cenY = self.iY[np.array(self.peaks[:, 0], dtype=np.int64), np.array(self.peaks[:, 1], dtype=np.int64), np.array(self.peaks[:, 2], dtype=np.int64)] + 0.5 x = cenX - self.parent.cx # args.center[0] y = cenY - self.parent.cy # args.center[1] pixSize = float(self.parent.det.pixel_size(self.parent.evt)) detdis = float(self.parent.detectorDistance) z = detdis / pixSize * np.ones(x.shape) # pixels wavelength = 12.407002 / float(self.parent.photonEnergy) # Angstrom norm = np.sqrt(x ** 2 + y ** 2 + z ** 2) qPeaks = (np.array([x, y, z]) / norm - np.array([[0.], [0.], [1.]])) / wavelength [meanClosestNeighborDist, self.pairsFoundPerSpot] = self.calculate_likelihood(qPeaks) else: self.pairsFoundPerSpot = 0 if self.parent.args.v >= 1: print("Num peaks found: ", self.numPeaksFound, self.peaks.shape, self.pairsFoundPerSpot) # update clen self.parent.geom.updateClen(self.parent.facility) self.parent.index.clearIndexedPeaks() # Save image and peaks in cheetah cxi file self.saveCheetahFormat(self.parent.facility) if self.parent.index.showIndexedPeaks: self.parent.index.updateIndex() self.drawPeaks() if self.parent.args.v >= 1: print("Done updateClassification") def calculate_likelihood(self, qPeaks): nPeaks = int(qPeaks.shape[1]) selfD = distance.cdist(qPeaks.transpose(), qPeaks.transpose(), 'euclidean') sortedSelfD = np.sort(selfD) closestNeighborDist = sortedSelfD[:, 1] meanClosestNeighborDist = np.median(closestNeighborDist) closestPeaks = [None] * nPeaks coords = qPeaks.transpose() pairsFound = 0. for ii in range(nPeaks): index = np.where(selfD[ii, :] == closestNeighborDist[ii]) closestPeaks[ii] = coords[list(index[0]), :].copy() p = coords[ii, :] flip = 2 * p - closestPeaks[ii] d = distance.cdist(coords, flip, 'euclidean') sigma = closestNeighborDist[ii] / 4. mu = 0. bins = d vals = np.exp(-(bins - mu) ** 2 / (2. * sigma ** 2)) weight = np.sum(vals) pairsFound += weight pairsFound = pairsFound / 2. pairsFoundPerSpot = pairsFound / float(nPeaks) return [meanClosestNeighborDist, pairsFoundPerSpot] def getMaxRes(self, posX, posY, centerX, centerY): maxRes = np.max(np.sqrt((posX-centerX)**2 + (posY-centerY)**2)) if self.parent.args.v >= 1: print("maxRes: ", maxRes) return maxRes # in pixels def assemblePeakPos(self, peaks): self.ix = self.parent.det.indexes_x(self.parent.evt) self.iy = self.parent.det.indexes_y(self.parent.evt) if self.ix is None: (_, dim0, dim1) = self.parent.calib.shape self.iy = np.tile(np.arange(dim0), [dim1, 1]) self.ix = np.transpose(self.iy) self.iX = np.array(self.ix, dtype=np.int64) self.iY = np.array(self.iy, dtype=np.int64) if len(self.iX.shape) == 2: self.iX = np.expand_dims(self.iX, axis=0) self.iY = np.expand_dims(self.iY, axis=0) cenX = self.iX[np.array(peaks[:, 0], dtype=np.int64), np.array(peaks[:, 1], dtype=np.int64), np.array( peaks[:, 2], dtype=np.int64)] + 0.5 cenY = self.iY[np.array(peaks[:, 0], dtype=np.int64), np.array(peaks[:, 1], dtype=np.int64), np.array( peaks[:, 2], dtype=np.int64)] + 0.5 return cenX, cenY def drawPeaks(self): self.parent.img.clearPeakMessage() if self.showPeaks: if self.peaks is not None and self.numPeaksFound > 0: cenX, cenY = self.assemblePeakPos(self.peaks) self.peaksMaxRes = self.getMaxRes(cenX, cenY, self.parent.cx, self.parent.cy) diameter = self.peakRadius*2+1 self.parent.img.peak_feature.setData(cenY, cenX, symbol='s', \ size=diameter, brush=(255,255,255,0), \ pen=pg.mkPen({'color': "c", 'width': 4}), pxMode=False) #FF0 # Write number of peaks found xMargin = -self.parent.data.shape[0]#5 # pixels yMargin = 5#0 # pixels maxX = np.max(self.ix) + xMargin maxY = np.max(self.iy) - yMargin myMessage = '<div style="text-align: center"><span style="color: cyan; font-size: 12pt;">Peaks=' + \ str(self.numPeaksFound) + \ ' <br>Res=' + str(int(self.peaksMaxRes)) + \ ' <br>Like=' + str(round(self.pairsFoundPerSpot,3)) + \ '<br></span></div>' self.parent.img.peak_text = pg.TextItem(html=myMessage, anchor=(0, 0)) self.parent.img.win.getView().addItem(self.parent.img.peak_text) self.parent.img.peak_text.setPos(maxY, maxX) else: self.parent.img.peak_feature.setData([], [], pxMode=False) self.parent.img.peak_text = pg.TextItem(html='', anchor=(0, 0)) self.parent.img.win.getView().addItem(self.parent.img.peak_text) self.parent.img.peak_text.setPos(0,0) else: self.parent.img.peak_feature.setData([], [], pxMode=False) if self.hitParam_extra.endswith('.cxi'): # Draw peaks from a cxi file # Read cxi file containing peaks f = h5py.File(self.hitParam_extra) npeaks = f['/entry_1/result_1/nPeaksAll'][self.parent.eventNumber] row2d = f['/entry_1/result_1/peakYPosRawAll'][self.parent.eventNumber, :npeaks] col2d = f['/entry_1/result_1/peakXPosRawAll'][self.parent.eventNumber, :npeaks] f.close() # Convert cheetah peaks to psana peaks s, r, c = utils.convert_peaks_to_psana(self.parent.detInfo, row2d, col2d) peaks = np.array([s, r, c]).T # Display peaks if peaks is not None and len(peaks) > 0: if self.parent.facility == self.parent.facilityLCLS: cenX, cenY = self.assemblePeakPos(peaks) diameter = int(self.hitParam_alg1_radius) * 2 + 1 self.parent.img.filePeak_feature.setData(cenY, cenX, symbol='s', \ size=diameter, brush=(255, 255, 255, 0), \ pen=pg.mkPen({'color': "y", 'width': 2}), pxMode=False) # FF0 else: self.parent.img.filePeak_feature.setData([], [], pxMode=False) else: self.parent.img.filePeak_feature.setData([], [], pxMode=False) if self.parent.args.v >= 1: print("Done drawPeaks") self.parent.geom.drawCentre()
amax_thr=hitParam_alg1_amax_thr, atot_thr=hitParam_alg1_atot_thr, \ son_min=hitParam_alg1_son_min) # Start your translate_xtc_demo.py before you start this script context = zmq.Context() zmq_socket = context.socket(zmq.PUSH) zmq_socket.bind("tcp://127.0.0.1:5557") for i, evt in enumerate(ds.events()): if i == nevents: break raw = det.raw(evt) calib = det.calib(evt) peaks = alg.peak_finder_v3r3(calib, rank=int(hitParam_alg1_rank), r0=peakRadius, dr=hitParam_alg1_dr, nsigm=hitParam_alg1_son_min, mask=mask.astype(np.uint16)) npeaks = peaks.shape[0] ebeam = ebeamDet.get(evt) photonEnergy = epics.value('SIOC:SYS0:ML00:AO541') evtId = evt.get(EventId) sec = evtId.time()[0] nsec = evtId.time()[1] timestamp = (sec << 32) | nsec if raw is not None and npeaks >= minPeaks: print("Event: ", i) evtDict = {}
class PeakFinder: def __init__(self, exp, run, detname, evt, detector, algorithm, hitParam_alg_npix_min, hitParam_alg_npix_max, hitParam_alg_amax_thr, hitParam_alg_atot_thr, hitParam_alg_son_min, streakMask_on, streakMask_sigma, streakMask_width, userMask_path, psanaMask_on, psanaMask_calib, psanaMask_status, psanaMask_edges, psanaMask_central, psanaMask_unbond, psanaMask_unbondnrs, medianFilterOn=0, medianRank=5, radialFilterOn=0, distance=0.0, windows=None, **kwargs): self.exp = exp self.run = run self.detname = detname self.det = detector self.algorithm = algorithm self.maxRes = 0 self.npix_min = hitParam_alg_npix_min self.npix_max = hitParam_alg_npix_max self.amax_thr = hitParam_alg_amax_thr self.atot_thr = hitParam_alg_atot_thr self.son_min = hitParam_alg_son_min self.streakMask_on = str2bool(streakMask_on) self.streakMask_sigma = streakMask_sigma self.streakMask_width = streakMask_width self.userMask_path = userMask_path self.psanaMask_on = str2bool(psanaMask_on) self.psanaMask_calib = str2bool(psanaMask_calib) self.psanaMask_status = str2bool(psanaMask_status) self.psanaMask_edges = str2bool(psanaMask_edges) self.psanaMask_central = str2bool(psanaMask_central) self.psanaMask_unbond = str2bool(psanaMask_unbond) self.psanaMask_unbondnrs = str2bool(psanaMask_unbondnrs) self.medianFilterOn = medianFilterOn self.medianRank = medianRank self.radialFilterOn = radialFilterOn self.distance = distance self.windows = windows self.userMask = None self.psanaMask = None self.streakMask = None self.userPsanaMask = None self.combinedMask = None # Make user mask if self.userMask_path is not None: self.userMask = np.load(self.userMask_path) if facility == 'LCLS': # Make psana mask self.psanaMask = detector.mask(run, calib=self.psanaMask_calib, status=self.psanaMask_status, edges=self.psanaMask_edges, central=self.psanaMask_central, unbond=self.psanaMask_unbond, unbondnbrs=self.psanaMask_unbondnrs) # Combine userMask and psanaMask self.userPsanaMask = np.ones_like(self.psanaMask, dtype=np.int16) if self.userMask is not None: self.userPsanaMask *= self.userMask if self.psanaMask_on: self.userPsanaMask *= self.psanaMask # Powder of hits and misses self.powderHits = None self.powderMisses = None # set algorithm specific parameters if algorithm == 1: self.hitParam_alg1_thr_low = kwargs["alg1_thr_low"] self.hitParam_alg1_thr_high = kwargs["alg1_thr_high"] self.hitParam_alg1_rank = int(kwargs["alg1_rank"]) self.hitParam_alg1_radius = int(kwargs["alg1_radius"]) self.hitParam_alg1_dr = kwargs["alg1_dr"] elif algorithm >= 2: self.hitParam_alg1_thr_low = kwargs["alg1_thr_low"] self.hitParam_alg1_thr_high = kwargs["alg1_thr_high"] self.hitParam_alg1_rank = int(kwargs["alg1_rank"]) self.hitParam_alg1_radius = int(kwargs["alg1_radius"]) self.hitParam_alg1_dr = kwargs["alg1_dr"] if facility == 'LCLS': self.access = kwargs["access"] if self.algorithm == 1: self.alg = PyAlgos(mask=None, pbits=0) self.peakRadius = int(self.hitParam_alg1_radius) self.alg.set_peak_selection_pars(npix_min=self.npix_min, npix_max=self.npix_max, \ amax_thr=self.amax_thr, atot_thr=self.atot_thr, \ son_min=self.son_min) elif self.algorithm >= 2: self.alg = PyAlgos(mask=None, pbits=0) self.peakRadius = int(self.hitParam_alg1_radius) self.alg.set_peak_selection_pars(npix_min=self.npix_min, npix_max=self.npix_max, \ amax_thr=self.amax_thr, atot_thr=self.atot_thr, \ son_min=self.son_min) if facility == 'LCLS': self.StreakMask = myskbeam.StreakMask(self.det, evt, width=self.streakMask_width, sigma=self.streakMask_sigma) #self.cx, self.cy = self.det.point_indexes(evt, pxy_um=(0, 0)) try: self.cy, self.cx = self.det.point_indexes( evt, pxy_um=(0, 0), pix_scale_size_um=None, xy0_off_pix=None, cframe=gu.CFRAME_PSANA, fract=True) except AttributeError: self.cx, self.cy = self.det.point_indexes(evt, pxy_um=(0, 0)) self.iX = np.array(self.det.indexes_x(evt), dtype=np.int64) self.iY = np.array(self.det.indexes_y(evt), dtype=np.int64) if len(self.iX.shape) == 2: self.iX = np.expand_dims(self.iX, axis=0) self.iY = np.expand_dims(self.iY, axis=0) # Initialize radial background subtraction self.setupExperiment() if self.radialFilterOn: self.setupRadialBackground() self.updatePolarizationFactor() def setupExperiment(self): access = 'exp=' + str(self.exp) + ':run=' + str(self.run) + ':idx' if 'ffb' in self.access.lower(): access += ':dir=/reg/d/ffb/' + self.exp[:3] + '/' + self.exp + '/xtc' self.ds = psana.DataSource(access) self.run = next(self.ds.runs()) self.times = self.run.times() self.eventTotal = len(self.times) self.env = self.ds.env() self.evt = self.run.event(self.times[0]) self.det = psana.Detector(str(self.detname), self.env) self.det.do_reshape_2d_to_3d(flag=True) def setupRadialBackground(self): self.geo = self.det.geometry( self.run ) # self.geo = GeometryAccess(self.parent.geom.calibPath+'/'+self.parent.geom.calibFile) self.xarr, self.yarr, self.zarr = self.geo.get_pixel_coords() self.ix = self.det.indexes_x(self.evt) self.iy = self.det.indexes_y(self.evt) if self.ix is None: self.iy = np.tile(np.arange(self.userMask.shape[1]), [self.userMask.shape[2], 1]) self.ix = np.transpose(self.iy) self.iX = np.array(self.ix, dtype=np.int64) self.iY = np.array(self.iy, dtype=np.int64) if len(self.iX.shape) == 2: self.iX = np.expand_dims(self.iX, axis=0) self.iY = np.expand_dims(self.iY, axis=0) self.mask = self.geo.get_pixel_mask( mbits=0o0377 ) # mask for 2x1 edges, two central columns, and unbound pixels with their neighbours self.rb = RadialBkgd(self.xarr, self.yarr, mask=self.mask, radedges=None, nradbins=100, phiedges=(0, 360), nphibins=1) def updatePolarizationFactor(self): # FIXME: handle vertical/horizontal polarization properly self.pf = polarization_factor(self.rb.pixel_rad(), self.rb.pixel_phi() + 90, self.distance * 1e6) # convert to um def findPeaks(self, calib, evt, minPeaks=15, thr_high=None, thr_low=None): #t0 = time.time() if facility == 'LCLS': if self.streakMask_on: # make new streak mask self.streakMask = self.StreakMask.getStreakMaskCalib(evt) # Apply background correction if self.medianFilterOn: calib -= median_filter_ndarr(calib, self.medianRank) if self.radialFilterOn: self.pf.shape = calib.shape # FIXME: shape is 1d calib = self.rb.subtract_bkgd(calib * self.pf) calib.shape = self.userPsanaMask.shape # FIXME: shape is 1d self.calib = calib # save background subtracted calib as an attribute if self.streakMask is not None: self.combinedMask = self.userPsanaMask * self.streakMask else: self.combinedMask = self.userPsanaMask #t1 = time.time() # set algorithm specific parameters if self.algorithm == 1: if facility == 'LCLS': # v1 - aka Droplet Finder - two-threshold peak-finding algorithm in restricted region # around pixel with maximal intensity. if thr_high is None: # use gui input self.peakRadius = int(self.hitParam_alg1_radius) self.peaks = self.alg.peak_finder_v4r3( calib, thr_low=self.hitParam_alg1_thr_low, thr_high=self.hitParam_alg1_thr_high, rank=self.hitParam_alg1_rank, r0=self.hitParam_alg1_radius, dr=self.hitParam_alg1_dr, mask=self.combinedMask.astype(np.uint16)) else: self.peaks = self.alg.findPeaks(calib, npix_min=self.npix_min, npix_max=self.npix_max, atot_thr=self.atot_thr, son_min=self.son_min, thr_low=thr_low, thr_high=thr_high, mask=self.combinedMask) elif self.algorithm == 2: if facility == 'LCLS': # Adaptive peak finder v3r3 self.peakRadius = int(self.hitParam_alg1_radius) self.peaks = self.alg.peak_finder_v3r3( calib, rank=int(self.hitParam_alg1_rank), r0=self.peakRadius, dr=self.hitParam_alg1_dr, nsigm=self.son_min, mask=self.combinedMask.astype(np.uint16)) elif self.algorithm == 3: if facility == 'LCLS': # perform binning here binr = 2 binc = 2 downCalib = sm.block_reduce(calib, block_size=(1, binr, binc), func=np.sum) downWeight = sm.block_reduce(self.combinedMask, block_size=(1, binr, binc), func=np.sum) warr = np.zeros_like(downCalib, dtype='float32') ind = np.where(downWeight > 0) warr[ind] = downCalib[ind] / downWeight[ind] upCalib = utils.upsample(warr, calib.shape, binr, binc) self.peakRadius = int(self.hitParam_alg1_radius) self.peaks = self.alg.peak_finder_v3r3( upCalib, rank=int(self.hitParam_alg1_rank), r0=self.peakRadius, dr=self.hitParam_alg1_dr, nsigm=self.son_min, mask=self.combinedMask.astype(np.uint16)) #t2 = time.time() self.numPeaksFound = self.peaks.shape[0] if self.numPeaksFound >= minPeaks: if facility == 'LCLS': cenX = self.iX[np.array(self.peaks[:, 0], dtype=np.int64), np.array(self.peaks[:, 1], dtype=np.int64), np.array(self.peaks[:, 2], dtype=np.int64)] + 0.5 cenY = self.iY[np.array(self.peaks[:, 0], dtype=np.int64), np.array(self.peaks[:, 1], dtype=np.int64), np.array(self.peaks[:, 2], dtype=np.int64)] + 0.5 self.maxRes = getMaxRes(cenX, cenY, self.cx, self.cy) else: self.maxRes = 0 #t3 = time.time() if self.numPeaksFound >= minPeaks: if self.powderHits is None: self.powderHits = calib else: self.powderHits = np.maximum(self.powderHits, calib) else: if self.powderMisses is None: self.powderMisses = calib else: self.powderMisses = np.maximum(self.powderMisses, calib) if self.powderHits is None: self.powderHits = np.zeros_like(calib) if self.powderMisses is None: self.powderMisses = np.zeros_like(calib)
evt = run.event(t) t5 = time.time() calib = det.calib(evt) if calib is None: print 'None' continue [seg, row, col] = calib.shape imgs = calib * mask img_h = imgs.shape[1] img_w = imgs.shape[2] t6 = time.time() # psana peak finder psana_peaks = alg.peak_finder_v3r3(imgs, rank=rank, r0=3, dr=2, nsigm=10, mask=mask) t7 = time.time() # peaknet peak finder _imgs = imgs.reshape(-1, 1, imgs.shape[1], imgs.shape[2]) results = peaknet.predict(_imgs, conf_thresh=0.5) t8 = time.time() peaknet_peaks = conv_peaks_to_psana(results, img_h, img_w) print("Get evt {0:.3f} sec".format(t5 - t4)) print("det.calib {0:.3f} sec".format(t6 - t5)) print("psana peaks {0:.3f} sec".format(t7 - t6)) print("peaknet peaks {0:.3f} sec".format(t8 - t7))
def test_pf(tname): ##----------------------------- PF = V4 # default if tname == '1': PF = V1 if tname == '2': PF = V2 if tname == '3': PF = V3 if tname == '4': PF = V4 SKIP = 0 EVTMAX = 5 + SKIP DO_PLOT_IMAGE = True DO_PLOT_PIXEL_STATUS = False #True if PF in (V2,V4) else False DO_PLOT_CONNECED_PIXELS = False #True if PF in (V2,V3,V4) else False DO_PLOT_LOCAL_MAXIMUMS = False #True if PF == V3 else False DO_PLOT_LOCAL_MINIMUMS = False #True if PF == V3 else False shape = (1000, 1000) # Pixel image indexes #arr3d = np.array((1,shape[0],shape[1])) INDS = np.indices((1, shape[0], shape[1]), dtype=np.int64) imRow, imCol = INDS[1, :], INDS[2, :] #iX = np.array(det.indexes_x(evt), dtype=np.int64) #- xoffset #iY = np.array(det.indexes_y(evt), dtype=np.int64) #- yoffset fs = (8, 7) # (11,10) ##----------------------------- fig5, axim5, axcb5, imsh5 = gg.fig_axim_axcb_imsh( figsize=fs) if DO_PLOT_LOCAL_MINIMUMS else (None, None, None, None) fig4, axim4, axcb4, imsh4 = gg.fig_axim_axcb_imsh( figsize=fs) if DO_PLOT_LOCAL_MAXIMUMS else (None, None, None, None) fig3, axim3, axcb3, imsh3 = gg.fig_axim_axcb_imsh( figsize=fs) if DO_PLOT_CONNECED_PIXELS else (None, None, None, None) fig2, axim2, axcb2, imsh2 = gg.fig_axim_axcb_imsh( figsize=fs) if DO_PLOT_PIXEL_STATUS else (None, None, None, None) fig1, axim1, axcb1, imsh1 = gg.fig_axim_axcb_imsh( figsize=fs) if DO_PLOT_IMAGE else (None, None, None, None) ##----------------------------- alg = PyAlgos(windows=None, mask=None, pbits=10) # 0177777) if PF == V1: alg.set_peak_selection_pars(npix_min=0, npix_max=1e6, amax_thr=0, atot_thr=0, son_min=6) elif PF == V2: alg.set_peak_selection_pars(npix_min=0, npix_max=1e6, amax_thr=0, atot_thr=0, son_min=6) elif PF == V3: alg.set_peak_selection_pars(npix_min=0, npix_max=1e6, amax_thr=0, atot_thr=0, son_min=8) elif PF == V4: alg.set_peak_selection_pars(npix_min=0, npix_max=1e6, amax_thr=0, atot_thr=0, son_min=6) alg.print_attributes() for ev in range(EVTMAX): ev1 = ev + 1 if ev < SKIP: continue #if ev>=EVTMAX : break print 50 * '_', '\nEvent %04d' % ev1 img, peaks_sim = image_with_random_peaks(shape) # --- for debugging #np.save('xxx-image', img) #np.save('xxx-peaks', np.array(peaks_sim)) #img = np.load('xxx-image-crash.npy') #peaks_sim = np.load('xxx-peaks-crash.npy') # --- peaks_gen = [(0, r, c, a, a * s, 9 * s * s) for r, c, a, s in peaks_sim] t0_sec = time() #peaks = alg.peak_finder_v1(img, thr_low=20, thr_high=40, radius=6, dr=2) if PF == V1 else\ # alg.peak_finder_v2r1(img, thr=30, r0=7, dr=2) if PF == V2 else\ # alg.peak_finder_v3r2(img, rank=5, r0=7, dr=2, nsigm=3) if PF == V3 else\ # alg.peak_finder_v4r2(img, thr_low=20, thr_high=40, rank=6, r0=7, dr=2) peaks = None if PF == V1 else\ None if PF == V2 else\ alg.peak_finder_v3r3(img, rank=5, r0=7, dr=2, nsigm=3) if PF == V3 else\ alg.peak_finder_v4r3(img, thr_low=20, thr_high=40, rank=6, r0=7, dr=2) print ' Time consumed by the peak_finder = %10.6f(sec)' % (time() - t0_sec) map2 = reshape_to_2d(alg.maps_of_pixel_status( )) if DO_PLOT_PIXEL_STATUS else None # np.zeros((10,10)) map3 = reshape_to_2d(alg.maps_of_connected_pixels( )) if DO_PLOT_CONNECED_PIXELS else None # np.zeros((10,10)) map4 = reshape_to_2d(alg.maps_of_local_maximums( )) if DO_PLOT_LOCAL_MAXIMUMS else None # np.zeros((10,10)) map5 = reshape_to_2d(alg.maps_of_local_minimums( )) if DO_PLOT_LOCAL_MINIMUMS else None # np.zeros((10,10)) print 'arrays are extracted' #print_arr(map2, 'map_of_pixel_status') #print_arr(map3, 'map_of_connected_pixels') #maps.shape = shape print 'Simulated peaks:' for i, (r0, c0, a0, sigma) in enumerate(peaks_sim): print ' %04d row=%6.1f col=%6.1f amp=%6.1f sigma=%6.3f' % ( i, r0, c0, a0, sigma) #plot_image(img) print 'Found peaks:' print hdr reg = 'IMG' for pk in peaks: seg,row,col,npix,amax,atot,rcent,ccent,rsigma,csigma,\ rmin,rmax,cmin,cmax,bkgd,rms,son = pk[0:17] rec = fmt % (ev, reg, seg, row, col, npix, amax, atot, rcent, ccent, rsigma, csigma,\ rmin, rmax, cmin, cmax, bkgd, rms, son) #,\ #imrow, imcol, xum, yum, rum, phi) print rec if DO_PLOT_PIXEL_STATUS: gg.plot_imgcb(fig2, axim2, axcb2, imsh2, map2, amin=0, amax=30, title='Pixel status, ev: %04d' % ev1) gg.move_fig(fig2, x0=0, y0=30) if DO_PLOT_CONNECED_PIXELS: cmin, cmax = (map3.min(), map3.max()) if map3 is not None else (None, None) print 'Connected pixel groups min/max:', cmin, cmax gg.plot_imgcb(fig3, axim3, axcb3, imsh3, map3, amin=cmin, amax=cmax, title='Connected pixel groups, ev: %04d' % ev1) gg.move_fig(fig3, x0=100, y0=30) if DO_PLOT_LOCAL_MAXIMUMS: gg.plot_imgcb(fig4, axim4, axcb4, imsh4, map4, amin=0, amax=10, title='Local maximums, ev: %04d' % ev1) gg.move_fig(fig4, x0=200, y0=30) if DO_PLOT_LOCAL_MINIMUMS: gg.plot_imgcb(fig5, axim5, axcb5, imsh5, map5, amin=0, amax=10, title='Local minimums, ev: %04d' % ev1) gg.move_fig(fig5, x0=300, y0=30) if DO_PLOT_IMAGE: #nda = maps_of_conpix_arc #nda = maps_of_conpix_equ #img = det.image(evt, nda)[xoffset:xoffset+xsize,yoffset:yoffset+ysize] #img = det.image(evt, mask_img*nda)[xoffset:xoffset+xsize,yoffset:yoffset+ysize] #img = det.image(evt, maps_of_conpix_equ)[xoffset:xoffset+xsize,yoffset:yoffset+ysize] ave, rms = img.mean(), img.std() amin, amax = ave - 1 * rms, ave + 8 * rms axim1.clear() if imsh1 is not None: del imsh1 imsh1 = None gg.plot_imgcb(fig1, axim1, axcb1, imsh1, img, amin=amin, amax=amax, title='Image, ev: %04d' % ev1) gg.move_fig(fig1, x0=400, y0=30) gg.plot_peaks_on_img(peaks_gen, axim1, imRow, imCol, color='g', lw=5) #, pbits=3) gg.plot_peaks_on_img(peaks, axim1, imRow, imCol, color='w') #, pbits=3) fig1.canvas.draw() # re-draw figure content #gg.plotHistogram(nda, amp_range=(-100,100), bins=200, title='Event %d' % i) gg.show(mode='do not hold') gg.show()
#nda = subtract_bkgd(nda, nda_bkgd, mask=nda_smask, winds=winds_bkgd, pbits=0) #nda *= nda_smask #det.common_mode_apply(evt, nda) #print ' ----> calibration dt = %f sec' % (time()-t1_sec) #print_ndarr(nda, 'data: raw-peds') t0_sec = time() peaks = None peaks_rec = None if TEST_BW_COMP : alg.set_mask(qmask) peaks = alg.peak_finder_v3r3(nda, rank=5, r0=7, dr=2, nsigm=5) if PF == V3 else\ alg.peak_finder_v4r3(nda, thr_low=20, thr_high=80, rank=5, r0=7, dr=2) if PF == V4 else None #peaks = alg.peak_finder_v3r2(nda, rank=5, r0=7, dr=2, nsigm=5) if PF == V3 else\ # alg.peak_finder_v4r2(nda, thr_low=20, thr_high=80, rank=5, r0=7, dr=2) if PF == V4 else None peaks_rec = peaks else : peaks = peaks_adaptive(nda, qmask, rank=5, r0=7, dr=2, nsigm=5,\ npix_min=2, npix_max=None, amax_thr=0, atot_thr=0, son_min=10) if PF == V3 else\ peaks_droplet (nda, qmask, thr_low=20, thr_high=80, rank=5, r0=7, dr=2,\ npix_min=2, npix_max=None, amax_thr=0, atot_thr=0, son_min=5) if PF == V4 else None #for p in peaks : # #print dir(p) # print ' seg:%4d, row:%4d, col:%4d, npix:%4d, son:%4.1f' % (p.seg, p.row, p.col, p.npix, p.son) peaks_rec = [(int(p.seg), int(p.row), int(p.col), p.amp_max, p.amp_tot, int(p.npix)) for p in peaks]
class adaptiveAlgorithm(abstractAlgorithm.abstractAlgorithm): def __init__(self): """ Initialize instance of imported PyAlgos algorithm. """ self.alg = PyAlgos(mask=None, pbits=0) self.setDefaultParams() def setParams(self): """ Use the algorithm's function, set_peak_selection_pars, to set parameters. """ self.alg.set_peak_selection_pars(npix_min=self.alg1_npix_min, npix_max=self.alg1_npix_max, amax_thr=self.alg1_amax_thr, atot_thr=self.alg1_atot_thr, son_min=self.alg1_son_min) def initParams(self, **kwargs): """ Save the values of the parameters from kwargs. Arguments: **kwargs -- dictionary of parameters, either default or user inputted """ self.alg1_npix_min = kwargs["npix_min"] self.alg1_npix_max = kwargs["npix_max"] self.alg1_amax_thr = kwargs["amax_thr"] self.alg1_atot_thr = kwargs["atot_thr"] self.alg1_son_min = kwargs["son_min"] self.alg1_rank = kwargs["rank"] self.alg1_r0 = kwargs["r0"] self.alg1_dr = kwargs["dr"] self.alg1_nsigm = kwargs["nsigm"] self.setParams() def algorithm(self, nda, mask, kw=None): """ Uses peak_finder_v3r3 (a.k.a. adaptive peak finder) to find peaks on an image. Arguments: nda -- detector image mask -- detector mask kw -- dictionary or None, if None default parameters are used, otherwise kw is used to initialize parameters """ if kw == None: self.initParams(**self.default_params) else: self.initParams(**kw) self.peaks = self.alg.peak_finder_v3r3(nda, rank=self.alg1_rank, r0=self.alg1_r0, dr=self.alg1_dr, nsigm=self.alg1_nsigm, mask=mask) return self.peaks def getDefaultParams(self): """ Return the default parameters in the form of a string, for Psocake to display """ return json.dumps(self.default_params) def setDefaultParams(self): #The default parameters for the adaptive peak finding algorithm #self.default_params_str = "{\"npix_min\": 2,\"npix_max\":30,\"amax_thr\":300, \"atot_thr\":600,\"son_min\":10, \"rank\":3, \"r0\":3, \"dr\":2, \"nsigm\":5 }" self.default_params = { "npix_min": 2, "npix_max": 30, "amax_thr": 300, "atot_thr": 600, "son_min": 10, "rank": 3, "r0": 3, "dr": 2, "nsigm": 5 }