def makemovie1(self): run = self.s['run'] sundownDate = self.s['sundownDate'] obsDate = self.s['obsDate'] stride = int(self.s['stride']) seq5 = self.s['seq5'].split() for seq in seq5: inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb") cosmicTimeList = pickle.load(inFile) binContents = pickle.load(inFile) cfn = "cosmicMask-%s.h5"%seq intervals = ObsFile.readCosmicIntervalFromFile(cfn) for interval in intervals: print "interval=",interval fn = FileName(run, sundownDate,obsDate+"-"+seq) obsFile = ObsFile(fn.obs()) obsFile.loadTimeAdjustmentFile(fn.timeAdjustments()) i0=interval[0] i1=interval[1] intervalTime = i1-i0 dt = intervalTime/2 beginTime = max(0,i0-0.000200) endTime = beginTime + 0.001 integrationTime = endTime-beginTime nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1)) timeHgValues = np.zeros(nBins, dtype=np.int64) ymax = sys.float_info.max/100.0 for iRow in range(obsFile.nRow): for iCol in range(obsFile.nCol): gtpl = obsFile.getTimedPacketList(iRow,iCol, beginTime,integrationTime) ts = (gtpl['timestamps'] - beginTime)*obsFile.ticksPerSec ts64 = np.round(ts).astype(np.uint64) tsBinner.tsBinner(ts64, timeHgValues) plt.clf() plt.plot(timeHgValues, label="data") x0 = (i0-beginTime)*obsFile.ticksPerSec x1 = (i1-beginTime)*obsFile.ticksPerSec plt.fill_between((x0,x1),(0,0), (ymax,ymax), alpha=0.2, color='red') plt.yscale("symlog",linthreshy=0.9) plt.xlim(0,1000) plt.ylim(-0.1,300) tick0 = int(np.round(i0*obsFile.ticksPerSec)) plotfn = "cp-%05d-%s-%s-%s-%09d"%(timeHgValues.sum(),run,obsDate,seq,tick0) plt.title(plotfn) plt.legend() plt.savefig(plotfn+".png") plt.xlabel("nSigma=%d stride=%d threshold=%d"%(int(self.s['nSigma']),int(self.s['stride']),int(self.s['threshold']))) print "plotfn=",plotfn os.system("convert -delay 0 `ls -r cp*png` cp.gif")
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=True): ''' Test run of obsFile.writePhotonList. fileName can be used to specify the output file name. If not specified, default name/location is used. Now includes test for pixel remapping.... ''' #Details of example obs file to run test on. # run = 'PAL2012' # date = '20121207' # tstamp = '20121208-074649' # calTstamp='20121208-070505' # fluxTstamp='20121208-133002' run = 'PAL2012' date = '20121208' tstamp = '20121209-120530' centroidTstamp = '20121209-120530' calTstamp='20121209-131132' fluxTstamp='20121209-020416' flatTstamp='20121209-021036' if doPixRemap==True: pixRemapFileName = FileName(run=run).pixRemap() else: pixRemapFileName = None #Load up the obs file obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) #Load up associated calibrations obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln()) obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln()) obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln()) obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments()) obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask()) obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList()) #And write out the results.... obsFile.writePhotonList(outputFileName,firstSec,integrationTime, pixRemapFileName=pixRemapFileName)
NumFiles.append(len(seq)) NumFiles = sum(NumFiles) print (NumFiles)*expTime/integrationTime,'frames to make' for iSeq in range(len(seqs)): timestampList = timestampLists[iSeq] print timestampList wfn = wvlCalFilenames[iSeq] ffn = flatCalFilenames[iSeq] sunsetDate = sunsetDates[iSeq] for i,ts in enumerate(timestampList): print 'loading',ts obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs() ob = ObsFile(obsFn) ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments()) index1 = obsFn.find('_') hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:] if not os.path.exists(hotPixFn): hp.findHotPixels(obsFn,hotPixFn) print "Flux file pixel mask saved to %s"%(hotPixFn) ob.loadHotPixCalFile(hotPixFn,switchOnMask=True) ob.loadWvlCalFile(wfn) ob.loadFlatCalFile(ffn) ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff) bad_solution_mask=np.zeros((46,44)) bad_count=0; for y in range(46): for x in range(44): if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff):
class Cosmic: def __init__(self, fn, beginTime=0, endTime='exptime', nBinsPerSec=10, flashMergeTime=1.0, applyCosmicMask = False, loggingLevel=logging.CRITICAL, loggingHandler=logging.StreamHandler()): """ Opens fileName in MKID_RAW_PATH, sets roachList endTime is exclusive """ self.logger = logging.getLogger("cosmic") self.logger.setLevel(loggingLevel) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') loggingHandler.setFormatter(formatter) self.logger.addHandler(loggingHandler) self.logger.info("Cosmic: begin init for obsFile=%s"%fn.obs()) self.fn = fn self.fileName = fn.obs(); self.file = ObsFile(self.fileName) # apply Matt's time fix timeAdjustments = self.fn.timeAdjustments() if os.path.exists(timeAdjustments): self.file.loadTimeAdjustmentFile(timeAdjustments) # apply Julian's time masks timeMaskFile = self.fn.timeMask(); if os.path.exists(timeMaskFile): self.file.loadHotPixCalFile(timeMaskFile,switchOnMask=True) # apply standard mask if applyCosmicMask: self.file.loadStandardCosmicMask() self._setRoachList() self._setAllSecs() self.exptime = self.file.getFromHeader('exptime') if endTime =='exptime': self.endTime = float(self.exptime) else: self.endTime = float(endTime) if ( (self.endTime > self.exptime) or (endTime < 0)): raise RuntimeError("bad endTime: endTime=%s exptime=%s" % \ (str(endTime),str(self.exptime))) self.beginTime = float(beginTime) self.timeHgs = "none" self.nBinsPerSec = nBinsPerSec self.flashMergeTime = flashMergeTime self.times = \ np.arange(self.beginTime, self.endTime, 1.0/self.nBinsPerSec) # for measuring flashes, indexed by roach name self.rMean = {} # mean from meanclip self.rSigma = {} # sigma from meanclip self.rNSurvived = {} # number of survivors from meanclip self.rNormed = {} # (value-mean)/sigma self.flashInterval = {} self.logger.info("Cosmic: end init: beginTime=%s endTime=%s"%(str(self.beginTime),str(self.endTime))) def __del__(self): """ Close any open files """ # print "now in Cosmic.__del__ for ",self.fileName try: del self.file except: pass def _setRoachList(self): self.roachList = [] for row in self.file.beamImage: for name in row: roachName = name.split("/")[1] if roachName not in self.roachList: self.roachList.append(roachName) self.roachList.sort() def _setAllSecs(self): nRow = self.file.nRow nCol = self.file.nCol self.allSecs = \ dict( ((i,j),None) for i in range(nRow) for j in range(nCol)) for iRow in np.arange(nRow): for iCol in np.arange(nCol): self.allSecs[iRow,iCol] = \ self.file.file.getNode(self.file.beamImage[iRow][iCol]) def nPhoton(self, beginTime=0, endTime='expTime'): """ trivial example of counting the number of photons in a file """ nPhoton = 0 for iRow in range(self.file.nRow): for iCol in range(self.file.nCol): for iSec in range(self.beginTime, self.endTime): sec = self.allSecs[iRow,iCol][iSec] nPhoton += len(sec) return nPhoton def findFlashes(self, clipsig=3.0, maxiter=5,\ converge_num=0.05, verbose=0, flashsig=6): """ find flashes by looking at the time histograms. Calculate the mean,sigma using meanclip and the parameters clipsig, maxiter, converge_num A time bin has a flash if the normalized signal (measured-mean)/sigma is larger than flashsig. """ # make the blank data structures if self.timeHgs == "none": self.makeTimeHgs() self.flashInterval["all"] = [] # find the flashes in each roach for roach in self.roachList: self.rMean[roach],self.rSigma[roach],self.rNSurvived[roach] = \ meanclip.meanclip(\ np.array(self.timeHgs[roach]), clipsig, maxiter, converge_num,\ verbose) self.rNormed[roach] = \ (self.timeHgs[roach]-self.rMean[roach])/self.rSigma[roach] self.flashInterval[roach] = interval() prev = 0 a = self.rNormed[roach] for i in range(len(a)): this = a[i] > flashsig if (this != prev): if (this): iBegin = i else: self.flashInterval[roach] = \ self.flashInterval[roach] | interval[iBegin,i] prev = this if (prev): self.flashInterval[roach] = \ self.flashInterval[roach] | interval[iBegin,i] # union of all flashes self.flashInterval["all"] = interval() for roach in self.roachList: self.flashInterval["all"] = \ self.flashInterval["all"] | self.flashInterval[roach] # look for gaps smaller than self.flashMergeTime and plug them dMax = self.nBinsPerSec*self.flashMergeTime extrema = self.flashInterval["all"].extrema for i in range(len(extrema)/2-1): i0 = extrema[2*i+1][0] i1 = extrema[2*(i+1)][0] if (i1-i0) <= dMax: t0 = self.beginTime + float(i0)/self.nBinsPerSec t1 = self.beginTime + float(i1)/self.nBinsPerSec self.flashInterval["all"] = \ self.flashInterval["all"] | interval[i0,i1] # convert to ticks since the beginning of the data file rlAll = list(self.roachList) rlAll.append("all") ticksPerSecond = int(1.0/self.file.tickDuration) offset = self.beginTime*ticksPerSecond scale = 1.0/(self.file.tickDuration*self.nBinsPerSec) for roach in rlAll: self.flashInterval[roach] = offset+scale*self.flashInterval[roach] def writeFlashesToHdf5(self,overwrite=1): """ write intervals with flashes to the timeMask file """ # get the output file name, and make the directory if you need to cmFileName = self.fn.cosmicMask() (cmDir,name) = os.path.split(cmFileName) if not os.path.exists(cmDir): os.makedirs(cmDir) # write parameters used to find flashes h5f = tables.openFile(cmFileName, 'w') fnode = filenode.newNode(h5f, where='/', name='timeMaskHdr') fnode.attrs.beginTime = self.beginTime fnode.attrs.endTime = self.endTime fnode.attrs.nBinsPerSec = self.nBinsPerSec fnode.attrs.flashMergeTime = self.flashMergeTime fnode.close(); # write the times where flashes are located tbl = h5f.createTable('/','timeMask',TimeMask.TimeMask,"Time Mask") rlAll = list(self.roachList) rlAll.append("all") for roach in rlAll: extrema = self.flashInterval[roach].extrema for i in range(len(extrema)/2): row = tbl.row row['tBegin'] = int(extrema[2*i][0]) row['tEnd'] = int(extrema[2*i+1][0]) if (roach == "all"): reason = "Merged Flash" else: reason = "Flash in %s" % roach row['reason'] = TimeMask.timeMaskReason[reason] row.append() tbl.flush() tbl.close() h5f.close() def makeTimeHgs(self): """ Fill in the timeHgs variable This is a dictionary, indexed by the roach name, of the time histograms """ self.timeHgs = {} for iSec in range(self.beginTime, self.endTime): self.logger.info("in makeTimeHgs iSec=%4d / %4d" % (iSec,self.endTime)) hgsThisSec = {} for iRow in range(self.file.nRow): for iCol in range(self.file.nCol): sec = self.allSecs[iRow,iCol][iSec] if len(sec) > 0: times = sec & self.file.timestampMask hg,edges = \ np.histogram(times,bins=self.nBinsPerSec, \ range=(0,1.0/self.file.tickDuration)) roachName = \ self.file.beamImage[iRow][iCol].split("/")[1] if not hgsThisSec.has_key(roachName): hgsThisSec[roachName] = \ np.zeros(self.nBinsPerSec,dtype=np.int64) hgsThisSec[roachName] += hg for roachName in hgsThisSec.keys(): if not self.timeHgs.has_key(roachName): self.timeHgs[roachName] = [] self.timeHgs[roachName] += list(hgsThisSec[roachName]) def plotTimeHgs(self): """ Plot the time HGS in plt structure, with legend """ plt.clf() plt.figure(1) keys = self.timeHgs.keys() keys.sort() plt.subplot(211) for roachName in keys: hg = self.timeHgs[roachName] plt.plot(self.times, hg,label=roachName) plt.legend() dt = 1.0/self.nBinsPerSec plt.ylabel("photons/%.2f sec" % dt) plt.title("Cosmic timeHgs "+ self.fileName) plt.subplot(212) for roachName in keys: plt.plot(self.times, \ self.rNormed[roachName],label=roachName) plt.xlabel("time (sec)") plt.ylim(-23,30) dt = 1.0/self.nBinsPerSec plt.ylabel("normalized photons/%.2f sec" % dt) y = -5 x0 = self.beginTime + 0.1*(self.endTime-self.beginTime) xmax = plt.xlim()[1] rlAll = list(self.roachList) rlAll.append("all") for roach in rlAll: print "plot for roach=",roach plt.plot([x0,xmax],[y,y], linestyle=":", color="gray") plt.text(x0, y, roach, fontsize=8, va="center") extrema = self.flashInterval[roach].extrema for i in range(len(extrema)/2): t0 = (extrema[2*i][0] - 0.5)*self.file.tickDuration t1 = (extrema[2*i+1][0] - 0.5)*self.file.tickDuration plt.plot([t0,t1],[y,y],'r', linewidth=4) y -= 2 def findCosmics(self, stride=10, threshold=100, populationMax=2000, nSigma=5, writeCosmicMask=False, ppsStride=10000): """ Find cosmics ray suspects. Histogram the number of photons recorded at each timeStamp. When the number of photons in a group of stride timeStamps is greater than threshold in second iSec, add (iSec,timeStamp) to cosmicTimeLists. Also keep track of the histogram of the number of photons per stride timeStamps. return a dictionary of 'populationHg', 'cosmicTimeLists', 'binContents', 'timeHgValues', 'interval', 'frameSum', and 'pps' populationHg is a histogram of the number of photons in each time bin. This is a poisson distribution with a long tail due to cosmic events cosmicTimeLists is a numpy array of all the sequences that are suspects for cosmic rays binContents corresponds to cosmicTimeLists. For each time in cosmicTimeLists, binContents is the number of photons detected at that time. timeHgValues is a histogram of the number of photons in each time interval frameSum is a two dimensional numpy array of the number of photons detected by each pixel interval is the interval of data to be masked out pps is photons per second, calculated every ppsStride bins. """ self.logger.info("findCosmics: begin stride=%d threshold=%d populationMax=%d nSigma=%d writeCosmicMask=%s"%(stride,threshold,populationMax,nSigma,writeCosmicMask)) exptime = self.endTime-self.beginTime nBins = int(np.round(self.file.ticksPerSec*exptime+1)) bins = np.arange(0, nBins, 1) timeHgValues,frameSum = self.getTimeHgAndFrameSum(self.beginTime,self.endTime) remainder = len(timeHgValues)%ppsStride if remainder > 0: temp = timeHgValues[:-remainder] else: temp = timeHgValues ppsTime = (ppsStride*self.file.tickDuration) pps = np.sum(temp.reshape(-1, ppsStride), axis=1)/ppsTime self.logger.info("findCosmics: call populationFromTimeHgValues") pfthgv = Cosmic.populationFromTimeHgValues\ (timeHgValues,populationMax,stride,threshold) #now build up all of the intervals in seconds self.logger.info("findCosmics: build up intervals: nCosmicTime=%d"%len(pfthgv['cosmicTimeList'])) i = interval() iCount = 0 secondsPerTick = self.file.tickDuration for cosmicTime in pfthgv['cosmicTimeList']: #t0 = max(0,self.beginTime+(cosmicTime-50)/1.e6) #t1 = min(self.endTime,self.beginTime+(cosmicTime+50)/1.e6) #intTime = t1-t0 t0 = self.beginTime+cosmicTime*secondsPerTick dt = stride*secondsPerTick t1 = t0+dt left = max(self.beginTime, t0-nSigma*dt) right = min(self.endTime, t1+2*nSigma*dt) i = i | interval[left,right] self.logger.debug("findCosmics: iCount=%d t0=%f t1=%f left=%f right=%f"%(iCount,t0,t1,left,right)) iCount+=1 tMasked = Cosmic.countMaskedBins(i) ppmMasked = 1000000*tMasked/(self.endTime-self.beginTime) retval = {} retval['timeHgValues'] = timeHgValues retval['populationHg'] = pfthgv['populationHg'] retval['cosmicTimeList'] = pfthgv['cosmicTimeList'] retval['binContents'] = pfthgv['binContents'] retval['frameSum'] = frameSum retval['interval'] = i retval['ppmMasked'] = ppmMasked retval['pps'] = pps retval['ppsTime'] = ppsTime if writeCosmicMask: cfn = self.fn.cosmicMask() self.logger.info("findCosmics: write masks to =%s"%cfn) ObsFile.writeCosmicIntervalToFile(i, self.file.ticksPerSec, cfn,self.beginTime, self.endTime, stride, threshold, nSigma, populationMax) self.logger.info("findCosmics: end with ppm masked=%d"%ppmMasked) return retval def getTimeHgAndFrameSum(self, beginTime, endTime): integrationTime = endTime - beginTime nBins = int(np.round(self.file.ticksPerSec*integrationTime+1)) timeHgValues = np.zeros(nBins, dtype=np.int64) frameSum = np.zeros((self.file.nRow,self.file.nCol)) self.logger.info("get all time stamps for integrationTime=%f"%integrationTime) for iRow in range(self.file.nRow): #print "Cosmic.findCosmics: iRow=",iRow for iCol in range(self.file.nCol): # getTimedPacketList is slow. Use getPackets instead. #gtpl = self.file.getTimedPacketList(iRow,iCol,beginTime, # integrationTime) gtpl = self.file.getPackets(iRow,iCol, beginTime,integrationTime) timestamps = gtpl['timestamps'] if timestamps.size > 0: timestamps = \ (timestamps - beginTime)*self.file.ticksPerSec # per Matt S. suggestion 2013-07-09 ts32 = np.round(timestamps).astype(np.uint32) tsBinner.tsBinner32(ts32, timeHgValues) frameSum[iRow,iCol] += ts32.size return timeHgValues,frameSum @staticmethod def countMaskedBins(maskInterval): retval = 0 for x in maskInterval: retval += x[1]-x[0] return retval @staticmethod def populationFromTimeHgValues(timeHgValues,populationMax,stride,threshold): """ Rebin the timgHgValues histogram by combining stride bins. If stride > 1, then bin a second time after shifting by stride/2 Create populationHg, a histogram of the number of photons in the large bins. Also, create (and then sort) a list cosmicTimeList of the start of bins (in original time units) of overpopulated bins that have more than threshold number of photons. return a dictionary containing populationHg and cosmicTimeList """ popRange = (-0.5,populationMax-0.5) if stride==1: populationHg = np.histogram(\ timeHgValues, populationMax, range=popRange) cosmicTimeList = np.where(timeHgValues > threshold)[0] binContents = np.extract(timeHgValues > threshold, timeHgValues) else: # rebin the timeHgValues before counting the populations length = timeHgValues.size remainder = length%stride if remainder == 0: end = length else: end = -remainder timeHgValuesTrimmed = timeHgValues[0:end] timeHgValuesRebinned0 = np.reshape( timeHgValuesTrimmed, [length/stride, stride]).sum(axis=1) populationHg0 = np.histogram( timeHgValuesRebinned0, populationMax, range=popRange) cosmicTimeList0 = stride*np.where(\ timeHgValuesRebinned0 > threshold)[0] binContents0 = np.extract(timeHgValuesRebinned0 > threshold, timeHgValuesRebinned0) timeHgValuesRebinned1 = np.reshape( timeHgValuesTrimmed[stride/2:-stride/2], [(length-stride)/stride, stride]).sum(axis=1) populationHg1 = np.histogram(\ timeHgValuesRebinned1, populationMax, range=popRange) cosmicTimeList1 = (stride/2)+stride*np.where(\ timeHgValuesRebinned1 > threshold)[0] binContents1 = np.extract(timeHgValuesRebinned1 > threshold, timeHgValuesRebinned1) populationHg = (populationHg0[0]+populationHg1[0],\ populationHg0[1]) cosmicTimeList = np.concatenate((cosmicTimeList0,cosmicTimeList1)) binContents = np.concatenate((binContents0, binContents1)) args = np.argsort(cosmicTimeList) cosmicTimeList = cosmicTimeList[args] binContents = binContents[args] cosmicTimeList.sort() retval = {} retval['populationHg'] = populationHg retval['cosmicTimeList'] = cosmicTimeList retval['binContents'] = binContents return retval def makeMovies(self,beginTick, endTick, backgroundFrame, accumulate=False): tick0 = np.uint64(beginTick) tick1 = np.uint64(endTick) for iRow in range(cosmic.file.nRow): for iCol in range(cosmic.file.nCol): gtpl = self.getTimedPacketList(iRow,iCol,sec0,1) timestamps = gtpl['timestamps'] timestamps *= cosmic.file.ticksPerSec ts32 = timestamps.astype(np.uint32) for ts in ts32: tindex = ts-t0 try: listOfPixelsToMark[tindex].append((iRow,iCol)) except IndexError: pass for tick in range(t0,t1): frames.append(frameSum) title = makeTitle(tick,t0,t1) titles.append(title) mfn0 = "m-%s-%s-%s-%s-%010d-%010d-i.gif"%(run,sundownDate,obsDate,seq,t0,t1) utils.makeMovie(frames, titles, outName=mfn0, delay=0.1, colormap=mpl.cm.gray, listOfPixelsToMark=listOfPixelsToMark, pixelMarkColor='red') for i in range(len(listOfPixelsToMark)-1): listOfPixelsToMark[i+1].extend(listOfPixelsToMark[i]) mfn1 = "m-%s-%s-%s-%s-%010d-%010d-a.gif"%(run,sundownDate,obsDate,seq,t0,t1) utils.makeMovie(frames, titles, outName=mfn1, delay=0.1, colormap=mpl.cm.gray, listOfPixelsToMark=listOfPixelsToMark, pixelMarkColor='green') def fitDecayTime(self,t0Sec,lengthSec=200,plotFileName='none'): print "hello from fitDecayTime" timedPacketList = self.file.getTimedPacketList( iRow, iCol, sec0, lengthSec) def fitExpon(self, t0, t1): """ Fit an exponential to all photons from time t0 to time t1 t0 and t1 are in ticks, 1e6 ticks per second return a dictionary of: timeStamps,fitParams,chi2 """ firstSec = int(t0/1e6) # in seconds integrationTime = 1+int((t1-t0)/1e6) # in seconds nBins = integrationTime*1e6 # number of microseconds; one bin per microsecond timeHgValues = np.zeros(nBins, dtype=np.int64) print "firstSec=",firstSec," integrationTime=",integrationTime for iRow in range(self.file.nRow): for iCol in range(self.file.nCol): timedPacketList = self.file.getTimedPacketList( iRow, iCol, firstSec=firstSec, integrationTime=integrationTime) timeStamps = timedPacketList['timestamps'] if (len(timeStamps) > 0): # covert the time values to microseconds, and # make it the type np.uint64 # round per Matt S. suggestion 2013-07-09 #ts64 = (timeStamps).astype(np.uint64) ts32round = np.round(timeStamps).astype(np.uint32) tsBinner.tsBinner(ts32round, timeHgValues) temp = 1e6*(timeStamps-firstSec) for i in range(len(timeStamps)): ts32 = ((timeStamps-firstSec)*1e6).astype(np.uint32) # add these timestamps to the histogram timeHgValues remain0 = int(t0%1e6) remain1 = int(t1%1e6) timeHgValues = timeHgValues[remain0:remain1] x = np.arange(len(timeHgValues)) y = timeHgValues xArray = np.arange(0, dtype=np.int64) yArray = np.arange(0, dtype=np.int64) for i in range(len(x)): if y[i] > 2: xArray = np.append(xArray,i) yArray = np.append(yArray,y[i]) ySigma = np.sqrt(yArray) mean = (x*y).sum()/float(y.sum()) bExponGuess = 1/mean aExponGuess = bExponGuess*timeHgValues.sum() cExponGuess = 0 dExponGuess = 0 pExponGuess = [aExponGuess, bExponGuess, cExponGuess, dExponGuess] bGaussGuess = mean avgx2 = (x*x*y).sum()/float(y.sum()) cGaussGuess = np.sqrt(avgx2-bGaussGuess*bGaussGuess) aGaussGuess = (timeHgValues.sum()/(cGaussGuess*np.sqrt(2*np.pi))) pGaussGuess = [aGaussGuess, bGaussGuess, cGaussGuess] xLimit = [bGaussGuess-4*cGaussGuess, bGaussGuess+4*cGaussGuess] retval = {'timeHgValues':timeHgValues, 'pExponFit':pExponGuess, 'pGaussFit':pGaussGuess, 'xLimit':xLimit, 'cGaussGuess':cGaussGuess, 'timeStamps':timeStamps} return retval @staticmethod def intervalTime(intervals): """ return the time (in seconds) masked by the intervals """ time = 0 for interval in intervals: time += interval[1]-interval[0] return time @staticmethod def funcExpon(x, a, b, c, d): retval = a*np.exp(-b*(x-d)) + c retval[x < d] = 0 return retval @staticmethod def funcGauss(x, a, b, c): return a*np.exp(-(x-b)**2/(2.*c**2))
run = 'PAL2012' sundownDate = '20121211' obsDate = '20121212' for seq in seq5: inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb") cosmicTimeList = pickle.load(inFile) binContents = pickle.load(inFile) cfn = "cosmicMax-%s.h5"%seq intervals = ObsFile.readCosmicIntervalFromFile(cfn) for interval in intervals: print "interval=",interval fn = FileName(run, sundownDate,obsDate+"-"+seq) obsFile = ObsFile(fn.obs()) obsFile.loadTimeAdjustmentFile(fn.timeAdjustments()) i0=interval[0] i1=interval[1] intervalTime = i1-i0 dt = intervalTime/2 beginTime = max(0,i0-0.000200) endTime = beginTime + 0.001 integrationTime = endTime-beginTime nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1)) timeHgValues = np.zeros(nBins, dtype=np.int64) ymax = sys.float_info.max/100.0 for iRow in range(obsFile.nRow): for iCol in range(obsFile.nCol): gtpl = obsFile.getTimedPacketList(iRow,iCol, beginTime,integrationTime) ts = (gtpl['timestamps'] - beginTime)*obsFile.ticksPerSec
class DisplayStack(QMainWindow): def __init__(self): # 0) Start up gui QWidget.__init__(self, parent=None) self.ui = Ui_DisplayStack_gui() self.ui.setupUi(self) # Initialize Variables self.initializeVariables() # Initialize run and target, load param file self.chooseRun() ''' # Browse wavecal button self.ui.wavelengthCalibrationSolutionButton.clicked.connect(self.chooseWavelengthFile) # Browse flatcal button self.ui.flatCalibrationSolutionButton.clicked.connect(self.chooseFlatFile) ''' # Use wavelength calibration checkbox self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal) # Use flat calibration checkbox self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal) # Buttons for obs list creation self.ui.addButton.clicked.connect(self.addObservation) self.ui.removeButton.clicked.connect(self.removeObservation) self.ui.clearButton.clicked.connect(self.createObsList) # Start process button self.ui.stackButton.clicked.connect(self.stackProcess) # Load image stack button self.ui.loadStackButton.clicked.connect(self.chooseStack) # 1) Initialize Variables def initializeVariables(self): # Define path names self.displayStackPath = '/Scratch/DisplayStack/' self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/' self.defaultFlatPath = '/Scratch/flatCalSolnFiles/' # Arrays with run names and buttons self.runNames = ['LICK2012','PAL2012','PAL2013'] self.runButtons = [self.ui.lick2012Button, self.ui.pal2012Button, self.ui.pal2013Button] # Arrays with target names and buttons self.lick2012TargetNames = ['LICK2012_Object'] self.lick2012TargetButtons = [self.ui.lick2012ObjectButton] self.pal2012TargetNames = ['SDSS_J0651', 'SDSS_J0926'] self.pal2012TargetButtons = [self.ui.sdss0651Button,self.ui.sdss0926Button] self.pal2013TargetNames = ['PAL2013_Object'] self.pal2013TargetButtons = [self.ui.pal2013ObjectButton] self.targetNames = [self.lick2012TargetNames, self.pal2012TargetNames, self.pal2013TargetNames] self.targetButtons = [self.lick2012TargetButtons, self.pal2012TargetButtons, self.pal2013TargetButtons] # Activate run and target buttons for iRun in range(len(self.runButtons)): self.runButtons[iRun].clicked.connect(self.chooseRun) for iTarget in range(len(self.targetButtons[iRun])): self.targetButtons[iRun][iTarget].clicked.connect(self.chooseTarget) self.ui.sunsetList.itemClicked.connect(self.createObsList) def testFunction(self): print 'test' # 2) Choose Run def chooseRun(self): for iRun in range(len(self.runButtons)): for iTarget in range(len(self.targetButtons[iRun])): self.targetButtons[iRun][iTarget].setVisible(False) if self.runButtons[iRun].isChecked(): self.runNumber = iRun for iTarget in range(len(self.targetButtons[iRun])): self.targetButtons[iRun][iTarget].setVisible(True) self.run = self.runNames[self.runNumber] self.targetNumber = 0 self.targetButtons[self.runNumber][self.targetNumber].setChecked(True) self.target = self.targetNames[self.runNumber][self.targetNumber] self.loadParamFile() # 3) Choose Target def chooseTarget(self): for iTarget in range(len(self.targetButtons[self.runNumber])): if self.targetButtons[self.runNumber][iTarget].isChecked(): self.targetNumber = iTarget self.target = self.targetNames[self.runNumber][self.targetNumber] self.loadParamFile() # 4) Load Param File def loadParamFile(self): try: self.paramData = readDict() self.paramData.read_from_file(self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict') self.obsTimes = np.array(self.paramData['obsTimes']) self.utcDates = self.paramData['utcDates'] self.sunsetDates = self.paramData['sunsetDates'] self.calTimestamps = self.paramData['calTimestamps'] self.flatCalDates = self.paramData['flatCalDates'] print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.createSunsetList() #self.createObsList() self.createWavelengthList() self.createFlatList() self.paramFileExists = True except IOError: print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.ui.sunsetList.clear() self.ui.obsList.clear() self.ui.inputList.clear() self.ui.wavelengthList.clear() self.ui.flatList.clear() self.paramFileExists = False # 5) Choose Obs File # Create list of available sunset dates def createSunsetList(self): self.ui.sunsetList.clear() for iDate in range(len(self.sunsetDates)): self.ui.sunsetList.addItem(self.sunsetDates[iDate]) # Create Initial Obs file list def createObsList(self): self.ui.obsList.clear() self.ui.inputList.clear() self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] for iObs in range(len(self.singleDayObservations)): self.ui.obsList.addItem(self.singleDayObservations[iObs]) # Add observation to input list def addObservation(self): self.selectedObs = self.ui.obsList.currentItem() self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs)) self.ui.inputList.addItem(self.selectedObs) self.ui.inputList.sortItems() # Remove observation from input list def removeObservation(self): self.removedObs = self.ui.inputList.currentItem() self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs)) self.ui.obsList.addItem(self.removedObs) self.ui.obsList.sortItems() # 6) Load settings def loadSettings(self): self.validSettings = True # Run and target information self.run = self.runNames[self.runNumber] self.target = self.targetNames[self.runNumber][self.targetNumber] # General settings self.integrationTime = int(self.ui.integrationTimeLine.text()) self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked() self.useHotPixelMasking = self.ui.hotPixelBox.isChecked() # Wavelength calibration settings self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked() self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text()) self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text()) # Flat calibration settings self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked() self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked() self.fileCount = self.ui.inputList.count() self.weighted = self.useFlatCalibration self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration) self.scaleByEffInt = self.useHotPixelMasking if self.ui.sunsetList.currentItem() != None: if self.fileCount == 0: print 'Please select files to process...' self.validSettings = False if self.useWavelengthCalibration: if self.ui.wavelengthList.currentItem() == None: print 'Please select wavelength calibration...' self.validSettings = False else: self.selectedWvlCal = self.ui.wavelengthList.currentItem().text() self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln()) if self.useFlatCalibration: if self.ui.flatList.currentItem() == None: print 'Please select flat calibration...' self.validSettings = False else: self.flatCalNight = self.ui.flatList.currentItem().text() self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln()) else: print 'Please select sunset night...' self.validSettings = False # 7) Load hot pixel mask def loadHotMask(self): self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5') if not os.path.exists(self.hotPixelFilename): hp.findHotPixels(self.obsFn,self.hotPixelFilename) print "Hot pixel mask saved to %s"%(self.hotPixelFilename) self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True) # 8) Create wavelength cal file list def createWavelengthList(self): self.ui.wavelengthList.clear() for iCal in range(len(self.calTimestamps)): self.ui.wavelengthList.addItem(self.calTimestamps[iCal]) # Enable/disable wavecal options def useWaveCal(self): if self.ui.wavelengthCalibrationBox.isChecked(): self.ui.lowerWavelengthCutoffLine.setEnabled(True) self.ui.lowerWavelengthCutoffLabel.setEnabled(True) self.ui.upperWavelengthCutoffLine.setEnabled(True) self.ui.upperWavelengthCutoffLabel.setEnabled(True) self.ui.wavelengthList.setEnabled(True) self.ui.flatCalibrationBox.setEnabled(True) self.ui.flatCalibrationBox.setChecked(True) self.ui.deadPixelBox.setEnabled(True) self.ui.deadPixelBox.setChecked(True) self.ui.flatList.setEnabled(True) else: self.ui.lowerWavelengthCutoffLine.setEnabled(False) self.ui.lowerWavelengthCutoffLabel.setEnabled(False) self.ui.upperWavelengthCutoffLine.setEnabled(False) self.ui.upperWavelengthCutoffLabel.setEnabled(False) self.ui.wavelengthList.setEnabled(False) self.ui.flatCalibrationBox.setEnabled(False) self.ui.flatCalibrationBox.setChecked(False) self.ui.deadPixelBox.setChecked(False) self.ui.deadPixelBox.setEnabled(False) self.ui.flatList.setEnabled(False) # 9) Create flat cal file list def createFlatList(self): self.ui.flatList.clear() for iCal in range(len(self.flatCalDates)): self.ui.flatList.addItem(self.flatCalDates[iCal]) # Enable/disable flatcal options def useFlatCal(self): if self.ui.flatCalibrationBox.isChecked(): self.ui.deadPixelBox.setEnabled(True) self.ui.deadPixelBox.setChecked(True) self.ui.flatList.setEnabled(True) else: self.ui.deadPixelBox.setChecked(False) self.ui.deadPixelBox.setEnabled(False) self.ui.flatList.setEnabled(False) # 10) Load dead pixel mask def loadDeadMask(self): self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz') if not os.path.exists(self.deadPixelFilename): self.deadMask = self.ob.getDeadPixels() np.savez(self.deadPixelFilename,deadMask = self.deadMask) print "Dead pixel mask saved to %s"%(self.deadPixelFilename) else: self.deadFile = np.load(self.deadPixelFilename) self.deadMask = self.deadFile['deadMask'] # 12) Create output name def createOutputName(self): self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's') if self.useWavelengthCalibration and self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.npz') elif self.useWavelengthCalibration and not self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.npz') elif not self.waveLengthCalibration and self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_hp.npz') else: self.outputFilename = str(self.rawName + '.npz') # Start process for creating image stacks def stackProcess(self): # Check for valid params file if self.paramFileExists: # 6) Load settings choosen from gui self.loadSettings() if self.validSettings: # Loop through all files in input list for iFile in range(self.fileCount): # Create ObsFile instance self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text() self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs()) print 'Processing file ' + self.obsFn + '...' self.ob = ObsFile(self.obsFn) # Load time adjustment file if self.useTimeAdjustment: print 'Loading time adjustment file...' self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments()) # 7) Load hot pixel mask if self.useHotPixelMasking: print 'Loading hot pixel mask...' self.loadHotMask() # 8) Load wave cal solution if self.useWavelengthCalibration: print 'Loading wavelength calibration...' self.ob.loadWvlCalFile(self.wvlCalFilename) # 9) Load flatcal solution if self.useFlatCalibration: print 'Loading flat calibration...' self.ob.loadFlatCalFile(self.flatCalFilename) # 10) Load dead pixel mask if self.useDeadPixelMasking: print 'Loading dead pixel mask...' self.loadDeadMask() # 11) Set wavelength cutoffs if self.useWavelengthCalibration: print 'Setting wavelength cutoffs...' self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff) # Timing self.unix = self.ob.getFromHeader('unixtime') self.startJD = self.unix/86400.+2440587.5 self.exptime = self.ob.getFromHeader('exptime') self.times = [] self.frames = [] # 11) Create Image Stack print 'Stacking images...' for iSec in np.arange(0,self.exptime,self.integrationTime): #add seconds offset to julian date, move jd to center of bin self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.) 'Creating frame for time ' + self.jd self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt) self.frame = self.frameData['image'] if self.useDeadPixelMasking: self.frame[self.deadMask == 0] = np.nan self.frames.append(self.frame) self.cube = np.dstack(self.frames) self.times = np.array(self.times) # 12) Create output file self.createOutputName() print 'Saving image stack to ' + self.outputFilename np.savez(self.outputFilename, stack=self.cube, jd=self.times) # Invalid params file else: print 'Invalid parameter file...' # Choose an image stack def chooseStack(self): self.defaultLoadStackDirectory = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks') self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("NPZ (*.npz)")) if self.stackName == '': print 'No file chosen' else: self.loadStack() # Load that image stack for viewing def loadStack(self):
NumFiles = sum(NumFiles) print (NumFiles)*expTime/integrationTime,'frames to make' for iSeq in range(len(seqs)): timestampList = timestampLists[iSeq] print timestampList wfn = wvlCalFilenames[iSeq] ffn = flatCalFilenames[iSeq] sunsetDate = sunsetDates[iSeq] for i,ts in enumerate(timestampList): print 'loading',ts obsFn = FileName(run=run,date=sunsetDate,tstamp=ts,mkidDataDir=mkidDataDir, intermDir=intermDir).obs() ob = ObsFile(obsFn) ob.loadTimeAdjustmentFile(FileName(run=run,mkidDataDir=mkidDataDir, intermDir=intermDir).timeAdjustments()) #index1 = obsFn.find('_') #hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:] hotPixFn = FileName(run=run,date=sunsetDate,tstamp=ts,mkidDataDir=mkidDataDir, intermDir=intermDir).timeMask() if not os.path.exists(hotPixFn): hp.findHotPixels(obsFn,hotPixFn) print "Flux file pixel mask saved to %s"%(hotPixFn) print 'Loading cal files' ob.loadHotPixCalFile(hotPixFn,switchOnMask=True) ob.loadWvlCalFile(wfn) ob.loadFlatCalFile(ffn) ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)
# test the theory that that extra photons show up 23 clock ticks # after the beginning of each second. Plot properties of the photons offsets = [23, 100, 1234, 54321] #offsets = [23] beginTime = 0 expTime = 300 pickleFile = open("csb2.pkl","wb") fn = FileName(run, sundownDate, obsDate+"-"+seq) pickle.dump(fn,pickleFile) obsFile = ObsFile(fn.obs()) obsFile.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments()) # Matt's time fix timeMaskFile = fn.timeMask() if os.path.exists(timeMaskFile): obsFile.loadHotPixCalFile(timeMaskfile, switchOnMask=True) if False: iRow0 = 25 iRow1 = iRow0+1 iCol0 = 40 iCol1 = iCol0+1 else: iRow0 = 0 iRow1 = obsFile.nRow iCol0 = 0 iCol1 = obsFile.nCol
class DisplayStack(QMainWindow): def __init__(self): # Start up gui QWidget.__init__(self, parent=None) self.ui = Ui_DisplayStack_gui() self.ui.setupUi(self) # Initialize Variables self.initializeVariables() # Lists and buttons used for specifying run and target information. # Click item in runList to select a run. Run number corresponds to array index. Load up corresponding target list. self.ui.runList.itemClicked.connect(self.selectRun) # Click item in targetList to select a target. self.ui.targetList.itemClicked.connect(self.selectTarget) # Click button in self.ui.targetButton.clicked.connect(self.loadTarget) self.ui.sunsetList.itemClicked.connect(self.createObsList) # Use wavelength calibration checkbox self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal) # Use flat calibration checkbox self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal) # Buttons for obs list creation self.ui.addButton.clicked.connect(self.addObservation) self.ui.removeButton.clicked.connect(self.removeObservation) self.ui.clearButton.clicked.connect(self.createObsList) # Start process button self.ui.stackButton.clicked.connect(self.stackProcess) # Load image stack button self.ui.loadStackButton.clicked.connect(self.chooseStack) # Initialize Variables def initializeVariables(self): # Define path names self.displayStackPath = '/Scratch/DisplayStack/' self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/PAL2012/master_cals/' self.defaultFlatPath = '/Scratch/flatCalSolnFiles/' # Load and display list of run names from /Scratch/DisplayStack/runList.dict self.loadRunData() # Load list of target names from /Scratch/DisplayStack/runName/runName.dict, for all runs self.loadTargetData() # Function to load list of run names. Runs on initialization. def loadRunData(self): # Load run data from /Scratch/DisplayStack/runList.dict self.runData = readDict() self.runData.read_from_file(self.displayStackPath + '/runList.dict') self.runNames = np.array(self.runData['runs']) # Populate runList table with run names. for iRun in range(len(self.runNames)): self.ui.runList.addItem(self.runNames[iRun]) # Function to load a table of target names. def loadTargetData(self): self.targetNames = [] # Cycle through runs and extract target name information from various dictionaries. for iRun in range(len(self.runNames)): self.targetData = readDict() self.targetData.read_from_file(self.displayStackPath + self.runNames[iRun] + '/' + self.runNames[iRun] + '.dict') self.iTargets = np.array(self.targetData['targets']) self.targetNames.append(self.iTargets) # Function to select a run and populate the target list for that particular run. def selectRun(self): # Clear list of target information for previously selected run. self.ui.targetList.clear() # Define a run number by the index of the selected run. self.runNumber = self.ui.runList.row(self.ui.runList.currentItem()) # Populate targetList table with target names for selected run. for iTarget in range(len(self.targetNames[self.runNumber])): self.ui.targetList.addItem(self.targetNames[self.runNumber][iTarget]) def selectTarget(self): self.targetNumber = self.ui.targetList.row(self.ui.targetList.currentItem()) def loadTarget(self): self.run = self.runNames[self.runNumber] self.target = self.targetNames[self.runNumber][self.targetNumber] try: self.paramName = self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.paramData = readDict() self.paramData.read_from_file(self.paramName) self.obsTimes = np.array(self.paramData['obsTimes']) self.utcDates = self.paramData['utcDates'] self.sunsetDates = self.paramData['sunsetDates'] self.calTimestamps = self.paramData['calTimestamps'] self.flatCalDates = self.paramData['flatCalDates'] self.RA = self.paramData['RA'] self.Dec = self.paramData['Dec'] self.hourAngleOffset = self.paramData['HA_offset'] print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.createSunsetList() #self.createObsList() self.createWavelengthList() self.createFlatList() self.paramFileExists = True except IOError: print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.ui.sunsetList.clear() self.ui.obsList.clear() self.ui.inputList.clear() self.ui.wavelengthList.clear() self.ui.flatList.clear() self.paramFileExists = False # Choose Obs File # Create list of available sunset dates def createSunsetList(self): self.ui.sunsetList.clear() for iDate in range(len(self.sunsetDates)): self.ui.sunsetList.addItem(self.sunsetDates[iDate]) # Create Initial Obs file list def createObsList(self): self.ui.obsList.clear() self.ui.inputList.clear() self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] for iObs in range(len(self.singleDayObservations)): self.ui.obsList.addItem(self.singleDayObservations[iObs]) # Add observation to input list def addObservation(self): self.selectedObs = self.ui.obsList.currentItem() self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs)) self.ui.inputList.addItem(self.selectedObs) self.ui.inputList.sortItems() # Remove observation from input list def removeObservation(self): self.removedObs = self.ui.inputList.currentItem() self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs)) self.ui.obsList.addItem(self.removedObs) self.ui.obsList.sortItems() # Load settings def loadSettings(self): self.validSettings = True # Run and target information self.run = self.runNames[self.runNumber] self.target = self.targetNames[self.runNumber][self.targetNumber] # General settings self.integrationTime = int(self.ui.integrationTimeLine.text()) self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked() self.useHotPixelMasking = self.ui.hotPixelBox.isChecked() # Wavelength calibration settings self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked() self.useBestWavelengthCalibration = self.ui.bestWavelengthCalibrationBox.isChecked() self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text()) self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text()) # Flat calibration settings self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked() self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked() self.fileCount = self.ui.inputList.count() self.weighted = self.useFlatCalibration self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration) self.scaleByEffInt = self.useHotPixelMasking if self.ui.sunsetList.currentItem() != None: if self.fileCount == 0: print 'Please select files to process...' self.validSettings = False if self.useWavelengthCalibration: if self.useBestWavelengthCalibration: print 'Using best wavelength calibration...' elif self.ui.wavelengthList.currentItem() == None: print 'Please select wavelength calibration...' self.validSettings = False else: self.selectedWvlCal = self.ui.wavelengthList.currentItem().text() print "Chose wvl cal: ", str(self.defaultWavelengthPath+self.selectedWvlCal) self.wvlCalFilename = str(self.defaultWavelengthPath+self.selectedWvlCal) #self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln()) if self.useFlatCalibration: if self.ui.flatList.currentItem() == None: print 'Please select flat calibration...' self.validSettings = False else: self.flatCalNight = self.ui.flatList.currentItem().text() self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln()) else: print 'Please select sunset night...' self.validSettings = False # Load hot pixel mask def loadHotMask(self): #self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5') self.hotPixelFilename = str(FileName(obsFile = self.ob).timeMask()) if not os.path.exists(self.hotPixelFilename): hp.findHotPixels(obsFile=self.ob,outputFileName=self.hotPixelFilename) print "Hot pixel mask saved to %s"%(self.hotPixelFilename) self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True) # Create wavelength cal file list def createWavelengthList(self): self.ui.wavelengthList.clear() for iCal in range(len(self.calTimestamps)): self.ui.wavelengthList.addItem(self.calTimestamps[iCal]) # Enable/disable wavecal options def useWaveCal(self): if self.ui.wavelengthCalibrationBox.isChecked(): self.ui.lowerWavelengthCutoffLine.setEnabled(True) self.ui.lowerWavelengthCutoffLabel.setEnabled(True) self.ui.upperWavelengthCutoffLine.setEnabled(True) self.ui.upperWavelengthCutoffLabel.setEnabled(True) self.ui.wavelengthList.setEnabled(True) self.ui.flatCalibrationBox.setEnabled(True) self.ui.flatCalibrationBox.setChecked(True) self.ui.deadPixelBox.setEnabled(True) self.ui.deadPixelBox.setChecked(True) self.ui.flatList.setEnabled(True) self.ui.bestWavelengthCalibrationBox.setEnabled(True) self.ui.bestWavelengthCalibrationBox.setChecked(True) else: self.ui.lowerWavelengthCutoffLine.setEnabled(False) self.ui.lowerWavelengthCutoffLabel.setEnabled(False) self.ui.upperWavelengthCutoffLine.setEnabled(False) self.ui.upperWavelengthCutoffLabel.setEnabled(False) self.ui.wavelengthList.setEnabled(False) self.ui.flatCalibrationBox.setEnabled(False) self.ui.flatCalibrationBox.setChecked(False) self.ui.deadPixelBox.setChecked(False) self.ui.deadPixelBox.setEnabled(False) self.ui.flatList.setEnabled(False) self.ui.bestWavelengthCalibrationBox.setEnabled(False) self.ui.bestWavelengthCalibrationBox.setChecked(False) # Create flat cal file list def createFlatList(self): self.ui.flatList.clear() for iCal in range(len(self.flatCalDates)): self.ui.flatList.addItem(self.flatCalDates[iCal]) # Enable/disable flatcal options def useFlatCal(self): if self.ui.flatCalibrationBox.isChecked(): self.ui.deadPixelBox.setEnabled(True) self.ui.deadPixelBox.setChecked(True) self.ui.flatList.setEnabled(True) else: self.ui.deadPixelBox.setChecked(False) self.ui.deadPixelBox.setEnabled(False) self.ui.flatList.setEnabled(False) # Load dead pixel mask def loadDeadMask(self): self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz') if not os.path.exists(self.deadPixelFilename): self.deadMask = self.ob.getDeadPixels() np.savez(self.deadPixelFilename,deadMask = self.deadMask) print "Dead pixel mask saved to %s"%(self.deadPixelFilename) else: self.deadFile = np.load(self.deadPixelFilename) self.deadMask = self.deadFile['deadMask'] # Describe the structure of the header row class headerDescription(tables.IsDescription): targetName = tables.StringCol(100, dflt='') run = tables.StringCol(100, dflt='') obsFileName = tables.StringCol(100, dflt='') wvlCalFileName = tables.StringCol(100, dflt=np.nan) flatCalFileName = tables.StringCol(100, dflt='') deadPixFileName = tables.StringCol(100, dflt='') hotPixFileName = tables.StringCol(100, dflt='') nCol = tables.UInt32Col(dflt=-1) nRow = tables.UInt32Col(dflt=-1) lowWvlCutoff = tables.Float64Col(dflt=np.nan) highWvlCutoff = tables.Float64Col(dflt=np.nan) exptime = tables.Float64Col(dflt=np.nan) lst = tables.StringCol(100, dflt='') integrationTime = tables.Float64Col(dflt=np.nan) RA = tables.StringCol(100, dflt='') Dec = tables.StringCol(100, dflt='') HA_offset = tables.Float64Col(dflt=0.0) # Create output name def createOutputName(self): self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's') if self.useWavelengthCalibration and self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.h5') elif self.useWavelengthCalibration and not self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.h5') elif not self.useWavelengthCalibration and self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_hp.h5') else: self.outputFilename = str(self.rawName + '.h5') def createH5File(self): # Create header and data group and table names headerGroupName = 'header' headerTableName = 'header' dataGroupName = 'stack' dataTableName = 'stack' timeTableName = 'time' # Create lookup names for header information runColName = 'run' targetColName = 'targetName' obsFileColName = 'obsFileName' wvlCalFileColName = 'wvlCalFileName' flatCalFileColName = 'flatCalFileName' nRowColName = 'nRow' nColColName = 'nCol' RAColName = 'RA' DecColName = 'Dec' deadPixColName = 'deadPixFileName' hotPixColName = 'hotPixFileName' lowWvlColName = 'lowWvlCutoff' highWvlColName = 'highWvlCutoff' expTimeColName = 'exptime' lstColName = 'lst' integrationTimeColName = 'integrationTime' HA_offsetColName = 'HA_offset' # Create and h5 output file, create header and data groups fileh = tables.openFile(self.outputFilename, mode='w') headerGroup = fileh.createGroup("/", headerGroupName, 'Header') stackGroup = fileh.createGroup("/", dataGroupName, 'Image Stack') # Create row for header information headerTable = fileh.createTable(headerGroup, headerTableName, self.headerDescription, 'Header Info') header = headerTable.row # Fill in the header with possibly useful information. header[runColName] = self.run header[targetColName] = self.target header[obsFileColName] = self.obsFn header[nColColName] = self.numberCols header[nRowColName] = self.numberRows header[RAColName] = self.RA header[DecColName] = self.Dec header[expTimeColName] = self.exptime header[lstColName] = self.lst header[integrationTimeColName] = self.integrationTime header[HA_offsetColName] = self.hourAngleOffset if self.useDeadPixelMasking: header[deadPixColName] = self.deadPixelFilename if self.useHotPixelMasking: header[hotPixColName] = self.hotPixelFilename if self.useWavelengthCalibration: header[wvlCalFileColName] = self.ob.wvlCalFileName header[lowWvlColName] = self.lowerWavelengthCutoff header[highWvlColName] = self.upperWavelengthCutoff if self.useFlatCalibration: header[flatCalFileColName] = self.flatCalFilename header.append() # Create an h5 array for the midtime of each frame in the image cube. timeTable = fileh.createCArray(stackGroup, timeTableName, Float64Atom(), (1,len(self.times))) timeTable[:] = self.times # Create an h5 table for the image cube. stackTable = fileh.createCArray(stackGroup, dataTableName, Float64Atom(), (self.numberRows,self.numberCols, self.cube.shape[2])) stackTable[:] = self.cube # Flush the h5 output file fileh.flush() fileh.close() # Start process for creating image stacks def stackProcess(self): # Check for valid params file if self.paramFileExists: # Load settings choosen from gui self.loadSettings() if self.validSettings: # Loop through all files in input list for iFile in range(self.fileCount): # Create ObsFile instance self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text() self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs()) print 'Processing file ' + self.obsFn + '...' self.ob = ObsFile(self.obsFn) self.numberRows = self.ob.nRow self.numberCols = self.ob.nCol # Load time adjustment file if self.useTimeAdjustment: print 'Loading time adjustment file...' self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments()) # Load hot pixel mask if self.useHotPixelMasking: print 'Loading hot pixel mask...' self.loadHotMask() # Load wave cal solution if self.useWavelengthCalibration: if self.useBestWavelengthCalibration: print 'Loading best wavelength calibration...' self.ob.loadBestWvlCalFile() else: print 'Loading selected wavelength calibration...' self.ob.loadWvlCalFile(self.wvlCalFilename) # Load flatcal solution if self.useFlatCalibration: print 'Loading flat calibration...' self.ob.loadFlatCalFile(self.flatCalFilename) # Load dead pixel mask if self.useDeadPixelMasking: print 'Loading dead pixel mask...' self.loadDeadMask() # Set wavelength cutoffs if self.useWavelengthCalibration: print 'Setting wavelength cutoffs...' self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff) # Timing self.unix = self.ob.getFromHeader('unixtime') self.startJD = self.unix/86400.+2440587.5 self.exptime = self.ob.getFromHeader('exptime') self.lst = self.ob.getFromHeader('lst') self.times = [] self.frames = [] # Create Image Stack print 'Stacking images...' for iSec in np.arange(0,self.exptime,self.integrationTime): #add seconds offset to julian date, move jd to center of bin self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.) self.times.append(self.jd) print 'Creating frame for time ' + str(self.jd) self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt) self.frame = self.frameData['image'] if self.ui.verticalFlipBox.isChecked(): self.frame = np.flipud(self.frame) if self.useDeadPixelMasking: self.frame[self.deadMask == 0] = np.nan self.frames.append(self.frame) self.cube = np.dstack(self.frames) self.times = np.array(self.times) # Create output file self.createOutputName() print 'Saving image stack to ' + self.outputFilename self.createH5File() # Invalid params file else: print 'Invalid parameter file...' # Choose an image stack def chooseStack(self): self.defaultLoadStackDirectory = str(self.displayStackPath) self.stackName = '' self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("H5 (*.h5)")) if self.stackName == '': print 'No file chosen' else: loadStackApp = LoadImageStack.LoadImageStack(stackName = self.stackName) loadStackApp.show() loadStackApp.exec_()
def main(): #open the sky file for hr9087 run = 'PAL2012' date = '20121210' wvlCal = '20121211-052230' obsTimestamp = '20121211-051650' flatCalDate = '20121211' flatCalTstamp = '20121212-074700' obsFN = FileName(run=run,date=date,tstamp=obsTimestamp) obsFileName = obsFN.obs() timeMaskFileName = obsFN.timeMask() wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln() flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln() if not os.path.exists(timeMaskFileName): print 'Running hotpix for ',obsFileName hp.findHotPixels(obsFileName,timeMaskFileName) print "Flux file pixel mask saved to %s"%(timeMaskFileName) obs = ObsFile(obsFileName) obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments()) obs.loadWvlCalFile(wvlFileName) obs.loadFlatCalFile(flatFileName) obs.loadHotPixCalFile(timeMaskFileName) #obs.setWvlCutoffs(4000,8000) #get image before and after flat cal print 'getting images' beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True) rawCubeDict = obs.getSpectralCube(weighted=False) rawCube = np.array(rawCubeDict['cube'],dtype=np.double) effIntTime = rawCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) rawCube *= maxIntTime / effIntTime3d rawCube[np.isnan(rawCube)] = 0 rawCube[rawCube == np.inf] = 0 beforeImg = np.sum(rawCube,axis=-1) print 'finished first cube' flatCubeDict = obs.getSpectralCube(weighted=True) flatCube = np.array(flatCubeDict['cube'],dtype=np.double) effIntTime = flatCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) flatCube *= maxIntTime / effIntTime3d flatCube[np.isnan(flatCube)] = 0 flatCube[flatCube == np.inf] = 0 afterImg = np.sum(flatCube,axis=-1) plotArray(title='before flatcal',image=beforeImg) plotArray(title='after flatcal',image=afterImg) print 'before sdev',np.std(beforeImg[afterImg!=0]) print 'after sdev',np.std(afterImg[afterImg!=0]) np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
app_mask = aperture(15,8,8) y_values,x_values = np.where(app_mask==0) unixOffset=0.0 timestamps =[] tic = time() for iSeq in range(len(seqs)): timestampList = timestampLists[iSeq] wfn = wvlCalFilenames[iSeq] ffn = flatCalFilenames[iSeq] sunsetDate = sunsetDates[iSeq] for i,ts in enumerate(timestampList): print 'Loading',ts obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs() ob = ObsFile(obsFn) ob.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments()) exptime = ob.getFromHeader('exptime') unixtime= ob.getFromHeader('unixtime') if i == 0: unixOffset = unixtime exptimes[i]=exptime index1 = obsFn.find('_') hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:] if not os.path.exists(hotPixFn): hp.findHotPixels(obsFn,hotPixFn) print "Flux file pixel mask saved to %s"%(hotPixFn) ob.loadHotPixCalFile(hotPixFn,switchOnMask=True) ob.loadWvlCalFile(wfn) ob.loadFlatCalFile(ffn) ob.setWvlCutoffs(3000,8000) for j in range(len(x_values)):