def generateObsObjectList(obsFNs,wvlLowerLimit=3000, wvlUpperLimit=13000,beammapFileName = None,loadHotPix=True,loadWvlCal=True,loadFlatCal=True,loadSpectralCal=True): obsFiles = [] for obsFN in obsFNs: if type(obsFN) == type(''): #Full path to obs file obs = ObsFile(obsFN) else: #FileName object obs = ObsFile(obsFN.obs()) if beammapFileName is not None and os.path.isfile(beammapFileName): obs.loadBeammapFile(beammapFileName) obs.setWvlCutoffs(wvlLowerLimit=wvlLowerLimit, wvlUpperLimit=wvlUpperLimit) if loadHotPix: if not os.path.isfile(FileName(obsFile=obs).timeMask()): print "generating hp file ", FileName(obsFile=obs).timeMask() hp.findHotPixels(obsFile=obs,outputFileName=FileName(obsFile=obs).timeMask()) obs.loadHotPixCalFile(FileName(obsFile=obs).timeMask(),reasons=['hot pixel','dead pixel']) if loadWvlCal: obs.loadBestWvlCalFile() if loadFlatCal: #obs.loadFlatCalFile(FileName(obsFile=obs).flatSoln()) obs.loadFlatCalFile('/Scratch/flatCalSolnFiles/flatsol_1s.h5') if loadSpectralCal: pass obsFiles.append(obs) return obsFiles
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess,hotPath,flatPath): obs = ObsFile(obsPath) print obsPath,obs.getFromHeader('exptime'),obs if not os.path.exists(hotPath): hp.findHotPixels(obsFile=obs,outputFileName=hotPath) obs.loadHotPixCalFile(hotPath,switchOnMask=False) obs.loadBestWvlCalFile() obs.loadFlatCalFile(flatPath) obs.setWvlCutoffs(3000,8000) cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath) print 'done centroid',centroidPath del obs
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=True): ''' Test run of obsFile.writePhotonList. fileName can be used to specify the output file name. If not specified, default name/location is used. Now includes test for pixel remapping.... ''' #Details of example obs file to run test on. # run = 'PAL2012' # date = '20121207' # tstamp = '20121208-074649' # calTstamp='20121208-070505' # fluxTstamp='20121208-133002' run = 'PAL2012' date = '20121208' tstamp = '20121209-120530' centroidTstamp = '20121209-120530' calTstamp='20121209-131132' fluxTstamp='20121209-020416' flatTstamp='20121209-021036' if doPixRemap==True: pixRemapFileName = FileName(run=run).pixRemap() else: pixRemapFileName = None #Load up the obs file obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) #Load up associated calibrations obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln()) obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln()) obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln()) obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments()) obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask()) obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList()) #And write out the results.... obsFile.writePhotonList(outputFileName,firstSec,integrationTime, pixRemapFileName=pixRemapFileName)
class Cosmic: def __init__(self, fn, beginTime=0, endTime='exptime', nBinsPerSec=10, flashMergeTime=1.0, applyCosmicMask = False, loggingLevel=logging.CRITICAL, loggingHandler=logging.StreamHandler()): """ Opens fileName in MKID_RAW_PATH, sets roachList endTime is exclusive """ self.logger = logging.getLogger("cosmic") self.logger.setLevel(loggingLevel) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') loggingHandler.setFormatter(formatter) self.logger.addHandler(loggingHandler) self.logger.info("Cosmic: begin init for obsFile=%s"%fn.obs()) self.fn = fn self.fileName = fn.obs(); self.file = ObsFile(self.fileName) # apply Matt's time fix timeAdjustments = self.fn.timeAdjustments() if os.path.exists(timeAdjustments): self.file.loadTimeAdjustmentFile(timeAdjustments) # apply Julian's time masks timeMaskFile = self.fn.timeMask(); if os.path.exists(timeMaskFile): self.file.loadHotPixCalFile(timeMaskFile,switchOnMask=True) # apply standard mask if applyCosmicMask: self.file.loadStandardCosmicMask() self._setRoachList() self._setAllSecs() self.exptime = self.file.getFromHeader('exptime') if endTime =='exptime': self.endTime = float(self.exptime) else: self.endTime = float(endTime) if ( (self.endTime > self.exptime) or (endTime < 0)): raise RuntimeError("bad endTime: endTime=%s exptime=%s" % \ (str(endTime),str(self.exptime))) self.beginTime = float(beginTime) self.timeHgs = "none" self.nBinsPerSec = nBinsPerSec self.flashMergeTime = flashMergeTime self.times = \ np.arange(self.beginTime, self.endTime, 1.0/self.nBinsPerSec) # for measuring flashes, indexed by roach name self.rMean = {} # mean from meanclip self.rSigma = {} # sigma from meanclip self.rNSurvived = {} # number of survivors from meanclip self.rNormed = {} # (value-mean)/sigma self.flashInterval = {} self.logger.info("Cosmic: end init: beginTime=%s endTime=%s"%(str(self.beginTime),str(self.endTime))) def __del__(self): """ Close any open files """ # print "now in Cosmic.__del__ for ",self.fileName try: del self.file except: pass def _setRoachList(self): self.roachList = [] for row in self.file.beamImage: for name in row: roachName = name.split("/")[1] if roachName not in self.roachList: self.roachList.append(roachName) self.roachList.sort() def _setAllSecs(self): nRow = self.file.nRow nCol = self.file.nCol self.allSecs = \ dict( ((i,j),None) for i in range(nRow) for j in range(nCol)) for iRow in np.arange(nRow): for iCol in np.arange(nCol): self.allSecs[iRow,iCol] = \ self.file.file.getNode(self.file.beamImage[iRow][iCol]) def nPhoton(self, beginTime=0, endTime='expTime'): """ trivial example of counting the number of photons in a file """ nPhoton = 0 for iRow in range(self.file.nRow): for iCol in range(self.file.nCol): for iSec in range(self.beginTime, self.endTime): sec = self.allSecs[iRow,iCol][iSec] nPhoton += len(sec) return nPhoton def findFlashes(self, clipsig=3.0, maxiter=5,\ converge_num=0.05, verbose=0, flashsig=6): """ find flashes by looking at the time histograms. Calculate the mean,sigma using meanclip and the parameters clipsig, maxiter, converge_num A time bin has a flash if the normalized signal (measured-mean)/sigma is larger than flashsig. """ # make the blank data structures if self.timeHgs == "none": self.makeTimeHgs() self.flashInterval["all"] = [] # find the flashes in each roach for roach in self.roachList: self.rMean[roach],self.rSigma[roach],self.rNSurvived[roach] = \ meanclip.meanclip(\ np.array(self.timeHgs[roach]), clipsig, maxiter, converge_num,\ verbose) self.rNormed[roach] = \ (self.timeHgs[roach]-self.rMean[roach])/self.rSigma[roach] self.flashInterval[roach] = interval() prev = 0 a = self.rNormed[roach] for i in range(len(a)): this = a[i] > flashsig if (this != prev): if (this): iBegin = i else: self.flashInterval[roach] = \ self.flashInterval[roach] | interval[iBegin,i] prev = this if (prev): self.flashInterval[roach] = \ self.flashInterval[roach] | interval[iBegin,i] # union of all flashes self.flashInterval["all"] = interval() for roach in self.roachList: self.flashInterval["all"] = \ self.flashInterval["all"] | self.flashInterval[roach] # look for gaps smaller than self.flashMergeTime and plug them dMax = self.nBinsPerSec*self.flashMergeTime extrema = self.flashInterval["all"].extrema for i in range(len(extrema)/2-1): i0 = extrema[2*i+1][0] i1 = extrema[2*(i+1)][0] if (i1-i0) <= dMax: t0 = self.beginTime + float(i0)/self.nBinsPerSec t1 = self.beginTime + float(i1)/self.nBinsPerSec self.flashInterval["all"] = \ self.flashInterval["all"] | interval[i0,i1] # convert to ticks since the beginning of the data file rlAll = list(self.roachList) rlAll.append("all") ticksPerSecond = int(1.0/self.file.tickDuration) offset = self.beginTime*ticksPerSecond scale = 1.0/(self.file.tickDuration*self.nBinsPerSec) for roach in rlAll: self.flashInterval[roach] = offset+scale*self.flashInterval[roach] def writeFlashesToHdf5(self,overwrite=1): """ write intervals with flashes to the timeMask file """ # get the output file name, and make the directory if you need to cmFileName = self.fn.cosmicMask() (cmDir,name) = os.path.split(cmFileName) if not os.path.exists(cmDir): os.makedirs(cmDir) # write parameters used to find flashes h5f = tables.openFile(cmFileName, 'w') fnode = filenode.newNode(h5f, where='/', name='timeMaskHdr') fnode.attrs.beginTime = self.beginTime fnode.attrs.endTime = self.endTime fnode.attrs.nBinsPerSec = self.nBinsPerSec fnode.attrs.flashMergeTime = self.flashMergeTime fnode.close(); # write the times where flashes are located tbl = h5f.createTable('/','timeMask',TimeMask.TimeMask,"Time Mask") rlAll = list(self.roachList) rlAll.append("all") for roach in rlAll: extrema = self.flashInterval[roach].extrema for i in range(len(extrema)/2): row = tbl.row row['tBegin'] = int(extrema[2*i][0]) row['tEnd'] = int(extrema[2*i+1][0]) if (roach == "all"): reason = "Merged Flash" else: reason = "Flash in %s" % roach row['reason'] = TimeMask.timeMaskReason[reason] row.append() tbl.flush() tbl.close() h5f.close() def makeTimeHgs(self): """ Fill in the timeHgs variable This is a dictionary, indexed by the roach name, of the time histograms """ self.timeHgs = {} for iSec in range(self.beginTime, self.endTime): self.logger.info("in makeTimeHgs iSec=%4d / %4d" % (iSec,self.endTime)) hgsThisSec = {} for iRow in range(self.file.nRow): for iCol in range(self.file.nCol): sec = self.allSecs[iRow,iCol][iSec] if len(sec) > 0: times = sec & self.file.timestampMask hg,edges = \ np.histogram(times,bins=self.nBinsPerSec, \ range=(0,1.0/self.file.tickDuration)) roachName = \ self.file.beamImage[iRow][iCol].split("/")[1] if not hgsThisSec.has_key(roachName): hgsThisSec[roachName] = \ np.zeros(self.nBinsPerSec,dtype=np.int64) hgsThisSec[roachName] += hg for roachName in hgsThisSec.keys(): if not self.timeHgs.has_key(roachName): self.timeHgs[roachName] = [] self.timeHgs[roachName] += list(hgsThisSec[roachName]) def plotTimeHgs(self): """ Plot the time HGS in plt structure, with legend """ plt.clf() plt.figure(1) keys = self.timeHgs.keys() keys.sort() plt.subplot(211) for roachName in keys: hg = self.timeHgs[roachName] plt.plot(self.times, hg,label=roachName) plt.legend() dt = 1.0/self.nBinsPerSec plt.ylabel("photons/%.2f sec" % dt) plt.title("Cosmic timeHgs "+ self.fileName) plt.subplot(212) for roachName in keys: plt.plot(self.times, \ self.rNormed[roachName],label=roachName) plt.xlabel("time (sec)") plt.ylim(-23,30) dt = 1.0/self.nBinsPerSec plt.ylabel("normalized photons/%.2f sec" % dt) y = -5 x0 = self.beginTime + 0.1*(self.endTime-self.beginTime) xmax = plt.xlim()[1] rlAll = list(self.roachList) rlAll.append("all") for roach in rlAll: print "plot for roach=",roach plt.plot([x0,xmax],[y,y], linestyle=":", color="gray") plt.text(x0, y, roach, fontsize=8, va="center") extrema = self.flashInterval[roach].extrema for i in range(len(extrema)/2): t0 = (extrema[2*i][0] - 0.5)*self.file.tickDuration t1 = (extrema[2*i+1][0] - 0.5)*self.file.tickDuration plt.plot([t0,t1],[y,y],'r', linewidth=4) y -= 2 def findCosmics(self, stride=10, threshold=100, populationMax=2000, nSigma=5, writeCosmicMask=False, ppsStride=10000): """ Find cosmics ray suspects. Histogram the number of photons recorded at each timeStamp. When the number of photons in a group of stride timeStamps is greater than threshold in second iSec, add (iSec,timeStamp) to cosmicTimeLists. Also keep track of the histogram of the number of photons per stride timeStamps. return a dictionary of 'populationHg', 'cosmicTimeLists', 'binContents', 'timeHgValues', 'interval', 'frameSum', and 'pps' populationHg is a histogram of the number of photons in each time bin. This is a poisson distribution with a long tail due to cosmic events cosmicTimeLists is a numpy array of all the sequences that are suspects for cosmic rays binContents corresponds to cosmicTimeLists. For each time in cosmicTimeLists, binContents is the number of photons detected at that time. timeHgValues is a histogram of the number of photons in each time interval frameSum is a two dimensional numpy array of the number of photons detected by each pixel interval is the interval of data to be masked out pps is photons per second, calculated every ppsStride bins. """ self.logger.info("findCosmics: begin stride=%d threshold=%d populationMax=%d nSigma=%d writeCosmicMask=%s"%(stride,threshold,populationMax,nSigma,writeCosmicMask)) exptime = self.endTime-self.beginTime nBins = int(np.round(self.file.ticksPerSec*exptime+1)) bins = np.arange(0, nBins, 1) timeHgValues,frameSum = self.getTimeHgAndFrameSum(self.beginTime,self.endTime) remainder = len(timeHgValues)%ppsStride if remainder > 0: temp = timeHgValues[:-remainder] else: temp = timeHgValues ppsTime = (ppsStride*self.file.tickDuration) pps = np.sum(temp.reshape(-1, ppsStride), axis=1)/ppsTime self.logger.info("findCosmics: call populationFromTimeHgValues") pfthgv = Cosmic.populationFromTimeHgValues\ (timeHgValues,populationMax,stride,threshold) #now build up all of the intervals in seconds self.logger.info("findCosmics: build up intervals: nCosmicTime=%d"%len(pfthgv['cosmicTimeList'])) i = interval() iCount = 0 secondsPerTick = self.file.tickDuration for cosmicTime in pfthgv['cosmicTimeList']: #t0 = max(0,self.beginTime+(cosmicTime-50)/1.e6) #t1 = min(self.endTime,self.beginTime+(cosmicTime+50)/1.e6) #intTime = t1-t0 t0 = self.beginTime+cosmicTime*secondsPerTick dt = stride*secondsPerTick t1 = t0+dt left = max(self.beginTime, t0-nSigma*dt) right = min(self.endTime, t1+2*nSigma*dt) i = i | interval[left,right] self.logger.debug("findCosmics: iCount=%d t0=%f t1=%f left=%f right=%f"%(iCount,t0,t1,left,right)) iCount+=1 tMasked = Cosmic.countMaskedBins(i) ppmMasked = 1000000*tMasked/(self.endTime-self.beginTime) retval = {} retval['timeHgValues'] = timeHgValues retval['populationHg'] = pfthgv['populationHg'] retval['cosmicTimeList'] = pfthgv['cosmicTimeList'] retval['binContents'] = pfthgv['binContents'] retval['frameSum'] = frameSum retval['interval'] = i retval['ppmMasked'] = ppmMasked retval['pps'] = pps retval['ppsTime'] = ppsTime if writeCosmicMask: cfn = self.fn.cosmicMask() self.logger.info("findCosmics: write masks to =%s"%cfn) ObsFile.writeCosmicIntervalToFile(i, self.file.ticksPerSec, cfn,self.beginTime, self.endTime, stride, threshold, nSigma, populationMax) self.logger.info("findCosmics: end with ppm masked=%d"%ppmMasked) return retval def getTimeHgAndFrameSum(self, beginTime, endTime): integrationTime = endTime - beginTime nBins = int(np.round(self.file.ticksPerSec*integrationTime+1)) timeHgValues = np.zeros(nBins, dtype=np.int64) frameSum = np.zeros((self.file.nRow,self.file.nCol)) self.logger.info("get all time stamps for integrationTime=%f"%integrationTime) for iRow in range(self.file.nRow): #print "Cosmic.findCosmics: iRow=",iRow for iCol in range(self.file.nCol): # getTimedPacketList is slow. Use getPackets instead. #gtpl = self.file.getTimedPacketList(iRow,iCol,beginTime, # integrationTime) gtpl = self.file.getPackets(iRow,iCol, beginTime,integrationTime) timestamps = gtpl['timestamps'] if timestamps.size > 0: timestamps = \ (timestamps - beginTime)*self.file.ticksPerSec # per Matt S. suggestion 2013-07-09 ts32 = np.round(timestamps).astype(np.uint32) tsBinner.tsBinner32(ts32, timeHgValues) frameSum[iRow,iCol] += ts32.size return timeHgValues,frameSum @staticmethod def countMaskedBins(maskInterval): retval = 0 for x in maskInterval: retval += x[1]-x[0] return retval @staticmethod def populationFromTimeHgValues(timeHgValues,populationMax,stride,threshold): """ Rebin the timgHgValues histogram by combining stride bins. If stride > 1, then bin a second time after shifting by stride/2 Create populationHg, a histogram of the number of photons in the large bins. Also, create (and then sort) a list cosmicTimeList of the start of bins (in original time units) of overpopulated bins that have more than threshold number of photons. return a dictionary containing populationHg and cosmicTimeList """ popRange = (-0.5,populationMax-0.5) if stride==1: populationHg = np.histogram(\ timeHgValues, populationMax, range=popRange) cosmicTimeList = np.where(timeHgValues > threshold)[0] binContents = np.extract(timeHgValues > threshold, timeHgValues) else: # rebin the timeHgValues before counting the populations length = timeHgValues.size remainder = length%stride if remainder == 0: end = length else: end = -remainder timeHgValuesTrimmed = timeHgValues[0:end] timeHgValuesRebinned0 = np.reshape( timeHgValuesTrimmed, [length/stride, stride]).sum(axis=1) populationHg0 = np.histogram( timeHgValuesRebinned0, populationMax, range=popRange) cosmicTimeList0 = stride*np.where(\ timeHgValuesRebinned0 > threshold)[0] binContents0 = np.extract(timeHgValuesRebinned0 > threshold, timeHgValuesRebinned0) timeHgValuesRebinned1 = np.reshape( timeHgValuesTrimmed[stride/2:-stride/2], [(length-stride)/stride, stride]).sum(axis=1) populationHg1 = np.histogram(\ timeHgValuesRebinned1, populationMax, range=popRange) cosmicTimeList1 = (stride/2)+stride*np.where(\ timeHgValuesRebinned1 > threshold)[0] binContents1 = np.extract(timeHgValuesRebinned1 > threshold, timeHgValuesRebinned1) populationHg = (populationHg0[0]+populationHg1[0],\ populationHg0[1]) cosmicTimeList = np.concatenate((cosmicTimeList0,cosmicTimeList1)) binContents = np.concatenate((binContents0, binContents1)) args = np.argsort(cosmicTimeList) cosmicTimeList = cosmicTimeList[args] binContents = binContents[args] cosmicTimeList.sort() retval = {} retval['populationHg'] = populationHg retval['cosmicTimeList'] = cosmicTimeList retval['binContents'] = binContents return retval def makeMovies(self,beginTick, endTick, backgroundFrame, accumulate=False): tick0 = np.uint64(beginTick) tick1 = np.uint64(endTick) for iRow in range(cosmic.file.nRow): for iCol in range(cosmic.file.nCol): gtpl = self.getTimedPacketList(iRow,iCol,sec0,1) timestamps = gtpl['timestamps'] timestamps *= cosmic.file.ticksPerSec ts32 = timestamps.astype(np.uint32) for ts in ts32: tindex = ts-t0 try: listOfPixelsToMark[tindex].append((iRow,iCol)) except IndexError: pass for tick in range(t0,t1): frames.append(frameSum) title = makeTitle(tick,t0,t1) titles.append(title) mfn0 = "m-%s-%s-%s-%s-%010d-%010d-i.gif"%(run,sundownDate,obsDate,seq,t0,t1) utils.makeMovie(frames, titles, outName=mfn0, delay=0.1, colormap=mpl.cm.gray, listOfPixelsToMark=listOfPixelsToMark, pixelMarkColor='red') for i in range(len(listOfPixelsToMark)-1): listOfPixelsToMark[i+1].extend(listOfPixelsToMark[i]) mfn1 = "m-%s-%s-%s-%s-%010d-%010d-a.gif"%(run,sundownDate,obsDate,seq,t0,t1) utils.makeMovie(frames, titles, outName=mfn1, delay=0.1, colormap=mpl.cm.gray, listOfPixelsToMark=listOfPixelsToMark, pixelMarkColor='green') def fitDecayTime(self,t0Sec,lengthSec=200,plotFileName='none'): print "hello from fitDecayTime" timedPacketList = self.file.getTimedPacketList( iRow, iCol, sec0, lengthSec) def fitExpon(self, t0, t1): """ Fit an exponential to all photons from time t0 to time t1 t0 and t1 are in ticks, 1e6 ticks per second return a dictionary of: timeStamps,fitParams,chi2 """ firstSec = int(t0/1e6) # in seconds integrationTime = 1+int((t1-t0)/1e6) # in seconds nBins = integrationTime*1e6 # number of microseconds; one bin per microsecond timeHgValues = np.zeros(nBins, dtype=np.int64) print "firstSec=",firstSec," integrationTime=",integrationTime for iRow in range(self.file.nRow): for iCol in range(self.file.nCol): timedPacketList = self.file.getTimedPacketList( iRow, iCol, firstSec=firstSec, integrationTime=integrationTime) timeStamps = timedPacketList['timestamps'] if (len(timeStamps) > 0): # covert the time values to microseconds, and # make it the type np.uint64 # round per Matt S. suggestion 2013-07-09 #ts64 = (timeStamps).astype(np.uint64) ts32round = np.round(timeStamps).astype(np.uint32) tsBinner.tsBinner(ts32round, timeHgValues) temp = 1e6*(timeStamps-firstSec) for i in range(len(timeStamps)): ts32 = ((timeStamps-firstSec)*1e6).astype(np.uint32) # add these timestamps to the histogram timeHgValues remain0 = int(t0%1e6) remain1 = int(t1%1e6) timeHgValues = timeHgValues[remain0:remain1] x = np.arange(len(timeHgValues)) y = timeHgValues xArray = np.arange(0, dtype=np.int64) yArray = np.arange(0, dtype=np.int64) for i in range(len(x)): if y[i] > 2: xArray = np.append(xArray,i) yArray = np.append(yArray,y[i]) ySigma = np.sqrt(yArray) mean = (x*y).sum()/float(y.sum()) bExponGuess = 1/mean aExponGuess = bExponGuess*timeHgValues.sum() cExponGuess = 0 dExponGuess = 0 pExponGuess = [aExponGuess, bExponGuess, cExponGuess, dExponGuess] bGaussGuess = mean avgx2 = (x*x*y).sum()/float(y.sum()) cGaussGuess = np.sqrt(avgx2-bGaussGuess*bGaussGuess) aGaussGuess = (timeHgValues.sum()/(cGaussGuess*np.sqrt(2*np.pi))) pGaussGuess = [aGaussGuess, bGaussGuess, cGaussGuess] xLimit = [bGaussGuess-4*cGaussGuess, bGaussGuess+4*cGaussGuess] retval = {'timeHgValues':timeHgValues, 'pExponFit':pExponGuess, 'pGaussFit':pGaussGuess, 'xLimit':xLimit, 'cGaussGuess':cGaussGuess, 'timeStamps':timeStamps} return retval @staticmethod def intervalTime(intervals): """ return the time (in seconds) masked by the intervals """ time = 0 for interval in intervals: time += interval[1]-interval[0] return time @staticmethod def funcExpon(x, a, b, c, d): retval = a*np.exp(-b*(x-d)) + c retval[x < d] = 0 return retval @staticmethod def funcGauss(x, a, b, c): return a*np.exp(-(x-b)**2/(2.*c**2))
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) > 2: fileNum = str(sys.argv[2]) else: fileNum = "0" # science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile, "r") for line in f: params.append(line) f.close() datadir = params[0].split("=")[1].strip() flatdir = params[1].split("=")[1].strip() wvldir = params[2].split("=")[1].strip() obsfile = params[3].split("=")[1].strip() skyfile = params[4].split("=")[1].strip() flatfile = params[5].split("=")[1].strip() wvlfile = params[6].split("=")[1].strip() objectName = params[9].split("=")[1].strip() # wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121205" # objectName = "crabNight1" if len(params) > 10: xpix = int(params[10].split("=")[1].strip()) ypix = int(params[11].split("=")[1].strip()) apertureRadius = int(params[12].split("=")[1].strip()) startTime = int(params[13].split("=")[1].strip()) intTime = int(params[14].split("=")[1].strip()) obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) obs = ObsFile(obsFileName) # obs.loadWvlCalFile(wvlCalFileName) obs.loadBestWvlCalFile() obs.loadFlatCalFile(flatCalFileName) print "analyzing file %s" % (obsFileName) print "loaded data file and calibrations\n---------------------\n" nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") # wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow) # nWvlBins=len(wvlBinEdges)-1 # print np.shape(obsSpectra) # print nRow # print nCol # print nWvlBins # Apply Hot pixel masking before getting dead time correction # HotPixFile = getTimeMaskFileName(obsFileName) HotPixFile = FileName(obsFile=obs).timeMask() print "making hot pixel file ", HotPixFile if not os.path.exists(HotPixFile): # check if hot pix file already exists hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile) print "Flux file pixel mask saved to %s" % (HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s" % (HotPixFile) # GET RAW PIXEL COUNT IMAGE TO CALCULATE CORRECTION FACTORS print "Making raw cube to get dead time correction" cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=False) cube = np.array(cubeDict["cube"], dtype=np.double) wvlBinEdges = cubeDict["wvlBinEdges"] effIntTime = cubeDict["effIntTime"] print "median effective integration time = ", np.median(effIntTime) nWvlBins = len(wvlBinEdges) - 1 print "cube shape ", np.shape(cube) print "effIntTime shape ", np.shape(effIntTime) # add third dimension to effIntTime for broadcasting effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,)) # put cube into counts/s in each pixel cube /= effIntTime # CALCULATE DEADTIME CORRECTION # NEED TOTAL COUNTS PER SECOND FOR EACH PIXEL TO DO PROPERLY # ASSUMES SAME CORRECTION FACTOR APPLIED FOR EACH WAVELENGTH, MEANING NO WL DEPENDANCE ON DEAD TIME EFFECT DTCorr = np.zeros((np.shape(cube)[0], np.shape(cube)[1]), dtype=float) for f in range(0, np.shape(cube)[2]): print cube[:, :, f] print "-----------------------" DTCorr += cube[:, :, f] print DTCorr print "\n=====================\n" # Correct for 100 us dead time DTCorrNew = DTCorr / (1 - DTCorr * 100e-6) CorrFactors = DTCorrNew / DTCorr # This is what the frames need to be multiplied by to get their true values print "Dead time correction factors = " print CorrFactors # REMAKE CUBE WITH FLAT WEIGHTS AND APPLY DEAD TIME CORRECTION AS WELL print "Making Weighted cube" # load/generate hot pixel mask file # HotPixFile = getTimeMaskFileName(obsFileName) HotPixFile = FileName(obsFile=obs).timeMask() if not os.path.exists(HotPixFile): # check if hot pix file already exists hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile) print "Flux file pixel mask saved to %s" % (HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s" % (HotPixFile) cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=False) # cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=True) cube = np.array(cubeDict["cube"], dtype=np.double) wvlBinEdges = cubeDict["wvlBinEdges"] effIntTime = cubeDict["effIntTime"] print "median effective integration time = ", np.median(effIntTime) nWvlBins = len(wvlBinEdges) - 1 print "cube shape ", np.shape(cube) print "effIntTime shape ", np.shape(effIntTime) # add third dimension to effIntTime for broadcasting effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,)) # put cube into counts/s in each pixel cube /= effIntTime # add third dimension to CorrFactors for broadcasting CorrFactors = np.reshape(CorrFactors, np.shape(CorrFactors) + (1,)) # apply dead time correction factors cube *= CorrFactors # calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins), dtype=float) for n in xrange(nWvlBins): binsize = wvlBinEdges[n + 1] - wvlBinEdges[n] wvls[n] = wvlBinEdges[n] + (binsize / 2.0) print "wvls ", wvls # reshape cube for makeMovie movieCube = np.zeros((nWvlBins, np.shape(cube)[0], np.shape(cube)[1]), dtype=float) for i in xrange(nWvlBins): movieCube[i, :, :] = cube[:, :, i] # show individual frames as they are made to debug # plt.matshow(movieCube[i],vmin = 0, vmax = 100) # plt.show() print "movieCube shape ", np.shape(movieCube) print "wvls shape ", np.shape(wvls) # print cube # print "--------------------------" # print movieCube outdir = "/home/srmeeker/scratch/standards/" np.savez(outdir + "%s_raw_%s.npz" % (objectName, fileNum), stack=movieCube, wvls=wvls) utils.makeMovie( movieCube, frameTitles=wvls, cbar=True, outName=outdir + "%s_raw_%s.gif" % (objectName, fileNum), normMin=0, normMax=50, ) """
bad_time_intervals=self.laserFlashFile.getPixelBadTimes(row,col,reasons=['hot pixel']) for eachComponent in bad_time_intervals.components: #print eachComponent[0] self.axes.axvspan(eachComponent[0][0]*factor,eachComponent[0][1]*factor,facecolor='r',alpha=0.5) self.axes.axvspan(-2,-1,facecolor='r',alpha=0.5,label='hot') #print eachComponent[0][0],eachComponent[0][1] self.axes.legend() self.axes.set_xlabel('time [s/'+str(factor)+']') self.axes.set_ylabel('count rate [photons/s]') if __name__ == '__main__': filename = "/ScienceData/PAL2014/20141020/cal_20141021-052251.h5" laserFlashFile = ObsFile(filename) print 'Loading hotpix file' laserFlashFile.loadHotPixCalFile(FileName(obsFile=laserFlashFile).timeMask(), reasons=['hot pixel','laser not on']) print 'Making image' #result = laserFlashFile.getPixelCountImage(integrationTime= 2,getRawCount=True) #image = result['image'] #image = result['effIntTimes'] image = laserFlashFile.hotPixTimeMask.getEffIntTimeImage() print 'Making plot' plotArray(image,button_press_event=partial(clickCanvas,laserFlashFile))
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) >3: filenum = str('_'+sys.argv[3]) else: filenum = '_0' #science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() wvldir = params[2].split('=')[1].strip() obsfile = params[3].split('=')[1].strip() skyfile = params[4].split('=')[1].strip() flatfile = params[5].split('=')[1].strip() wvlfile = params[6].split('=')[1].strip() objectName = params[9].split('=')[1].strip() if len(params)>10: xpix = params[10].split('=')[1].strip() ypix = params[11].split('=')[1].strip() apertureRadius = params[12].split('=')[1].strip() #flux cal object parameter file params2 = [] param2file = sys.argv[2] f = open(param2file,'r') for line in f: params2.append(line) f.close() fluxdir = params2[7].split('=')[1].strip() fluxfile = params2[8].split('=')[1].strip() fluxCalObject = params2[9].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) print "obsfile = ",obsFileName print "skyfile = ",skyFileName print "wvlcal = ", wvlCalFileName print "flatcal = ", flatCalFileName print "fluxcal = ", fluxCalFileName print "object = ", objectName print "flux cal object = ", fluxCalObject print "\n---------------------\n" obs = ObsFile(obsFileName) obs.loadWvlCalFile(wvlCalFileName) obs.loadFlatCalFile(flatCalFileName) obs.loadFluxCalFile(fluxCalFileName) print "loaded data file and calibrations\n---------------------\n" HotPixFile = getTimeMaskFileName(obsFileName) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFileName,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") wvlBinEdges,obsSpectra,obsIntTime = loadSpectra(obs,nCol,nRow) nWvlBins=len(wvlBinEdges)-1 #divide spectrum by bin widths binWidths = np.empty(nWvlBins) for i in xrange(nWvlBins): binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i] obsSpectra = obsSpectra/binWidths #divide by effective integration time for iRow in xrange(nRow): for iCol in xrange(nCol): obsSpectra[iRow][iCol] = obsSpectra[iRow][iCol] / obsIntTime[iRow][iCol] #print np.shape(obsSpectra) #print nRow #print nCol #print nWvlBins medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins) print "target spectrum loaded\n---------------------\n" if skyfile != "None": sky = ObsFile(skyFileName) sky.loadWvlCalFile(wvlCalFileName) sky.loadFlatCalFile(flatCalFileName) sky.loadFluxCalFile(fluxCalFileName) skyTime = sky.getFromHeader("exptime") skyHotPixFile = getTimeMaskFileName(skyFileName) if not os.path.exists(skyHotPixFile): hp.findHotPixels(skyFileName,skyHotPixFile) print "Flux file pixel mask saved to %s"%(skyHotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(skyHotPixFile) wvlBinEdges,skySpectra,skyIntTime = loadSpectra(sky,nCol,nRow) skySpectra = skySpectra/binWidths #divide by effective integration time for iRow in xrange(nRow): for iCol in xrange(nCol): skySpectra[iRow][iCol] = skySpectra[iRow][iCol] / skyIntTime[iRow][iCol] skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins) #skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time print "sky spectrum loaded\n---------------------\n" else: #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny skySpectrum = medianObsSpectrum print "sky spectrum estimated as median of target file spectrum\n---------------------\n" #subtract sky spectrum from every pixel allSkySpectrum = obsSpectra-skySpectrum #set any negative values to 0 after sky subtraction allSkySpectrum[allSkySpectrum<0]=0 #take median of remaining sky subtracted spectra to get median object spectrum finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins) #load std spectrum for comparison realSpectra = loadStd(objectName,wvlBinEdges) print "real std spectrum loaded for reference\n---------------------\n" #create plots plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots" plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum) fullFluxPlotFileName = os.path.join(plotDir,plotFileName) pp = PdfPages(fullFluxPlotFileName) matplotlib.rcParams['font.size']=6 #calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins),dtype=float) for n in xrange(nWvlBins): binsize=wvlBinEdges[n+1]-wvlBinEdges[n] wvls[n] = (wvlBinEdges[n]+(binsize/2.0)) plt.figure() ax1 = plt.subplot(231) ax1.set_title('ARCONS median flat/flux cal\'d obs in counts') plt.plot(wvls,medianObsSpectrum) #plt.show() ax2 = plt.subplot(232) ax2.set_title('ARCONS median flat/flux cal\'d sky in counts') plt.plot(wvls,skySpectrum) #plt.show() ax5 = plt.subplot(233) ax5.set_title('Sensitivity Spectrum') plt.plot(wvls,obs.fluxWeights) ax3 = plt.subplot(234) ax3.set_title('MKID data minus sky in counts') plt.plot(wvls,finalSpectrum/max(finalSpectrum)) ax4 = plt.subplot(235) ax4.set_title('Rebinned Std Spectrum of %s'%(objectName)) plt.plot(wvls,realSpectra) #ax7 = plt.subplot(337) #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std') #plt.plot(wvls,fluxFactors*subtractedSpectra) pp.savefig() pp.close() #del obs #del sky print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName) txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt" txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum) fullFluxTxtFileName = os.path.join(txtDir,txtFileName) outarr = np.empty((len(medianObsSpectrum),2),dtype=float) outarr[:,0]=wvls outarr[:,1]=medianObsSpectrum #save sensitivity spectrum to file np.savetxt(fullFluxTxtFileName, outarr) print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
obsFn = FileName(run=run,date=sunsetDate,tstamp=centroidTimestamp).obs() wfn = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln() ffn = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).flatSoln() ffn = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5' # Create ObsFile instance ob = ObsFile(obsFn) # Load wavelength and flat cal solutions ob.loadWvlCalFile(wfn) ob.loadFlatCalFile(ffn) ob.setWvlCutoffs(3000,8000) # Load/generate hot pixel mask file index1 = obsFn.find('_') index2 = obsFn.find('-') hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:] if not os.path.exists(hotPixFn): hp.findHotPixels(obsFn,hotPixFn) print "Flux file pixel mask saved to %s"%(hotPixFn) ob.loadHotPixCalFile(hotPixFn,switchOnMask=False) print "Hot pixel mask loaded %s"%(hotPixFn) centroidCalc(ob, centroid_RA, centroid_DEC, guessTime=300, integrationTime=30, secondMaxCountsForDisplay=500)
run = 'PAL2014' date = '20140924' tstamp = '20140925-112538' obsFN = FileName(run=run,date=date,tstamp=tstamp) obsPath = obsFN.obs() flatPath = FileName(run=run,date=date).flatSoln() hotPath = obsFN.timeMask() centroidPath = obsFN.centroidList() fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln() obs = ObsFile(obsPath) if not os.path.exists(hotPath): hp.findHotPixels(obsPath,hotPath) obs.loadHotPixCalFile(hotPath,switchOnMask=False) obs.loadBestWvlCalFile() obs.loadFlatCalFile(flatPath) obs.loadFluxCalFile(fluxPath) obs.loadCentroidListFile(centroidPath) obs.setWvlCutoffs(3000,11000) centroidRa = '03:37:43.826' centroidDec = '14:15:14.828' haOffset = 150. # imgDict = obs.getPixelCountImage(integrationTime=60,scaleByEffInt=True) # plotArray(imgDict['image'])
illumFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).illumSoln() if not os.path.exists(timeMaskFileName): print 'Running hotpix for ',obsFileName hp.findHotPixels(inputFileName=obsFileName,outputFileName=timeMaskFileName) print "Flux file pixel mask saved to %s"%(timeMaskFileName) obs = ObsFile(obsFileName) exptime = obs.getFromHeader('exptime') #obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments()) if wvlCal != '': obs.loadWvlCalFile(wvlFileName) else: obs.loadBestWvlCalFile() obs.loadHotPixCalFile(timeMaskFileName) obs.setWvlCutoffs(lowerWvlCut,upperWvlCut) obs.loadFlatCalFile(flatFileName) #get image before and after flat cal #print 'getting images' rawCubeDict = obs.getSpectralCube(weighted=False) rawCube = np.array(rawCubeDict['cube'],dtype=np.double) effIntTime = rawCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) rawCube *= maxIntTime / effIntTime3d rawImg = np.sum(rawCube,axis=-1) rawImg[rawImg == 0] = np.nan
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False): ''' Test run of obsFile.writePhotonList. fileName can be used to specify the output file name. If not specified, default name/location is used. Now includes test for pixel remapping.... ''' #Details of example obs file to run test on. # run = 'PAL2012' # date = '20121207' # tstamp = '20121208-074649' # calTstamp='20121208-070505' # fluxTstamp='20121208-133002' run = 'PAL2013' date = '20131209' tstamp = '20131209-115553' centroidTstamp = tstamp calTstamp='20131209-132225' fluxTstamp='ones' flatTstamp='20131209' if doPixRemap==True: pixRemapFileName = FileName(run=run).pixRemap() else: pixRemapFileName = None #Load up the obs file obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln()) obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln()) obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln()) HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask() if not os.path.exists(HotPixFile): #check if hot pix file already exists hp.findHotPixels(obsFileName.obs(),HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obsFile.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) #hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8'' centroid_RA = '12:13:56.42' centroid_DEC = '32:56:30.8' CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList() #print "checking centroid file", CentFile #if not os.path.exists(CentFile): #check if cent pix file already exists cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10) # print "Flux file centroid cal saved to %s"%(CentFile) obsFile.loadCentroidListFile(CentFile) print "Centroid File loaded %s"%(CentFile) #Load up associated calibrations #obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments()) #obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask()) #obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList()) #And write out the results.... obsFile.writePhotonList(outputFileName,firstSec,integrationTime, pixRemapFileName=pixRemapFileName) #Read the results back in.... #photFile = photList.PhotFile(outputFilename) print "Wrote photon list file: ", outputFileName
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) >3: filenum = str('_'+sys.argv[3]) else: filenum = '_0' #science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() wvldir = params[2].split('=')[1].strip() obsfile = params[3].split('=')[1].strip() skyfile = params[4].split('=')[1].strip() flatfile = params[5].split('=')[1].strip() wvlfile = params[6].split('=')[1].strip() objectName = params[9].split('=')[1].strip() if len(params)>10: xpix = int(params[10].split('=')[1].strip()) ypix = int(params[11].split('=')[1].strip()) apertureRadius = int(params[12].split('=')[1].strip()) #flux cal object parameter file params2 = [] param2file = sys.argv[2] f = open(param2file,'r') for line in f: params2.append(line) f.close() fluxdir = params2[7].split('=')[1].strip() fluxfile = params2[8].split('=')[1].strip() fluxCalObject = params2[9].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) print "obsfile = ",obsFileName print "skyfile = ",skyFileName print "wvlcal = ", wvlCalFileName print "flatcal = ", flatCalFileName print "fluxcal = ", fluxCalFileName print "object = ", objectName print "flux cal object = ", fluxCalObject print "\n---------------------\n" obs = ObsFile(obsFileName) obs.loadWvlCalFile(wvlCalFileName) obs.loadFlatCalFile(flatCalFileName) obs.loadFluxCalFile(fluxCalFileName) print "loaded data file and calibrations\n---------------------\n" nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow) #nWvlBins=len(wvlBinEdges)-1 #print np.shape(obsSpectra) #print nRow #print nCol #print nWvlBins """ medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins) print "target spectrum loaded\n---------------------\n" if skyfile != "None": sky = ObsFile(skyFileName) sky.loadWvlCalFile(wvlCalFileName) sky.loadFlatCalFile(flatCalFileName) sky.loadFluxCalFile(fluxCalFileName) skyTime = sky.getFromHeader("exptime") wvlBinEdges,skySpectra = loadSpectra(sky,nCol,nRow) skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins) skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time print "sky spectrum loaded\n---------------------\n" else: #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny skySpectrum = calculateMedian(obsSpectra, nCol, nRow, nWvlBins) print "sky spectrum estimated as median of target file spectrum\n---------------------\n" #subtract sky spectrum from every pixel allSkySpectrum = obsSpectra-skySpectrum #set any negative values to 0 after sky subtraction allSkySpectrum[allSkySpectrum<0]=0 #take median of remaining sky subtracted spectra to get median object spectrum finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins) """ #load/generate hot pixel mask file HotPixFile = getTimeMaskFileName(obsFileName) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFileName,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) print "Making spectrum with Aperture Spectrum in ObsFile" #use Aperture Spectrum from obsfile medianObsSpectrum, wvlBinEdges = obs.getApertureSpectrum(pixelCol=ypix,pixelRow=xpix,radius1=apertureRadius, radius2 = apertureRadius*2.0,weighted=True, fluxWeighted=True, lowCut=3000, highCut=7000) nWvlBins=len(wvlBinEdges)-1 #load std spectrum for comparison try: realSpectra = loadStd(objectName,wvlBinEdges) print "real std spectrum loaded for reference\n---------------------\n" stdTitle = "Rebinned Std Spectrum of %s"%(objectName) except KeyError: print "Key Error loading MKIDStd" realSpectra = np.ones(nWvlBins) stdTitle = "No MKIDStd spectrum available for %s"%(objectName) #create plots plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots" plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum) fullFluxPlotFileName = os.path.join(plotDir,plotFileName) pp = PdfPages(fullFluxPlotFileName) matplotlib.rcParams['font.size']=6 #calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins),dtype=float) for n in xrange(nWvlBins): binsize=wvlBinEdges[n+1]-wvlBinEdges[n] wvls[n] = (wvlBinEdges[n]+(binsize/2.0)) plt.figure() ax1 = plt.subplot(221) ax1.set_title('ARCONS median flat/flux cal\'d obs in counts') ax1.set_xlim((4000,11000)) ax1.set_ylim((min(medianObsSpectrum[(wvls>4000) & (wvls<11000)]),max(medianObsSpectrum[(wvls>4000) & (wvls<8000)]))) plt.plot(wvls,medianObsSpectrum) #plt.show() #ax2 = plt.subplot(232) #ax2.set_title('ARCONS median flat/flux cal\'d sky in counts') #plt.plot(wvls,skySpectrum) #plt.show() ax5 = plt.subplot(223) ax5.set_title('Sensitivity Spectrum') ax5.set_xlim((3000,13000)) ax5.set_ylim((0,5)) plt.plot(wvls,obs.fluxWeights) #ax3 = plt.subplot(234) #ax3.set_title('MKID data minus sky in counts') #plt.plot(wvls,finalSpectrum/max(finalSpectrum)) ax4 = plt.subplot(222) ax4.set_title(stdTitle) plt.plot(wvls,realSpectra) #ax7 = plt.subplot(337) #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std') #plt.plot(wvls,fluxFactors*subtractedSpectra) pp.savefig() pp.close() #del obs #del sky print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName) txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt" txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum) fullFluxTxtFileName = os.path.join(txtDir,txtFileName) outarr = np.empty((len(medianObsSpectrum),2),dtype=float) outarr[:,0]=wvls outarr[:,1]=medianObsSpectrum #save sensitivity spectrum to file np.savetxt(fullFluxTxtFileName, outarr) print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
def main(): #open the sky file for hr9087 run = 'PAL2012' date = '20121210' wvlCal = '20121211-052230' obsTimestamp = '20121211-051650' flatCalDate = '20121211' flatCalTstamp = '20121212-074700' obsFN = FileName(run=run,date=date,tstamp=obsTimestamp) obsFileName = obsFN.obs() timeMaskFileName = obsFN.timeMask() wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln() flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln() if not os.path.exists(timeMaskFileName): print 'Running hotpix for ',obsFileName hp.findHotPixels(obsFileName,timeMaskFileName) print "Flux file pixel mask saved to %s"%(timeMaskFileName) obs = ObsFile(obsFileName) obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments()) obs.loadWvlCalFile(wvlFileName) obs.loadFlatCalFile(flatFileName) obs.loadHotPixCalFile(timeMaskFileName) #obs.setWvlCutoffs(4000,8000) #get image before and after flat cal print 'getting images' beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True) rawCubeDict = obs.getSpectralCube(weighted=False) rawCube = np.array(rawCubeDict['cube'],dtype=np.double) effIntTime = rawCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) rawCube *= maxIntTime / effIntTime3d rawCube[np.isnan(rawCube)] = 0 rawCube[rawCube == np.inf] = 0 beforeImg = np.sum(rawCube,axis=-1) print 'finished first cube' flatCubeDict = obs.getSpectralCube(weighted=True) flatCube = np.array(flatCubeDict['cube'],dtype=np.double) effIntTime = flatCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) flatCube *= maxIntTime / effIntTime3d flatCube[np.isnan(flatCube)] = 0 flatCube[flatCube == np.inf] = 0 afterImg = np.sum(flatCube,axis=-1) plotArray(title='before flatcal',image=beforeImg) plotArray(title='after flatcal',image=afterImg) print 'before sdev',np.std(beforeImg[afterImg!=0]) print 'after sdev',np.std(afterImg[afterImg!=0]) np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) >2: fileNum = str(sys.argv[2]) else: fileNum = '0' #science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() wvldir = params[2].split('=')[1].strip() obsfile = params[3].split('=')[1].strip() skyfile = params[4].split('=')[1].strip() flatfile = params[5].split('=')[1].strip() wvlfile = params[6].split('=')[1].strip() objectName = params[9].split('=')[1].strip() wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121206" if len(params)>10: xpix = int(params[10].split('=')[1].strip()) ypix = int(params[11].split('=')[1].strip()) apertureRadius = int(params[12].split('=')[1].strip()) startTime = int(params[13].split('=')[1].strip()) intTime =int(params[14].split('=')[1].strip()) obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) obs = ObsFile(obsFileName) obs.loadWvlCalFile(wvlCalFileName) obs.loadFlatCalFile(flatCalFileName) print "analyzing file %s"%(obsFileName) print "loaded data file and calibrations\n---------------------\n" nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow) #nWvlBins=len(wvlBinEdges)-1 #print np.shape(obsSpectra) #print nRow #print nCol #print nWvlBins #load/generate hot pixel mask file HotPixFile = getTimeMaskFileName(obsFileName) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFileName,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) ####### #EVERYTHING BEFORE HERE IS STANDARD FILE/CALFILE LOADING startWvl = 3000 #stopWvl = 7000 #for V-band stopWvl = 9000 #for R-band print "Making spectral cube" #for pg0220 first sec should be 80 since object is moving around before this #for pg0220A first sec should be 70, integration time is 140 #for landolt 9542 first sec should be 20, int time is -1 cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, wvlStart = startWvl, wvlStop = stopWvl, wvlBinEdges = [startWvl,stopWvl], weighted=False) cube= np.array(cubeDict['cube'], dtype=np.double) wvlBinEdges= cubeDict['wvlBinEdges'] effIntTime= cubeDict['effIntTime'] print "median effective integration time = ", np.median(effIntTime) nWvlBins=len(wvlBinEdges)-1 print "cube shape ", np.shape(cube) print "effIntTime shape ", np.shape(effIntTime) #add third dimension to effIntTime for broadcasting effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) cube /= effIntTime #put cube into counts/s #calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins),dtype=float) for n in xrange(nWvlBins): binsize=wvlBinEdges[n+1]-wvlBinEdges[n] wvls[n] = (wvlBinEdges[n]+(binsize/2.0)) print "wvls ",wvls #reshape cube for makeMovie movieCube = np.zeros((nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float) for i in xrange(nWvlBins): movieCube[i,:,:] = cube[:,:,i] #show individual frames as they are made to debug #plt.matshow(movieCube[i],vmin = 0, vmax = 100) #plt.show() print "movieCube shape ", np.shape(movieCube) print "wvls shape ", np.shape(wvls) #print cube #print "--------------------------" #print movieCube print "adding frames with wvl below ", stopWvl finalframe = np.zeros((1,np.shape(movieCube)[1],np.shape(movieCube)[2])) for f in xrange(len(wvls[wvls<stopWvl])): print wvls[f] finalframe[0]+=movieCube[f] plt.matshow(movieCube[f],vmin=0,vmax = 40) plt.show() movieCube = finalframe np.savez('%s_%s.npz'%(objectName,fileNum),stack=movieCube,wvls=wvls) print "Saved frame to .npz file" plt.matshow(movieCube[0],vmin=0,vmax = 40) plt.show()
# after the beginning of each second. Plot properties of the photons offsets = [23, 100, 1234, 54321] #offsets = [23] beginTime = 0 expTime = 300 pickleFile = open("csb2.pkl","wb") fn = FileName(run, sundownDate, obsDate+"-"+seq) pickle.dump(fn,pickleFile) obsFile = ObsFile(fn.obs()) obsFile.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments()) # Matt's time fix timeMaskFile = fn.timeMask() if os.path.exists(timeMaskFile): obsFile.loadHotPixCalFile(timeMaskfile, switchOnMask=True) if False: iRow0 = 25 iRow1 = iRow0+1 iCol0 = 40 iCol1 = iCol0+1 else: iRow0 = 0 iRow1 = obsFile.nRow iCol0 = 0 iCol1 = obsFile.nCol spt = obsFile.tickDuration masks = {}
print timestampList wfn = wvlCalFilenames[iSeq] ffn = flatCalFilenames[iSeq] sunsetDate = sunsetDates[iSeq] for i,ts in enumerate(timestampList): print 'loading',ts obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs() ob = ObsFile(obsFn) ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments()) index1 = obsFn.find('_') hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:] if not os.path.exists(hotPixFn): hp.findHotPixels(obsFn,hotPixFn) print "Flux file pixel mask saved to %s"%(hotPixFn) ob.loadHotPixCalFile(hotPixFn,switchOnMask=True) ob.loadWvlCalFile(wfn) ob.loadFlatCalFile(ffn) ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff) bad_solution_mask=np.zeros((46,44)) bad_count=0; for y in range(46): for x in range(44): if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff): bad_solution_mask[y][x] = 1 unix = ob.getFromHeader('unixtime') startJD = unix/86400.+2440587.5 nSecInFile = ob.getFromHeader('exptime') #tic = time()
fluxCalFileNames = ['/Scratch/fluxCalSolnFiles/20121206/fluxsol_20121207-124034.h5'] obsFn = FileName(run=run,date=sunsetDates[0],tstamp='20121209-120530').obs() ob = ObsFile(obsFn) print 'Loading wavelength calibration solution: ' + wvlCalFilenames[0] ob.loadWvlCalFile(wvlCalFilenames[0]) print 'Loading flat calibration solution: ' + flatCalFilenames[0] ob.loadFlatCalFile(flatCalFilenames[0]) ob.loadFluxCalFile(fluxCalFileNames[0]) #load/generate hot pixel mask file HotPixFile = getTimeMaskFileName(obsFn) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFn,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) ob.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) frame = ob.getPixelCountImage(firstSec=0,integrationTime=300,weighted=True) #hotPixMask = hotPixels.checkInterval(image=frame, firstSec=0, intTime=300, weighted=True, display=False)['mask'] #summed_array,bin_edges=ob.getApertureSpectrum(pixelCol=14,pixelRow=8,radius=7) ob.plotApertureSpectrum(pixelCol=14,pixelRow=8,radius=7,weighted = True,fluxWeighted=True,lowCut=3000,highCut=9000) ''' h = 6.626068*10**-34 c = 299792458.0 k = 1.3806503*10**-23 T = 10000.0