def main(): #load the current PAL2012 beammap from an obs file from that run, #a sky file for hr9087 run = 'PAL2012' date = '20121210' obsTimestamp = '20121211-051650' obsFN = FileName(run=run,date=date,tstamp=obsTimestamp) obsFileName = obsFN.obs() obs = ObsFile(obsFileName) #Load an existing definition of which pixels to remap where pixMap = remapPixels.PixelMap(obsFN.pixRemap()) pixMapSourceList,pixMapDestList = pixMap.getRemappedPix() #create a new h5 file and copy the existing beammap node hierarchy into it newBeammapPath = 'beamimage_PAL2012_corrected.h5' newBeammapFile = tables.openFile(newBeammapPath, mode='w') newBeammapFile.copyNode(obs.file.root.beammap, newparent=newBeammapFile.root, newname='beammap', recursive=True) newBeammap = newBeammapFile.root.beammap.beamimage #for each array in the h5, move the pixels according to pixMap newBeammap[:,:] = pixMap.remapArray(obs.beamImage) newBeammapFile.root.beammap.resfreq[:,:] = pixMap.remapArray(newBeammapFile.root.beammap.resfreq.read()) newBeammapFile.root.beammap.atten[:,:] = pixMap.remapArray(newBeammapFile.root.beammap.atten.read()) newBeammapFile.flush() newBeammapFile.close() obs.file.close()
def testLoadBeammap(): ''' Test if a remapped beammap is actually remapped ''' #open an obs file from PAL2012,the sky file for hr9087 #we'll use the embedded beammap file, which has some misplaced pixels run = 'PAL2012' date = '20121210' obsTimestamp = '20121211-051650' obsFN = FileName(run=run,date=date,tstamp=obsTimestamp) obsFileName = obsFN.obs() obs = ObsFile(obsFileName) beammapFileName = obsFN.beammap() #load the PixelMap for PAL2012 to know which pixels should be remapped pixMap = remapPixels.PixelMap(obsFN.pixRemap()) pixMapSourceList,pixMapDestList = pixMap.getRemappedPix() #load the corrected beammap into the obs obs.loadBeammapFile(beammapFileName) #check that each pixel that should be moved is moved #by comparing the embedded beammap and the loaded corrected one for source,dest in zip(pixMapSourceList,pixMapDestList): assert obs.beamImage[dest] == obs.file.root.beammap.beamimage[source] obs.file.close()
def makemovie1(self): run = self.s['run'] sundownDate = self.s['sundownDate'] obsDate = self.s['obsDate'] stride = int(self.s['stride']) seq5 = self.s['seq5'].split() for seq in seq5: inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb") cosmicTimeList = pickle.load(inFile) binContents = pickle.load(inFile) cfn = "cosmicMask-%s.h5"%seq intervals = ObsFile.readCosmicIntervalFromFile(cfn) for interval in intervals: print "interval=",interval fn = FileName(run, sundownDate,obsDate+"-"+seq) obsFile = ObsFile(fn.obs()) obsFile.loadTimeAdjustmentFile(fn.timeAdjustments()) i0=interval[0] i1=interval[1] intervalTime = i1-i0 dt = intervalTime/2 beginTime = max(0,i0-0.000200) endTime = beginTime + 0.001 integrationTime = endTime-beginTime nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1)) timeHgValues = np.zeros(nBins, dtype=np.int64) ymax = sys.float_info.max/100.0 for iRow in range(obsFile.nRow): for iCol in range(obsFile.nCol): gtpl = obsFile.getTimedPacketList(iRow,iCol, beginTime,integrationTime) ts = (gtpl['timestamps'] - beginTime)*obsFile.ticksPerSec ts64 = np.round(ts).astype(np.uint64) tsBinner.tsBinner(ts64, timeHgValues) plt.clf() plt.plot(timeHgValues, label="data") x0 = (i0-beginTime)*obsFile.ticksPerSec x1 = (i1-beginTime)*obsFile.ticksPerSec plt.fill_between((x0,x1),(0,0), (ymax,ymax), alpha=0.2, color='red') plt.yscale("symlog",linthreshy=0.9) plt.xlim(0,1000) plt.ylim(-0.1,300) tick0 = int(np.round(i0*obsFile.ticksPerSec)) plotfn = "cp-%05d-%s-%s-%s-%09d"%(timeHgValues.sum(),run,obsDate,seq,tick0) plt.title(plotfn) plt.legend() plt.savefig(plotfn+".png") plt.xlabel("nSigma=%d stride=%d threshold=%d"%(int(self.s['nSigma']),int(self.s['stride']),int(self.s['threshold']))) print "plotfn=",plotfn os.system("convert -delay 0 `ls -r cp*png` cp.gif")
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=True): ''' Test run of obsFile.writePhotonList. fileName can be used to specify the output file name. If not specified, default name/location is used. Now includes test for pixel remapping.... ''' #Details of example obs file to run test on. # run = 'PAL2012' # date = '20121207' # tstamp = '20121208-074649' # calTstamp='20121208-070505' # fluxTstamp='20121208-133002' run = 'PAL2012' date = '20121208' tstamp = '20121209-120530' centroidTstamp = '20121209-120530' calTstamp='20121209-131132' fluxTstamp='20121209-020416' flatTstamp='20121209-021036' if doPixRemap==True: pixRemapFileName = FileName(run=run).pixRemap() else: pixRemapFileName = None #Load up the obs file obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) #Load up associated calibrations obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln()) obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln()) obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln()) obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments()) obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask()) obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList()) #And write out the results.... obsFile.writePhotonList(outputFileName,firstSec,integrationTime, pixRemapFileName=pixRemapFileName)
sigList = [] run = 'PAL2012' sundownDate = '20121211' obsDate = '20121212' for seq in seq5: inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb") cosmicTimeList = pickle.load(inFile) binContents = pickle.load(inFile) cfn = "cosmicMax-%s.h5"%seq intervals = ObsFile.readCosmicIntervalFromFile(cfn) for interval in intervals: print "interval=",interval fn = FileName(run, sundownDate,obsDate+"-"+seq) obsFile = ObsFile(fn.obs()) obsFile.loadTimeAdjustmentFile(fn.timeAdjustments()) i0=interval[0] i1=interval[1] intervalTime = i1-i0 dt = intervalTime/2 beginTime = max(0,i0-0.000200) endTime = beginTime + 0.001 integrationTime = endTime-beginTime nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1)) timeHgValues = np.zeros(nBins, dtype=np.int64) ymax = sys.float_info.max/100.0 for iRow in range(obsFile.nRow): for iCol in range(obsFile.nCol): gtpl = obsFile.getTimedPacketList(iRow,iCol, beginTime,integrationTime)
def __init__(self,paramFile,plots=False,verbose=False): """ Opens flux file, prepares standard spectrum, and calculates flux factors for the file. Method is provided in param file. If 'relative' is selected, an obs file with standard star defocused over the entire array is expected, with accompanying sky file to do sky subtraction. If any other method is provided, 'absolute' will be done by default, wherein a point source is assumed to be present. The obs file is then broken into spectral frames with photometry (psf or aper) performed on each frame to generate the ARCONS observed spectrum. """ self.verbose=verbose self.plots = plots self.params = readDict() self.params.read_from_file(paramFile) run = self.params['run'] sunsetDate = self.params['fluxSunsetLocalDate'] self.fluxTstamp = self.params['fluxTimestamp'] skyTstamp = self.params['skyTimestamp'] wvlSunsetDate = self.params['wvlCalSunsetLocalDate'] wvlTimestamp = self.params['wvlCalTimestamp'] flatCalFileName = self.params['flatCalFileName'] needTimeAdjust = self.params['needTimeAdjust'] self.deadtime = float(self.params['deadtime']) #from firmware pulse detection self.timeSpacingCut = self.params['timeSpacingCut'] bLoadBeammap = self.params.get('bLoadBeammap',False) self.method = self.params['method'] self.objectName = self.params['object'] self.r = float(self.params['energyResolution']) self.photometry = self.params['photometry'] self.centroidRow = self.params['centroidRow'] self.centroidCol = self.params['centroidCol'] self.aperture = self.params['apertureRad'] self.annulusInner = self.params['annulusInner'] self.annulusOuter = self.params['annulusOuter'] self.collectingArea = self.params['collectingArea'] self.startTime = self.params['startTime'] self.intTime = self.params['integrationTime'] fluxFN = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp) self.fluxFileName = fluxFN.obs() self.fluxFile = ObsFile(self.fluxFileName) if self.plots: self.plotSavePath = os.environ['MKID_PROC_PATH']+os.sep+'fluxCalSolnFiles'+os.sep+run+os.sep+sunsetDate+os.sep+'plots'+os.sep if not os.path.exists(self.plotSavePath): os.mkdir(self.plotSavePath) if self.verbose: print "Created directory %s"%self.plotSavePath obsFNs = [fluxFN] self.obsList = [self.fluxFile] if self.startTime in ['',None]: self.startTime=0 if self.intTime in ['',None]: self.intTime=-1 if self.method=="relative": try: print "performing Relative Flux Calibration" skyFN = FileName(run=run,date=sunsetDate,tstamp=skyTstamp) self.skyFileName = skyFN.obs() self.skyFile = ObsFile(self.skyFileName) obsFNs.append(skyFN) self.obsList.append(self.skyFile) except: print "For relative flux calibration a sky file must be provided in param file" self.__del__() else: self.method='absolute' print "performing Absolute Flux Calibration" if self.photometry not in ['aperture','PSF']: self.photometry='PSF' #default to PSF fitting if no valid photometry selected timeMaskFileNames = [fn.timeMask() for fn in obsFNs] timeAdjustFileName = FileName(run=run).timeAdjustments() #make filename for output fluxCalSoln file self.fluxCalFileName = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp).fluxSoln() print "Creating flux cal: %s"%self.fluxCalFileName if wvlSunsetDate != '': wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln() if flatCalFileName =='': flatCalFileName=FileName(obsFile=self.fluxFile).flatSoln() #load cal files for flux file and, if necessary, sky file for iObs,obs in enumerate(self.obsList): if bLoadBeammap: print 'loading beammap',os.environ['MKID_BEAMMAP_PATH'] obs.loadBeammapFile(os.environ['MKID_BEAMMAP_PATH']) if wvlSunsetDate != '': obs.loadWvlCalFile(wvlCalFileName) else: obs.loadBestWvlCalFile() obs.loadFlatCalFile(flatCalFileName) obs.setWvlCutoffs(-1,-1) if needTimeAdjust: obs.loadTimeAdjustmentFile(timeAdjustFileName) timeMaskFileName = timeMaskFileNames[iObs] print timeMaskFileName if not os.path.exists(timeMaskFileName): print 'Running hotpix for ',obs hp.findHotPixels(obsFile=obs,outputFileName=timeMaskFileName,fwhm=np.inf,useLocalStdDev=True) print "Flux cal/sky file pixel mask saved to %s"%(timeMaskFileName) obs.loadHotPixCalFile(timeMaskFileName) if self.verbose: print "Loaded hot pixel file %s"%timeMaskFileName #get flat cal binning information since flux cal will need to match it self.wvlBinEdges = self.fluxFile.flatCalFile.root.flatcal.wavelengthBins.read() self.nWvlBins = self.fluxFile.flatWeights.shape[2] self.binWidths = np.empty((self.nWvlBins),dtype=float) self.binCenters = np.empty((self.nWvlBins),dtype=float) for i in xrange(self.nWvlBins): self.binWidths[i] = self.wvlBinEdges[i+1]-self.wvlBinEdges[i] self.binCenters[i] = (self.wvlBinEdges[i]+(self.binWidths[i]/2.0)) if self.method=='relative': print "Extracting ARCONS flux and sky spectra" self.loadRelativeSpectrum() print "Flux Spectrum loaded" self.loadSkySpectrum() print "Sky Spectrum loaded" elif self.method=='absolute': print "Extracting ARCONS point source spectrum" self.loadAbsoluteSpectrum() print "Loading standard spectrum" try: self.loadStdSpectrum(self.objectName) except KeyError: print "Invalid spectrum object name" self.__del__() sys.exit() print "Generating sensitivity curve" self.calculateFactors() print "Sensitivity Curve calculated" print "Writing fluxCal to file %s"%self.fluxCalFileName self.writeFactors(self.fluxCalFileName) if self.plots: self.makePlots() print "Done"
from util.ObsFile import ObsFile import os import hotpix.hotPixels as hp from util.FileName import FileName from util.popup import plotArray import astrometry.CentroidCalc as cc from photonlist.photlist import writePhotonList from headers.ArconsHeaders import PulsarPhotonList if __name__=='__main__': run = 'PAL2014' date = '20140924' tstamp = '20140925-112538' obsFN = FileName(run=run,date=date,tstamp=tstamp) obsPath = obsFN.obs() flatPath = FileName(run=run,date=date).flatSoln() hotPath = obsFN.timeMask() centroidPath = obsFN.centroidList() fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln() obs = ObsFile(obsPath) if not os.path.exists(hotPath): hp.findHotPixels(obsPath,hotPath) obs.loadHotPixCalFile(hotPath,switchOnMask=False) obs.loadBestWvlCalFile() obs.loadFlatCalFile(flatPath) obs.loadFluxCalFile(fluxPath) obs.loadCentroidListFile(centroidPath) obs.setWvlCutoffs(3000,11000)
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False): ''' Test run of obsFile.writePhotonList. fileName can be used to specify the output file name. If not specified, default name/location is used. Now includes test for pixel remapping.... ''' #Details of example obs file to run test on. # run = 'PAL2012' # date = '20121207' # tstamp = '20121208-074649' # calTstamp='20121208-070505' # fluxTstamp='20121208-133002' run = 'PAL2013' date = '20131209' tstamp = '20131209-115553' centroidTstamp = tstamp calTstamp='20131209-132225' fluxTstamp='ones' flatTstamp='20131209' if doPixRemap==True: pixRemapFileName = FileName(run=run).pixRemap() else: pixRemapFileName = None #Load up the obs file obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln()) obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln()) obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln()) HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask() if not os.path.exists(HotPixFile): #check if hot pix file already exists hp.findHotPixels(obsFileName.obs(),HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obsFile.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) #hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8'' centroid_RA = '12:13:56.42' centroid_DEC = '32:56:30.8' CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList() #print "checking centroid file", CentFile #if not os.path.exists(CentFile): #check if cent pix file already exists cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10) # print "Flux file centroid cal saved to %s"%(CentFile) obsFile.loadCentroidListFile(CentFile) print "Centroid File loaded %s"%(CentFile) #Load up associated calibrations #obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments()) #obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask()) #obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList()) #And write out the results.... obsFile.writePhotonList(outputFileName,firstSec,integrationTime, pixRemapFileName=pixRemapFileName) #Read the results back in.... #photFile = photList.PhotFile(outputFilename) print "Wrote photon list file: ", outputFileName
def main(): #open the sky file for hr9087 run = 'PAL2012' date = '20121210' wvlCal = '20121211-052230' obsTimestamp = '20121211-051650' flatCalDate = '20121211' flatCalTstamp = '20121212-074700' obsFN = FileName(run=run,date=date,tstamp=obsTimestamp) obsFileName = obsFN.obs() timeMaskFileName = obsFN.timeMask() wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln() flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln() if not os.path.exists(timeMaskFileName): print 'Running hotpix for ',obsFileName hp.findHotPixels(obsFileName,timeMaskFileName) print "Flux file pixel mask saved to %s"%(timeMaskFileName) obs = ObsFile(obsFileName) obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments()) obs.loadWvlCalFile(wvlFileName) obs.loadFlatCalFile(flatFileName) obs.loadHotPixCalFile(timeMaskFileName) #obs.setWvlCutoffs(4000,8000) #get image before and after flat cal print 'getting images' beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True) rawCubeDict = obs.getSpectralCube(weighted=False) rawCube = np.array(rawCubeDict['cube'],dtype=np.double) effIntTime = rawCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) rawCube *= maxIntTime / effIntTime3d rawCube[np.isnan(rawCube)] = 0 rawCube[rawCube == np.inf] = 0 beforeImg = np.sum(rawCube,axis=-1) print 'finished first cube' flatCubeDict = obs.getSpectralCube(weighted=True) flatCube = np.array(flatCubeDict['cube'],dtype=np.double) effIntTime = flatCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) flatCube *= maxIntTime / effIntTime3d flatCube[np.isnan(flatCube)] = 0 flatCube[flatCube == np.inf] = 0 afterImg = np.sum(flatCube,axis=-1) plotArray(title='before flatcal',image=beforeImg) plotArray(title='after flatcal',image=afterImg) print 'before sdev',np.std(beforeImg[afterImg!=0]) print 'after sdev',np.std(afterImg[afterImg!=0]) np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
import hotpix.hotPixels as hp from photonlist import photlist import matplotlib.pylab as plt run = 'PAL2013' date = '20131209' tstamps = ['20131209-115553','20131209-120100', '20131209-121634', '20131209-122152'] for tstamp in tstamps: centroidTstamp = tstamp calTstamp='20131209-132225' fluxTstamp='ones' flatTstamp='20131209' obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) frames = [] print '-----------------------' print tstamp for i in xrange(30): print "making frame ", i frame = obsFile.getFrame(10*i,10) #plt.matshow(frame,vmin=10000,vmax=40000,cmap=plt.cm.gray) #plt.show() frames.append(frame) makeMovie(frames,frameTitles=np.arange(30)*10,cbar=True,outName='%s.gif'%(tstamp), normMin=400, normMax=4000) CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList() obsFile.loadCentroidListFile(CentFile)
#obsTimestamp ='20140925-124603' #early twilight #obsTimestamp ='20140925-124706' #early twilight #obsTimestamp ='20140925-124809' #early twilight #obsTimestamp ='20140925-124912' #early twilight #obsTimestamp ='20140925-125015' #late twilight #obsTimestamp ='20140925-125118' #late twilight #obsTimestamp ='20140925-125221' #late twilight #obsTimestamp ='20140925-125324' #late twilight #obsTimestamp ='20140925-125427' #late twilight #date = '20140925' #obsTimestamp ='20140926-104528' #psr J0337 obsFN = FileName(run=run,date=date,tstamp=obsTimestamp) obsFileName = obsFN.obs() print obsFileName timeMaskFileName = obsFN.timeMask() if wvlCal != '': wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln() flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln() illumFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).illumSoln() if not os.path.exists(timeMaskFileName): print 'Running hotpix for ',obsFileName hp.findHotPixels(inputFileName=obsFileName,outputFileName=timeMaskFileName) print "Flux file pixel mask saved to %s"%(timeMaskFileName) obs = ObsFile(obsFileName) exptime = obs.getFromHeader('exptime')