def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=True): ''' Test run of obsFile.writePhotonList. fileName can be used to specify the output file name. If not specified, default name/location is used. Now includes test for pixel remapping.... ''' #Details of example obs file to run test on. # run = 'PAL2012' # date = '20121207' # tstamp = '20121208-074649' # calTstamp='20121208-070505' # fluxTstamp='20121208-133002' run = 'PAL2012' date = '20121208' tstamp = '20121209-120530' centroidTstamp = '20121209-120530' calTstamp='20121209-131132' fluxTstamp='20121209-020416' flatTstamp='20121209-021036' if doPixRemap==True: pixRemapFileName = FileName(run=run).pixRemap() else: pixRemapFileName = None #Load up the obs file obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) #Load up associated calibrations obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln()) obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln()) obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln()) obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments()) obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask()) obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList()) #And write out the results.... obsFile.writePhotonList(outputFileName,firstSec,integrationTime, pixRemapFileName=pixRemapFileName)
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) >3: filenum = str('_'+sys.argv[3]) else: filenum = '_0' #science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() wvldir = params[2].split('=')[1].strip() obsfile = params[3].split('=')[1].strip() skyfile = params[4].split('=')[1].strip() flatfile = params[5].split('=')[1].strip() wvlfile = params[6].split('=')[1].strip() objectName = params[9].split('=')[1].strip() if len(params)>10: xpix = params[10].split('=')[1].strip() ypix = params[11].split('=')[1].strip() apertureRadius = params[12].split('=')[1].strip() #flux cal object parameter file params2 = [] param2file = sys.argv[2] f = open(param2file,'r') for line in f: params2.append(line) f.close() fluxdir = params2[7].split('=')[1].strip() fluxfile = params2[8].split('=')[1].strip() fluxCalObject = params2[9].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) print "obsfile = ",obsFileName print "skyfile = ",skyFileName print "wvlcal = ", wvlCalFileName print "flatcal = ", flatCalFileName print "fluxcal = ", fluxCalFileName print "object = ", objectName print "flux cal object = ", fluxCalObject print "\n---------------------\n" obs = ObsFile(obsFileName) obs.loadWvlCalFile(wvlCalFileName) obs.loadFlatCalFile(flatCalFileName) obs.loadFluxCalFile(fluxCalFileName) print "loaded data file and calibrations\n---------------------\n" HotPixFile = getTimeMaskFileName(obsFileName) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFileName,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") wvlBinEdges,obsSpectra,obsIntTime = loadSpectra(obs,nCol,nRow) nWvlBins=len(wvlBinEdges)-1 #divide spectrum by bin widths binWidths = np.empty(nWvlBins) for i in xrange(nWvlBins): binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i] obsSpectra = obsSpectra/binWidths #divide by effective integration time for iRow in xrange(nRow): for iCol in xrange(nCol): obsSpectra[iRow][iCol] = obsSpectra[iRow][iCol] / obsIntTime[iRow][iCol] #print np.shape(obsSpectra) #print nRow #print nCol #print nWvlBins medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins) print "target spectrum loaded\n---------------------\n" if skyfile != "None": sky = ObsFile(skyFileName) sky.loadWvlCalFile(wvlCalFileName) sky.loadFlatCalFile(flatCalFileName) sky.loadFluxCalFile(fluxCalFileName) skyTime = sky.getFromHeader("exptime") skyHotPixFile = getTimeMaskFileName(skyFileName) if not os.path.exists(skyHotPixFile): hp.findHotPixels(skyFileName,skyHotPixFile) print "Flux file pixel mask saved to %s"%(skyHotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(skyHotPixFile) wvlBinEdges,skySpectra,skyIntTime = loadSpectra(sky,nCol,nRow) skySpectra = skySpectra/binWidths #divide by effective integration time for iRow in xrange(nRow): for iCol in xrange(nCol): skySpectra[iRow][iCol] = skySpectra[iRow][iCol] / skyIntTime[iRow][iCol] skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins) #skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time print "sky spectrum loaded\n---------------------\n" else: #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny skySpectrum = medianObsSpectrum print "sky spectrum estimated as median of target file spectrum\n---------------------\n" #subtract sky spectrum from every pixel allSkySpectrum = obsSpectra-skySpectrum #set any negative values to 0 after sky subtraction allSkySpectrum[allSkySpectrum<0]=0 #take median of remaining sky subtracted spectra to get median object spectrum finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins) #load std spectrum for comparison realSpectra = loadStd(objectName,wvlBinEdges) print "real std spectrum loaded for reference\n---------------------\n" #create plots plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots" plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum) fullFluxPlotFileName = os.path.join(plotDir,plotFileName) pp = PdfPages(fullFluxPlotFileName) matplotlib.rcParams['font.size']=6 #calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins),dtype=float) for n in xrange(nWvlBins): binsize=wvlBinEdges[n+1]-wvlBinEdges[n] wvls[n] = (wvlBinEdges[n]+(binsize/2.0)) plt.figure() ax1 = plt.subplot(231) ax1.set_title('ARCONS median flat/flux cal\'d obs in counts') plt.plot(wvls,medianObsSpectrum) #plt.show() ax2 = plt.subplot(232) ax2.set_title('ARCONS median flat/flux cal\'d sky in counts') plt.plot(wvls,skySpectrum) #plt.show() ax5 = plt.subplot(233) ax5.set_title('Sensitivity Spectrum') plt.plot(wvls,obs.fluxWeights) ax3 = plt.subplot(234) ax3.set_title('MKID data minus sky in counts') plt.plot(wvls,finalSpectrum/max(finalSpectrum)) ax4 = plt.subplot(235) ax4.set_title('Rebinned Std Spectrum of %s'%(objectName)) plt.plot(wvls,realSpectra) #ax7 = plt.subplot(337) #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std') #plt.plot(wvls,fluxFactors*subtractedSpectra) pp.savefig() pp.close() #del obs #del sky print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName) txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt" txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum) fullFluxTxtFileName = os.path.join(txtDir,txtFileName) outarr = np.empty((len(medianObsSpectrum),2),dtype=float) outarr[:,0]=wvls outarr[:,1]=medianObsSpectrum #save sensitivity spectrum to file np.savetxt(fullFluxTxtFileName, outarr) print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
obsFN = FileName(run=run,date=date,tstamp=tstamp) obsPath = obsFN.obs() flatPath = FileName(run=run,date=date).flatSoln() hotPath = obsFN.timeMask() centroidPath = obsFN.centroidList() fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln() obs = ObsFile(obsPath) if not os.path.exists(hotPath): hp.findHotPixels(obsPath,hotPath) obs.loadHotPixCalFile(hotPath,switchOnMask=False) obs.loadBestWvlCalFile() obs.loadFlatCalFile(flatPath) obs.loadFluxCalFile(fluxPath) obs.loadCentroidListFile(centroidPath) obs.setWvlCutoffs(3000,11000) centroidRa = '03:37:43.826' centroidDec = '14:15:14.828' haOffset = 150. # imgDict = obs.getPixelCountImage(integrationTime=60,scaleByEffInt=True) # plotArray(imgDict['image']) xGuess = 18 #col yGuess = 15 #row #cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath)
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False): ''' Test run of obsFile.writePhotonList. fileName can be used to specify the output file name. If not specified, default name/location is used. Now includes test for pixel remapping.... ''' #Details of example obs file to run test on. # run = 'PAL2012' # date = '20121207' # tstamp = '20121208-074649' # calTstamp='20121208-070505' # fluxTstamp='20121208-133002' run = 'PAL2013' date = '20131209' tstamp = '20131209-115553' centroidTstamp = tstamp calTstamp='20131209-132225' fluxTstamp='ones' flatTstamp='20131209' if doPixRemap==True: pixRemapFileName = FileName(run=run).pixRemap() else: pixRemapFileName = None #Load up the obs file obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln()) obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln()) obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln()) HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask() if not os.path.exists(HotPixFile): #check if hot pix file already exists hp.findHotPixels(obsFileName.obs(),HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obsFile.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) #hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8'' centroid_RA = '12:13:56.42' centroid_DEC = '32:56:30.8' CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList() #print "checking centroid file", CentFile #if not os.path.exists(CentFile): #check if cent pix file already exists cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10) # print "Flux file centroid cal saved to %s"%(CentFile) obsFile.loadCentroidListFile(CentFile) print "Centroid File loaded %s"%(CentFile) #Load up associated calibrations #obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments()) #obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask()) #obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList()) #And write out the results.... obsFile.writePhotonList(outputFileName,firstSec,integrationTime, pixRemapFileName=pixRemapFileName) #Read the results back in.... #photFile = photList.PhotFile(outputFilename) print "Wrote photon list file: ", outputFileName
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) >3: filenum = str('_'+sys.argv[3]) else: filenum = '_0' #science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() wvldir = params[2].split('=')[1].strip() obsfile = params[3].split('=')[1].strip() skyfile = params[4].split('=')[1].strip() flatfile = params[5].split('=')[1].strip() wvlfile = params[6].split('=')[1].strip() objectName = params[9].split('=')[1].strip() if len(params)>10: xpix = int(params[10].split('=')[1].strip()) ypix = int(params[11].split('=')[1].strip()) apertureRadius = int(params[12].split('=')[1].strip()) #flux cal object parameter file params2 = [] param2file = sys.argv[2] f = open(param2file,'r') for line in f: params2.append(line) f.close() fluxdir = params2[7].split('=')[1].strip() fluxfile = params2[8].split('=')[1].strip() fluxCalObject = params2[9].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) print "obsfile = ",obsFileName print "skyfile = ",skyFileName print "wvlcal = ", wvlCalFileName print "flatcal = ", flatCalFileName print "fluxcal = ", fluxCalFileName print "object = ", objectName print "flux cal object = ", fluxCalObject print "\n---------------------\n" obs = ObsFile(obsFileName) obs.loadWvlCalFile(wvlCalFileName) obs.loadFlatCalFile(flatCalFileName) obs.loadFluxCalFile(fluxCalFileName) print "loaded data file and calibrations\n---------------------\n" nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow) #nWvlBins=len(wvlBinEdges)-1 #print np.shape(obsSpectra) #print nRow #print nCol #print nWvlBins """ medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins) print "target spectrum loaded\n---------------------\n" if skyfile != "None": sky = ObsFile(skyFileName) sky.loadWvlCalFile(wvlCalFileName) sky.loadFlatCalFile(flatCalFileName) sky.loadFluxCalFile(fluxCalFileName) skyTime = sky.getFromHeader("exptime") wvlBinEdges,skySpectra = loadSpectra(sky,nCol,nRow) skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins) skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time print "sky spectrum loaded\n---------------------\n" else: #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny skySpectrum = calculateMedian(obsSpectra, nCol, nRow, nWvlBins) print "sky spectrum estimated as median of target file spectrum\n---------------------\n" #subtract sky spectrum from every pixel allSkySpectrum = obsSpectra-skySpectrum #set any negative values to 0 after sky subtraction allSkySpectrum[allSkySpectrum<0]=0 #take median of remaining sky subtracted spectra to get median object spectrum finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins) """ #load/generate hot pixel mask file HotPixFile = getTimeMaskFileName(obsFileName) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFileName,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) print "Making spectrum with Aperture Spectrum in ObsFile" #use Aperture Spectrum from obsfile medianObsSpectrum, wvlBinEdges = obs.getApertureSpectrum(pixelCol=ypix,pixelRow=xpix,radius1=apertureRadius, radius2 = apertureRadius*2.0,weighted=True, fluxWeighted=True, lowCut=3000, highCut=7000) nWvlBins=len(wvlBinEdges)-1 #load std spectrum for comparison try: realSpectra = loadStd(objectName,wvlBinEdges) print "real std spectrum loaded for reference\n---------------------\n" stdTitle = "Rebinned Std Spectrum of %s"%(objectName) except KeyError: print "Key Error loading MKIDStd" realSpectra = np.ones(nWvlBins) stdTitle = "No MKIDStd spectrum available for %s"%(objectName) #create plots plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots" plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum) fullFluxPlotFileName = os.path.join(plotDir,plotFileName) pp = PdfPages(fullFluxPlotFileName) matplotlib.rcParams['font.size']=6 #calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins),dtype=float) for n in xrange(nWvlBins): binsize=wvlBinEdges[n+1]-wvlBinEdges[n] wvls[n] = (wvlBinEdges[n]+(binsize/2.0)) plt.figure() ax1 = plt.subplot(221) ax1.set_title('ARCONS median flat/flux cal\'d obs in counts') ax1.set_xlim((4000,11000)) ax1.set_ylim((min(medianObsSpectrum[(wvls>4000) & (wvls<11000)]),max(medianObsSpectrum[(wvls>4000) & (wvls<8000)]))) plt.plot(wvls,medianObsSpectrum) #plt.show() #ax2 = plt.subplot(232) #ax2.set_title('ARCONS median flat/flux cal\'d sky in counts') #plt.plot(wvls,skySpectrum) #plt.show() ax5 = plt.subplot(223) ax5.set_title('Sensitivity Spectrum') ax5.set_xlim((3000,13000)) ax5.set_ylim((0,5)) plt.plot(wvls,obs.fluxWeights) #ax3 = plt.subplot(234) #ax3.set_title('MKID data minus sky in counts') #plt.plot(wvls,finalSpectrum/max(finalSpectrum)) ax4 = plt.subplot(222) ax4.set_title(stdTitle) plt.plot(wvls,realSpectra) #ax7 = plt.subplot(337) #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std') #plt.plot(wvls,fluxFactors*subtractedSpectra) pp.savefig() pp.close() #del obs #del sky print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName) txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt" txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum) fullFluxTxtFileName = os.path.join(txtDir,txtFileName) outarr = np.empty((len(medianObsSpectrum),2),dtype=float) outarr[:,0]=wvls outarr[:,1]=medianObsSpectrum #save sensitivity spectrum to file np.savetxt(fullFluxTxtFileName, outarr) print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
wvlCalFilenames = [FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln() for sunsetDate,calTimestamp in zip(sunsetDates,calTimestamps)] flatCalFilenames = [FileName(run=run,date=sunsetDate,tstamp=calTimestamp).flatSoln() for sunsetDate,calTimestamp in zip(sunsetDates,calTimestamps)] #wvlCalFilenames[0] = '/Scratch/waveCalSolnFiles/20121210/calsol_20121211-074031.h5' #wvlCalFilenames[1] = '/home/danica/optimusP/testing/forMatt/calsol_20121211-044853.h5' flatCalFilenames[0] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5' flatCalFilenames[1] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5' fluxCalFileNames = ['/Scratch/fluxCalSolnFiles/20121206/fluxsol_20121207-124034.h5'] obsFn = FileName(run=run,date=sunsetDates[0],tstamp='20121209-120530').obs() ob = ObsFile(obsFn) print 'Loading wavelength calibration solution: ' + wvlCalFilenames[0] ob.loadWvlCalFile(wvlCalFilenames[0]) print 'Loading flat calibration solution: ' + flatCalFilenames[0] ob.loadFlatCalFile(flatCalFilenames[0]) ob.loadFluxCalFile(fluxCalFileNames[0]) #load/generate hot pixel mask file HotPixFile = getTimeMaskFileName(obsFn) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFn,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) ob.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) frame = ob.getPixelCountImage(firstSec=0,integrationTime=300,weighted=True) #hotPixMask = hotPixels.checkInterval(image=frame, firstSec=0, intTime=300, weighted=True, display=False)['mask'] #summed_array,bin_edges=ob.getApertureSpectrum(pixelCol=14,pixelRow=8,radius=7) ob.plotApertureSpectrum(pixelCol=14,pixelRow=8,radius=7,weighted = True,fluxWeighted=True,lowCut=3000,highCut=9000)