def loadHotMask(self):
#self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5')
self.hotPixelFilename = str(FileName(obsFile = self.ob).timeMask())
if not os.path.exists(self.hotPixelFilename):
hp.findHotPixels(obsFile=self.ob,outputFileName=self.hotPixelFilename)
print "Hot pixel mask saved to %s"%(self.hotPixelFilename)
self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True)
def generateObsObjectList(obsFNs,wvlLowerLimit=3000, wvlUpperLimit=13000,beammapFileName = None,loadHotPix=True,loadWvlCal=True,loadFlatCal=True,loadSpectralCal=True):
obsFiles = []
for obsFN in obsFNs:
if type(obsFN) == type(''): #Full path to obs file
obs = ObsFile(obsFN)
else: #FileName object
obs = ObsFile(obsFN.obs())
if beammapFileName is not None and os.path.isfile(beammapFileName):
obs.loadBeammapFile(beammapFileName)
obs.setWvlCutoffs(wvlLowerLimit=wvlLowerLimit, wvlUpperLimit=wvlUpperLimit)
if loadHotPix:
if not os.path.isfile(FileName(obsFile=obs).timeMask()):
print "generating hp file ", FileName(obsFile=obs).timeMask()
hp.findHotPixels(obsFile=obs,outputFileName=FileName(obsFile=obs).timeMask())
obs.loadHotPixCalFile(FileName(obsFile=obs).timeMask(),reasons=['hot pixel','dead pixel'])
if loadWvlCal:
obs.loadBestWvlCalFile()
if loadFlatCal:
#obs.loadFlatCalFile(FileName(obsFile=obs).flatSoln())
obs.loadFlatCalFile('/Scratch/flatCalSolnFiles/flatsol_1s.h5')
if loadSpectralCal:
pass
obsFiles.append(obs)
return obsFiles
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess,hotPath,flatPath):
obs = ObsFile(obsPath)
print obsPath,obs.getFromHeader('exptime'),obs
if not os.path.exists(hotPath):
hp.findHotPixels(obsFile=obs,outputFileName=hotPath)
obs.loadHotPixCalFile(hotPath,switchOnMask=False)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatPath)
obs.setWvlCutoffs(3000,8000)
cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath)
print 'done centroid',centroidPath
del obs
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) >3:
filenum = str('_'+sys.argv[3])
else:
filenum = '_0'
#science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
wvldir = params[2].split('=')[1].strip()
obsfile = params[3].split('=')[1].strip()
skyfile = params[4].split('=')[1].strip()
flatfile = params[5].split('=')[1].strip()
wvlfile = params[6].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
if len(params)>10:
xpix = params[10].split('=')[1].strip()
ypix = params[11].split('=')[1].strip()
apertureRadius = params[12].split('=')[1].strip()
#flux cal object parameter file
params2 = []
param2file = sys.argv[2]
f = open(param2file,'r')
for line in f:
params2.append(line)
f.close()
fluxdir = params2[7].split('=')[1].strip()
fluxfile = params2[8].split('=')[1].strip()
fluxCalObject = params2[9].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
print "obsfile = ",obsFileName
print "skyfile = ",skyFileName
print "wvlcal = ", wvlCalFileName
print "flatcal = ", flatCalFileName
print "fluxcal = ", fluxCalFileName
print "object = ", objectName
print "flux cal object = ", fluxCalObject
print "\n---------------------\n"
obs = ObsFile(obsFileName)
obs.loadWvlCalFile(wvlCalFileName)
obs.loadFlatCalFile(flatCalFileName)
obs.loadFluxCalFile(fluxCalFileName)
print "loaded data file and calibrations\n---------------------\n"
HotPixFile = getTimeMaskFileName(obsFileName)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFileName,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
wvlBinEdges,obsSpectra,obsIntTime = loadSpectra(obs,nCol,nRow)
nWvlBins=len(wvlBinEdges)-1
#divide spectrum by bin widths
binWidths = np.empty(nWvlBins)
for i in xrange(nWvlBins):
binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i]
obsSpectra = obsSpectra/binWidths
#divide by effective integration time
for iRow in xrange(nRow):
for iCol in xrange(nCol):
obsSpectra[iRow][iCol] = obsSpectra[iRow][iCol] / obsIntTime[iRow][iCol]
#print np.shape(obsSpectra)
#print nRow
#print nCol
#print nWvlBins
medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
print "target spectrum loaded\n---------------------\n"
if skyfile != "None":
sky = ObsFile(skyFileName)
sky.loadWvlCalFile(wvlCalFileName)
sky.loadFlatCalFile(flatCalFileName)
sky.loadFluxCalFile(fluxCalFileName)
skyTime = sky.getFromHeader("exptime")
skyHotPixFile = getTimeMaskFileName(skyFileName)
if not os.path.exists(skyHotPixFile):
hp.findHotPixels(skyFileName,skyHotPixFile)
print "Flux file pixel mask saved to %s"%(skyHotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(skyHotPixFile)
wvlBinEdges,skySpectra,skyIntTime = loadSpectra(sky,nCol,nRow)
skySpectra = skySpectra/binWidths
#divide by effective integration time
for iRow in xrange(nRow):
for iCol in xrange(nCol):
skySpectra[iRow][iCol] = skySpectra[iRow][iCol] / skyIntTime[iRow][iCol]
skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
#skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
print "sky spectrum loaded\n---------------------\n"
else:
#if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
skySpectrum = medianObsSpectrum
print "sky spectrum estimated as median of target file spectrum\n---------------------\n"
#subtract sky spectrum from every pixel
allSkySpectrum = obsSpectra-skySpectrum
#set any negative values to 0 after sky subtraction
allSkySpectrum[allSkySpectrum<0]=0
#take median of remaining sky subtracted spectra to get median object spectrum
finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
#load std spectrum for comparison
realSpectra = loadStd(objectName,wvlBinEdges)
print "real std spectrum loaded for reference\n---------------------\n"
#create plots
plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
pp = PdfPages(fullFluxPlotFileName)
matplotlib.rcParams['font.size']=6
#calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins),dtype=float)
for n in xrange(nWvlBins):
binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
wvls[n] = (wvlBinEdges[n]+(binsize/2.0))
plt.figure()
ax1 = plt.subplot(231)
ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
plt.plot(wvls,medianObsSpectrum)
#plt.show()
ax2 = plt.subplot(232)
ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
plt.plot(wvls,skySpectrum)
#plt.show()
ax5 = plt.subplot(233)
ax5.set_title('Sensitivity Spectrum')
plt.plot(wvls,obs.fluxWeights)
ax3 = plt.subplot(234)
ax3.set_title('MKID data minus sky in counts')
plt.plot(wvls,finalSpectrum/max(finalSpectrum))
ax4 = plt.subplot(235)
ax4.set_title('Rebinned Std Spectrum of %s'%(objectName))
plt.plot(wvls,realSpectra)
#ax7 = plt.subplot(337)
#ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
#plt.plot(wvls,fluxFactors*subtractedSpectra)
pp.savefig()
pp.close()
#del obs
#del sky
print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)
txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
fullFluxTxtFileName = os.path.join(txtDir,txtFileName)
outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
outarr[:,0]=wvls
outarr[:,1]=medianObsSpectrum
#save sensitivity spectrum to file
np.savetxt(fullFluxTxtFileName, outarr)
print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
def main():
obsSequence0 = """
051516
052520
"""
obsSequence1 = """
033323
041843
045902
"""
obsSequence2 = """
050404
054424
"""
obsSequence3 = """
054926
062942
"""
run = "PAL2012"
obsSequences = [obsSequence1, obsSequence2, obsSequence3]
wvlCals = ["063518", "063518", "063518"]
flatCals = ["20121211", "20121211", "20121211"]
fluxCalDates = ["20121206", "20121206", "20121206"]
fluxCals = ["20121207-072055", "20121207-072055", "20121207-072055"]
# Row coordinate of center of crab pulsar for each obsSequence
centersRow = [29, 29, 10]
# Col coordinate of center of crab pulsar for each obsSequence
centersCol = [29, 30, 14]
obsUtcDate = "20121212"
obsUtcDates = ["20121212", "20121212", "20121212"]
obsFileNames = []
obsFileNameTimestamps = []
wvlFileNames = []
flatFileNames = []
fluxFileNames = []
timeMaskFileNames = []
for iSeq in range(len(obsSequences)):
obsSequence = obsSequences[iSeq]
obsSequence = obsSequence.strip().split()
obsFileNameTimestamps.append(obsSequence)
obsUtcDate = obsUtcDates[iSeq]
sunsetDate = str(int(obsUtcDate) - 1)
obsSequence = [obsUtcDates[iSeq] + "-" + ts for ts in obsSequence]
obsFileNames.append([FileName(run=run, date=sunsetDate, tstamp=ts).obs() for ts in obsSequence])
timeMaskFileNames.append([FileName(run=run, date=sunsetDate, tstamp=ts).timeMask() for ts in obsSequence])
wvlCalTstamp = obsUtcDate + "-" + wvlCals[iSeq]
wvlFileNames.append(FileName(run=run, date=sunsetDate, tstamp=wvlCalTstamp).calSoln())
fluxFileNames.append(FileName(run=run, date=fluxCalDates[iSeq], tstamp=fluxCals[iSeq]).fluxSoln())
flatFileNames.append(FileName(run=run, date=flatCals[iSeq], tstamp="").flatSoln())
for iSeq, obsSequence in enumerate(obsSequences):
obsSequence = obsSequence.strip().split()
print obsSequence
for iOb, obs in enumerate(obsSequence):
timeMaskFileName = timeMaskFileNames[iSeq][iOb]
if not os.path.exists(timeMaskFileName):
print "Running hotpix for ", obs
hp.findHotPixels(obsFileNames[iSeq][iOb], timeMaskFileName)
print "Flux file pixel mask saved to %s" % (timeMaskFileName)
apertureRadius = 4
obLists = [[ObsFile(fn) for fn in seq] for seq in obsFileNames]
tstampFormat = "%H:%M:%S"
# print 'fileName','headerUnix','headerUTC','logUnix','packetReceivedUnixTime'
for iSeq, obList in enumerate(obLists):
for iOb, ob in enumerate(obList):
print ob.fileName
centerRow = centersRow[iSeq]
centerCol = centersCol[iSeq]
circCol, circRow = circ(centerCol, centerRow)
ob.loadTimeAdjustmentFile(FileName(run="PAL2012").timeAdjustments())
ob.loadWvlCalFile(wvlFileNames[iSeq])
ob.loadFlatCalFile(flatFileNames[iSeq])
ob.loadFluxCalFile(fluxFileNames[iSeq])
timeMaskFileName = timeMaskFileNames[iSeq][iOb]
ob.loadHotPixCalFile(timeMaskFileName)
ob.setWvlCutoffs(None, None)
for iSeq, obList in enumerate(obLists):
for iOb, ob in enumerate(obList):
print ob.fileName
centerRow = centersRow[iSeq]
centerCol = centersCol[iSeq]
circCol, circRow = circ(centerCol, centerRow)
imgDict = ob.getPixelCountImage()
img = imgDict["image"]
utils.plotArray(img, showMe=False)
aperture = plt.Circle((centerCol, centerRow), rad, fill=False, color="g")
aperture2 = plt.Circle((centerCol, centerRow), 2 * rad, fill=False, color="g")
plt.gca().add_patch(aperture)
plt.gca().add_patch(aperture2)
plt.show()
if __name__=='__main__':
run = 'PAL2014'
date = '20140924'
tstamp = '20140925-112538'
obsFN = FileName(run=run,date=date,tstamp=tstamp)
obsPath = obsFN.obs()
flatPath = FileName(run=run,date=date).flatSoln()
hotPath = obsFN.timeMask()
centroidPath = obsFN.centroidList()
fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln()
obs = ObsFile(obsPath)
if not os.path.exists(hotPath):
hp.findHotPixels(obsPath,hotPath)
obs.loadHotPixCalFile(hotPath,switchOnMask=False)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatPath)
obs.loadFluxCalFile(fluxPath)
obs.loadCentroidListFile(centroidPath)
obs.setWvlCutoffs(3000,11000)
centroidRa = '03:37:43.826'
centroidDec = '14:15:14.828'
haOffset = 150.
# imgDict = obs.getPixelCountImage(integrationTime=60,scaleByEffInt=True)
# plotArray(imgDict['image'])
flatCalFilenames[0] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5'
flatCalFilenames[1] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5'
fluxCalFileNames = ['/Scratch/fluxCalSolnFiles/20121206/fluxsol_20121207-124034.h5']
obsFn = FileName(run=run,date=sunsetDates[0],tstamp='20121209-120530').obs()
ob = ObsFile(obsFn)
print 'Loading wavelength calibration solution: ' + wvlCalFilenames[0]
ob.loadWvlCalFile(wvlCalFilenames[0])
print 'Loading flat calibration solution: ' + flatCalFilenames[0]
ob.loadFlatCalFile(flatCalFilenames[0])
ob.loadFluxCalFile(fluxCalFileNames[0])
#load/generate hot pixel mask file
HotPixFile = getTimeMaskFileName(obsFn)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFn,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
ob.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
frame = ob.getPixelCountImage(firstSec=0,integrationTime=300,weighted=True)
#hotPixMask = hotPixels.checkInterval(image=frame, firstSec=0, intTime=300, weighted=True, display=False)['mask']
#summed_array,bin_edges=ob.getApertureSpectrum(pixelCol=14,pixelRow=8,radius=7)
ob.plotApertureSpectrum(pixelCol=14,pixelRow=8,radius=7,weighted = True,fluxWeighted=True,lowCut=3000,highCut=9000)
'''
h = 6.626068*10**-34
c = 299792458.0
k = 1.3806503*10**-23
def main():
obsSequence1="""
033323
033825
034327
034830
035332
035834
040336
040838
041341
041843
042346
042848
043351
043853
044355
044857
045359
045902
"""
obsSequence2="""
050404
050907
051409
051912
052414
052917
053419
053922
"""
obsSequence3="""
054926
055428
055930
060432
060934
061436
061938
062440
062942
"""
run = 'PAL2012'
obsSequences = [obsSequence1,obsSequence2,obsSequence3]
wvlCals = ['063518','063518','063518']
flatCals = ['20121211','20121211','20121211']
fluxCalDates = ['20121206','20121206','20121206']
fluxCals = ['20121207-072055','20121207-072055','20121207-072055']
#Row coordinate of center of crab pulsar for each obsSequence
centersRow = [29,29,10]
#Col coordinate of center of crab pulsar for each obsSequence
centersCol = [29,30,14]
obsUtcDate = '20121212'
obsUtcDates = ['20121212','20121212','20121212']
obsFileNames = []
obsFileNameTimestamps = []
wvlFileNames = []
flatFileNames = []
fluxFileNames = []
timeMaskFileNames = []
plFileNames = []
skyFileNames = []
for iSeq in range(len(obsSequences)):
obsSequence = obsSequences[iSeq]
obsSequence = obsSequence.strip().split()
obsFileNameTimestamps.append(obsSequence)
obsUtcDate = obsUtcDates[iSeq]
sunsetDate = str(int(obsUtcDate)-1)
obsSequence = [obsUtcDates[iSeq]+'-'+ts for ts in obsSequence]
obsFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).obs() for ts in obsSequence])
timeMaskFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).timeMask() for ts in obsSequence])
wvlCalTstamp = obsUtcDate+'-'+wvlCals[iSeq]
wvlFileNames.append(FileName(run=run,date=sunsetDate,tstamp=wvlCalTstamp).calSoln())
fluxFileNames.append(FileName(run=run,date=fluxCalDates[iSeq],tstamp=fluxCals[iSeq]).fluxSoln())
flatFileNames.append(FileName(run=run,date=flatCals[iSeq],tstamp='').flatSoln())
plFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).crabList() for ts in obsSequence])
skyFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).crabSkyList() for ts in obsSequence])
for iSeq,obsSequence in enumerate(obsSequences):
obsSequence = obsSequence.strip().split()
print obsSequence
for iOb,obs in enumerate(obsSequence):
timeMaskFileName = timeMaskFileNames[iSeq][iOb]
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obs
hp.findHotPixels(obsFileNames[iSeq][iOb],timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
apertureRadius = 5
obLists = [[ObsFile(fn)for fn in seq ] for seq in obsFileNames]
tstampFormat = '%H:%M:%S'
#print 'fileName','headerUnix','headerUTC','logUnix','packetReceivedUnixTime'
for iSeq,obList in enumerate(obLists):
for iOb,ob in enumerate(obList):
print ob.fileName
centerRow = centersRow[iSeq]
centerCol = centersCol[iSeq]
circCol,circRow = circ(centerCol,centerRow,radius=apertureRadius)
skyCol,skyRow = halfTorus(centerCol,centerRow,radiusInner=apertureRadius,radiusOuter=2*apertureRadius)
ob.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
ob.loadWvlCalFile(wvlFileNames[iSeq])
ob.loadFlatCalFile(flatFileNames[iSeq])
ob.loadFluxCalFile(fluxFileNames[iSeq])
timeMaskFileName = timeMaskFileNames[iSeq][iOb]
ob.loadHotPixCalFile(timeMaskFileName)
ob.setWvlCutoffs(4000,11000)
photlist.writePhotonList(ob,rowList=skyRow,colList=skyCol,filename=skyFileNames[iSeq][iOb])
print 'wrote sky list'
photlist.writePhotonList(ob,rowList=circRow,colList=circCol,filename=plFileNames[iSeq][iOb])
print 'wrote crab list'
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) >2:
fileNum = str(sys.argv[2])
else:
fileNum = '0'
#science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
wvldir = params[2].split('=')[1].strip()
obsfile = params[3].split('=')[1].strip()
skyfile = params[4].split('=')[1].strip()
flatfile = params[5].split('=')[1].strip()
wvlfile = params[6].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121206"
if len(params)>10:
xpix = int(params[10].split('=')[1].strip())
ypix = int(params[11].split('=')[1].strip())
apertureRadius = int(params[12].split('=')[1].strip())
startTime = int(params[13].split('=')[1].strip())
intTime =int(params[14].split('=')[1].strip())
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
obs = ObsFile(obsFileName)
obs.loadWvlCalFile(wvlCalFileName)
obs.loadFlatCalFile(flatCalFileName)
print "analyzing file %s"%(obsFileName)
print "loaded data file and calibrations\n---------------------\n"
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
#wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
#nWvlBins=len(wvlBinEdges)-1
#print np.shape(obsSpectra)
#print nRow
#print nCol
#print nWvlBins
#load/generate hot pixel mask file
HotPixFile = getTimeMaskFileName(obsFileName)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFileName,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
#######
#EVERYTHING BEFORE HERE IS STANDARD FILE/CALFILE LOADING
startWvl = 3000
#stopWvl = 7000 #for V-band
stopWvl = 9000 #for R-band
print "Making spectral cube"
#for pg0220 first sec should be 80 since object is moving around before this
#for pg0220A first sec should be 70, integration time is 140
#for landolt 9542 first sec should be 20, int time is -1
cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, wvlStart = startWvl, wvlStop = stopWvl, wvlBinEdges = [startWvl,stopWvl], weighted=False)
cube= np.array(cubeDict['cube'], dtype=np.double)
wvlBinEdges= cubeDict['wvlBinEdges']
effIntTime= cubeDict['effIntTime']
print "median effective integration time = ", np.median(effIntTime)
nWvlBins=len(wvlBinEdges)-1
print "cube shape ", np.shape(cube)
print "effIntTime shape ", np.shape(effIntTime)
#add third dimension to effIntTime for broadcasting
effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
cube /= effIntTime #put cube into counts/s
#calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins),dtype=float)
for n in xrange(nWvlBins):
binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
wvls[n] = (wvlBinEdges[n]+(binsize/2.0))
print "wvls ",wvls
#reshape cube for makeMovie
movieCube = np.zeros((nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float)
for i in xrange(nWvlBins):
movieCube[i,:,:] = cube[:,:,i]
#show individual frames as they are made to debug
#plt.matshow(movieCube[i],vmin = 0, vmax = 100)
#plt.show()
print "movieCube shape ", np.shape(movieCube)
print "wvls shape ", np.shape(wvls)
#print cube
#print "--------------------------"
#print movieCube
print "adding frames with wvl below ", stopWvl
finalframe = np.zeros((1,np.shape(movieCube)[1],np.shape(movieCube)[2]))
for f in xrange(len(wvls[wvls<stopWvl])):
print wvls[f]
finalframe[0]+=movieCube[f]
plt.matshow(movieCube[f],vmin=0,vmax = 40)
plt.show()
movieCube = finalframe
np.savez('%s_%s.npz'%(objectName,fileNum),stack=movieCube,wvls=wvls)
print "Saved frame to .npz file"
plt.matshow(movieCube[0],vmin=0,vmax = 40)
plt.show()
def __init__(self,paramFile,plots=False,verbose=False):
"""
Opens flux file, prepares standard spectrum, and calculates flux factors for the file.
Method is provided in param file. If 'relative' is selected, an obs file with standard star defocused over
the entire array is expected, with accompanying sky file to do sky subtraction.
If any other method is provided, 'absolute' will be done by default, wherein a point source is assumed
to be present. The obs file is then broken into spectral frames with photometry (psf or aper) performed
on each frame to generate the ARCONS observed spectrum.
"""
self.verbose=verbose
self.plots = plots
self.params = readDict()
self.params.read_from_file(paramFile)
run = self.params['run']
sunsetDate = self.params['fluxSunsetLocalDate']
self.fluxTstamp = self.params['fluxTimestamp']
skyTstamp = self.params['skyTimestamp']
wvlSunsetDate = self.params['wvlCalSunsetLocalDate']
wvlTimestamp = self.params['wvlCalTimestamp']
flatCalFileName = self.params['flatCalFileName']
needTimeAdjust = self.params['needTimeAdjust']
self.deadtime = float(self.params['deadtime']) #from firmware pulse detection
self.timeSpacingCut = self.params['timeSpacingCut']
bLoadBeammap = self.params.get('bLoadBeammap',False)
self.method = self.params['method']
self.objectName = self.params['object']
self.r = float(self.params['energyResolution'])
self.photometry = self.params['photometry']
self.centroidRow = self.params['centroidRow']
self.centroidCol = self.params['centroidCol']
self.aperture = self.params['apertureRad']
self.annulusInner = self.params['annulusInner']
self.annulusOuter = self.params['annulusOuter']
self.collectingArea = self.params['collectingArea']
self.startTime = self.params['startTime']
self.intTime = self.params['integrationTime']
fluxFN = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp)
self.fluxFileName = fluxFN.obs()
self.fluxFile = ObsFile(self.fluxFileName)
if self.plots:
self.plotSavePath = os.environ['MKID_PROC_PATH']+os.sep+'fluxCalSolnFiles'+os.sep+run+os.sep+sunsetDate+os.sep+'plots'+os.sep
if not os.path.exists(self.plotSavePath): os.mkdir(self.plotSavePath)
if self.verbose: print "Created directory %s"%self.plotSavePath
obsFNs = [fluxFN]
self.obsList = [self.fluxFile]
if self.startTime in ['',None]: self.startTime=0
if self.intTime in ['',None]: self.intTime=-1
if self.method=="relative":
try:
print "performing Relative Flux Calibration"
skyFN = FileName(run=run,date=sunsetDate,tstamp=skyTstamp)
self.skyFileName = skyFN.obs()
self.skyFile = ObsFile(self.skyFileName)
obsFNs.append(skyFN)
self.obsList.append(self.skyFile)
except:
print "For relative flux calibration a sky file must be provided in param file"
self.__del__()
else:
self.method='absolute'
print "performing Absolute Flux Calibration"
if self.photometry not in ['aperture','PSF']: self.photometry='PSF' #default to PSF fitting if no valid photometry selected
timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
timeAdjustFileName = FileName(run=run).timeAdjustments()
#make filename for output fluxCalSoln file
self.fluxCalFileName = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp).fluxSoln()
print "Creating flux cal: %s"%self.fluxCalFileName
if wvlSunsetDate != '':
wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
if flatCalFileName =='':
flatCalFileName=FileName(obsFile=self.fluxFile).flatSoln()
#load cal files for flux file and, if necessary, sky file
for iObs,obs in enumerate(self.obsList):
if bLoadBeammap:
print 'loading beammap',os.environ['MKID_BEAMMAP_PATH']
obs.loadBeammapFile(os.environ['MKID_BEAMMAP_PATH'])
if wvlSunsetDate != '':
obs.loadWvlCalFile(wvlCalFileName)
else:
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatCalFileName)
obs.setWvlCutoffs(-1,-1)
if needTimeAdjust:
obs.loadTimeAdjustmentFile(timeAdjustFileName)
timeMaskFileName = timeMaskFileNames[iObs]
print timeMaskFileName
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obs
hp.findHotPixels(obsFile=obs,outputFileName=timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
print "Flux cal/sky file pixel mask saved to %s"%(timeMaskFileName)
obs.loadHotPixCalFile(timeMaskFileName)
if self.verbose: print "Loaded hot pixel file %s"%timeMaskFileName
#get flat cal binning information since flux cal will need to match it
self.wvlBinEdges = self.fluxFile.flatCalFile.root.flatcal.wavelengthBins.read()
self.nWvlBins = self.fluxFile.flatWeights.shape[2]
self.binWidths = np.empty((self.nWvlBins),dtype=float)
self.binCenters = np.empty((self.nWvlBins),dtype=float)
for i in xrange(self.nWvlBins):
self.binWidths[i] = self.wvlBinEdges[i+1]-self.wvlBinEdges[i]
self.binCenters[i] = (self.wvlBinEdges[i]+(self.binWidths[i]/2.0))
if self.method=='relative':
print "Extracting ARCONS flux and sky spectra"
self.loadRelativeSpectrum()
print "Flux Spectrum loaded"
self.loadSkySpectrum()
print "Sky Spectrum loaded"
elif self.method=='absolute':
print "Extracting ARCONS point source spectrum"
self.loadAbsoluteSpectrum()
print "Loading standard spectrum"
try:
self.loadStdSpectrum(self.objectName)
except KeyError:
print "Invalid spectrum object name"
self.__del__()
sys.exit()
print "Generating sensitivity curve"
self.calculateFactors()
print "Sensitivity Curve calculated"
print "Writing fluxCal to file %s"%self.fluxCalFileName
self.writeFactors(self.fluxCalFileName)
if self.plots: self.makePlots()
print "Done"
def main(testRun=False):
#INPUTS:
# - testRun : If True, run only a single exposure.
#Try remapping the pixels - set to None to not do this....
pixRemapFileName = FileName(run='PAL2012').pixRemap()
obsSequence0="""
051516
052018
052520
"""
obsSequence1="""
033323
033825
034327
034830
035332
035834
040336
040838
041341
041843
042346
042848
043351
043853
044355
044857
045359
045902
"""
obsSequence2="""
050404
050907
051409
051912
052414
052917
053419
053922
"""
# 054424 - repointed mid-exposure, leave out for now.
obsSequence3="""
054926
055428
055930
060432
060934
061436
061938
062440
062942
"""
run = 'PAL2012'
obsSequences = [obsSequence0,obsSequence1,obsSequence2,obsSequence3]
#If test run is requested, override and just run one test exposure.
if testRun is True:
obsSequences = ['9999999',
'''
033323
''',
'9999999',
'9999999']
wvlCals = ['051341','063518','063518','063518']
flatCals = ['20121211','20121211','20121211','20121211']
#fluxCalDates = ['20121206','20121206','20121206','20121206']
#fluxCals = ['20121207-072055','20121207-072055','20121207-072055','20121207-072055']
fluxCalDates = ['20121211','20121211','20121211','20121211']
fluxCals = ['absolute_021727','absolute_021727','absolute_021727','absolute_021727']
#Row coordinate of center of crab pulsar for each obsSequence
centersRow = [9,30,29,10]
#Col coordinate of center of crab pulsar for each obsSequence
centersCol = [13,30,30,14]
centerRA = '05:34:31.93830'
centerDec = '+22:00:52.1758'
obsUtcDate = '20121212'
obsUtcDates = ['20121206','20121212','20121212','20121212']
obsFileNames = []
obsFileNameTimestamps = []
wvlFileNames = []
flatFileNames = []
fluxFileNames = []
timeMaskFileNames = []
centroidFileNames = []
for iSeq in range(len(obsSequences)):
obsSequence = obsSequences[iSeq]
obsSequence = obsSequence.strip().split()
obsFileNameTimestamps.append(obsSequence)
obsUtcDate = obsUtcDates[iSeq]
sunsetDate = str(int(obsUtcDate)-1)
obsSequence = [obsUtcDates[iSeq]+'-'+ts for ts in obsSequence]
obsFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).obs() for ts in obsSequence])
timeMaskFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).timeMask() for ts in obsSequence])
centroidFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).centroidList() for ts in obsSequence])
wvlCalTstamp = obsUtcDate+'-'+wvlCals[iSeq]
wvlFileNames.append(FileName(run=run,date=sunsetDate,tstamp=wvlCalTstamp).calSoln())
fluxFileNames.append(FileName(run=run,date=fluxCalDates[iSeq],tstamp=fluxCals[iSeq]).fluxSoln())
flatFileNames.append(FileName(run=run,date=flatCals[iSeq]).illumSoln()) # *** CURRENTLY USES ILLUMINATION SOLUTION! ***
centroidFileNames.append(FileName(run=run,date=sunsetDate,tstamp=ts).centroidList())
# #Make hot pixel masks if needed
# for iSeq,obsSequence in enumerate(obsSequences):
# obsSequence = obsSequence.strip().split()
# print obsSequence
# for iOb,obs in enumerate(obsSequence):
# timeMaskFileName = timeMaskFileNames[iSeq][iOb]
# if os.path.exists(obsFileNames[iSeq][iOb]) and not os.path.exists(timeMaskFileName):
# print 'Running hotpix for ',obs
# hp.findHotPixels(obsFileNames[iSeq][iOb],timeMaskFileName)
# print "Flux file pixel mask saved to %s"%(timeMaskFileName)
# else:
# print 'Skipping hot pixel mask creation for file '+obsFileNames[iSeq][iOb]
#apertureRadius = 4
obLists = [[ObsFile(fn)for fn in seq if os.path.exists(fn)] for seq in obsFileNames]
tstampFormat = '%H:%M:%S'
#print 'fileName','headerUnix','headerUTC','logUnix','packetReceivedUnixTime'
print '---------Making hot pixel masks/getting centroids-----------'
for iSeq,obList in enumerate(obLists):
for iOb,ob in enumerate(obList):
timeAdjFileName = FileName(run='PAL2012').timeAdjustments()
wvlCalFileName = wvlFileNames[iSeq]
flatCalFileName = flatFileNames[iSeq]
fluxCalFileName = fluxFileNames[iSeq]
timeMaskFileName = FileName(obsFile=ob).timeMask() #timeMaskFileNames[iSeq][iOb]
centroidFileName = FileName(obsFile=ob).centroidList()
if not os.path.exists(centroidFileName) or not os.path.exists(timeMaskFileName):
print 'Loading calibration files:'
print [os.path.basename(x) for x in [timeAdjFileName,wvlCalFileName,flatCalFileName, \
fluxCalFileName,timeMaskFileName, centroidFileName]]
ob.loadTimeAdjustmentFile(timeAdjFileName)
ob.loadWvlCalFile(wvlCalFileName)
ob.loadFlatCalFile(flatCalFileName)
ob.loadFluxCalFile(fluxCalFileName)
ob.setWvlCutoffs(3000.,12000.) #Keep most of the wavelength range for the purposes of centroiding/hot pixel detection.
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',ob.fileName
#Run hot pixel detection on *calibrated* file.
#But don't use flux weighting, as this scales by a large amount, and throws
#the poisson statistics way too much. (i.e., we really need to use
#counts *detected* for shot noise statistics, rather than
#estimated counts above the atmosphere....)
hp.findHotPixels(obsFile=ob,outputFileName=timeMaskFileName,
useRawCounts=False,weighted=True,fluxWeighted=False)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
else:
print 'Skipping hot pixel mask creation for file '+obsFileNames[iSeq][iOb]
if not os.path.exists(centroidFileName):
ob.loadHotPixCalFile(timeMaskFileName)
print 'Running CentroidCalc for ',ob.fileName
cc.centroidCalc(ob, centerRA, centerDec, outputFileName=centroidFileName,
guessTime=300, integrationTime=30, secondMaxCountsForDisplay=500,
xyapprox=[centersCol[iSeq],centersRow[iSeq]])
print "Centroiding calculations saved to %s"%(centroidFileName)
else:
print "Centroid file already exists - skipping: "+centroidFileName
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) >3:
filenum = str('_'+sys.argv[3])
else:
filenum = '_0'
#science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
wvldir = params[2].split('=')[1].strip()
obsfile = params[3].split('=')[1].strip()
skyfile = params[4].split('=')[1].strip()
flatfile = params[5].split('=')[1].strip()
wvlfile = params[6].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
if len(params)>10:
xpix = int(params[10].split('=')[1].strip())
ypix = int(params[11].split('=')[1].strip())
apertureRadius = int(params[12].split('=')[1].strip())
#flux cal object parameter file
params2 = []
param2file = sys.argv[2]
f = open(param2file,'r')
for line in f:
params2.append(line)
f.close()
fluxdir = params2[7].split('=')[1].strip()
fluxfile = params2[8].split('=')[1].strip()
fluxCalObject = params2[9].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
print "obsfile = ",obsFileName
print "skyfile = ",skyFileName
print "wvlcal = ", wvlCalFileName
print "flatcal = ", flatCalFileName
print "fluxcal = ", fluxCalFileName
print "object = ", objectName
print "flux cal object = ", fluxCalObject
print "\n---------------------\n"
obs = ObsFile(obsFileName)
obs.loadWvlCalFile(wvlCalFileName)
obs.loadFlatCalFile(flatCalFileName)
obs.loadFluxCalFile(fluxCalFileName)
print "loaded data file and calibrations\n---------------------\n"
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
#wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
#nWvlBins=len(wvlBinEdges)-1
#print np.shape(obsSpectra)
#print nRow
#print nCol
#print nWvlBins
"""
medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
print "target spectrum loaded\n---------------------\n"
if skyfile != "None":
sky = ObsFile(skyFileName)
sky.loadWvlCalFile(wvlCalFileName)
sky.loadFlatCalFile(flatCalFileName)
sky.loadFluxCalFile(fluxCalFileName)
skyTime = sky.getFromHeader("exptime")
wvlBinEdges,skySpectra = loadSpectra(sky,nCol,nRow)
skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
print "sky spectrum loaded\n---------------------\n"
else:
#if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
skySpectrum = calculateMedian(obsSpectra, nCol, nRow, nWvlBins)
print "sky spectrum estimated as median of target file spectrum\n---------------------\n"
#subtract sky spectrum from every pixel
allSkySpectrum = obsSpectra-skySpectrum
#set any negative values to 0 after sky subtraction
allSkySpectrum[allSkySpectrum<0]=0
#take median of remaining sky subtracted spectra to get median object spectrum
finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
"""
#load/generate hot pixel mask file
HotPixFile = getTimeMaskFileName(obsFileName)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFileName,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
print "Making spectrum with Aperture Spectrum in ObsFile"
#use Aperture Spectrum from obsfile
medianObsSpectrum, wvlBinEdges = obs.getApertureSpectrum(pixelCol=ypix,pixelRow=xpix,radius1=apertureRadius, radius2 = apertureRadius*2.0,weighted=True, fluxWeighted=True, lowCut=3000, highCut=7000)
nWvlBins=len(wvlBinEdges)-1
#load std spectrum for comparison
try:
realSpectra = loadStd(objectName,wvlBinEdges)
print "real std spectrum loaded for reference\n---------------------\n"
stdTitle = "Rebinned Std Spectrum of %s"%(objectName)
except KeyError:
print "Key Error loading MKIDStd"
realSpectra = np.ones(nWvlBins)
stdTitle = "No MKIDStd spectrum available for %s"%(objectName)
#create plots
plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
pp = PdfPages(fullFluxPlotFileName)
matplotlib.rcParams['font.size']=6
#calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins),dtype=float)
for n in xrange(nWvlBins):
binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
wvls[n] = (wvlBinEdges[n]+(binsize/2.0))
plt.figure()
ax1 = plt.subplot(221)
ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
ax1.set_xlim((4000,11000))
ax1.set_ylim((min(medianObsSpectrum[(wvls>4000) & (wvls<11000)]),max(medianObsSpectrum[(wvls>4000) & (wvls<8000)])))
plt.plot(wvls,medianObsSpectrum)
#plt.show()
#ax2 = plt.subplot(232)
#ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
#plt.plot(wvls,skySpectrum)
#plt.show()
ax5 = plt.subplot(223)
ax5.set_title('Sensitivity Spectrum')
ax5.set_xlim((3000,13000))
ax5.set_ylim((0,5))
plt.plot(wvls,obs.fluxWeights)
#ax3 = plt.subplot(234)
#ax3.set_title('MKID data minus sky in counts')
#plt.plot(wvls,finalSpectrum/max(finalSpectrum))
ax4 = plt.subplot(222)
ax4.set_title(stdTitle)
plt.plot(wvls,realSpectra)
#ax7 = plt.subplot(337)
#ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
#plt.plot(wvls,fluxFactors*subtractedSpectra)
pp.savefig()
pp.close()
#del obs
#del sky
print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)
txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
fullFluxTxtFileName = os.path.join(txtDir,txtFileName)
outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
outarr[:,0]=wvls
outarr[:,1]=medianObsSpectrum
#save sensitivity spectrum to file
np.savetxt(fullFluxTxtFileName, outarr)
print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
def main():
#open the sky file for hr9087
run = 'PAL2012'
date = '20121210'
wvlCal = '20121211-052230'
obsTimestamp = '20121211-051650'
flatCalDate = '20121211'
flatCalTstamp = '20121212-074700'
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
timeMaskFileName = obsFN.timeMask()
wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obsFileName
hp.findHotPixels(obsFileName,timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs = ObsFile(obsFileName)
obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
obs.loadWvlCalFile(wvlFileName)
obs.loadFlatCalFile(flatFileName)
obs.loadHotPixCalFile(timeMaskFileName)
#obs.setWvlCutoffs(4000,8000)
#get image before and after flat cal
print 'getting images'
beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True)
rawCubeDict = obs.getSpectralCube(weighted=False)
rawCube = np.array(rawCubeDict['cube'],dtype=np.double)
effIntTime = rawCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
rawCube *= maxIntTime / effIntTime3d
rawCube[np.isnan(rawCube)] = 0
rawCube[rawCube == np.inf] = 0
beforeImg = np.sum(rawCube,axis=-1)
print 'finished first cube'
flatCubeDict = obs.getSpectralCube(weighted=True)
flatCube = np.array(flatCubeDict['cube'],dtype=np.double)
effIntTime = flatCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
flatCube *= maxIntTime / effIntTime3d
flatCube[np.isnan(flatCube)] = 0
flatCube[flatCube == np.inf] = 0
afterImg = np.sum(flatCube,axis=-1)
plotArray(title='before flatcal',image=beforeImg)
plotArray(title='after flatcal',image=afterImg)
print 'before sdev',np.std(beforeImg[afterImg!=0])
print 'after sdev',np.std(afterImg[afterImg!=0])
np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
#date = '20140925'
#obsTimestamp ='20140926-104528' #psr J0337
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
print obsFileName
timeMaskFileName = obsFN.timeMask()
if wvlCal != '':
wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
illumFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).illumSoln()
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obsFileName
hp.findHotPixels(inputFileName=obsFileName,outputFileName=timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs = ObsFile(obsFileName)
exptime = obs.getFromHeader('exptime')
#obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
if wvlCal != '':
obs.loadWvlCalFile(wvlFileName)
else:
obs.loadBestWvlCalFile()
obs.loadHotPixCalFile(timeMaskFileName)
obs.setWvlCutoffs(lowerWvlCut,upperWvlCut)
obs.loadFlatCalFile(flatFileName)
def __init__(self,paramFile):
"""
opens flat file,sets wavelength binnning parameters, and calculates flat factors for the file
"""
self.params = readDict()
self.params.read_from_file(paramFile)
run = self.params['run']
sunsetDate = self.params['sunsetDate']
flatTstamp = self.params['flatTstamp']
wvlSunsetDate = self.params['wvlSunsetDate']
wvlTimestamp = self.params['wvlTimestamp']
obsSequence = self.params['obsSequence']
needTimeAdjust = self.params['needTimeAdjust']
self.deadtime = self.params['deadtime'] #from firmware pulse detection
self.intTime = self.params['intTime']
self.timeSpacingCut = self.params['timeSpacingCut']
self.nSigmaClip = self.params['nSigmaClip']
self.nNearest = self.params['nNearest']
obsFNs = [FileName(run=run,date=sunsetDate,tstamp=obsTstamp) for obsTstamp in obsSequence]
self.obsFileNames = [fn.obs() for fn in obsFNs]
self.obsList = [ObsFile(obsFileName) for obsFileName in self.obsFileNames]
timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
timeAdjustFileName = FileName(run=run).timeAdjustments()
print len(self.obsFileNames), 'flat files to co-add'
self.flatCalFileName = FileName(run=run,date=sunsetDate,tstamp=flatTstamp).illumSoln()
if wvlSunsetDate != '':
wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
for iObs,obs in enumerate(self.obsList):
if wvlSunsetDate != '':
obs.loadWvlCalFile(wvlCalFileName)
else:
obs.loadBestWvlCalFile()
if needTimeAdjust:
obs.loadTimeAdjustmentFile(timeAdjustFileName)
timeMaskFileName = timeMaskFileNames[iObs]
print timeMaskFileName
#Temporary step, remove old hotpix file
#if os.path.exists(timeMaskFileName):
# os.remove(timeMaskFileName)
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obs
hp.findHotPixels(self.obsFileNames[iObs],timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs.loadHotPixCalFile(timeMaskFileName)
self.wvlFlags = self.obsList[0].wvlFlagTable
self.nRow = self.obsList[0].nRow
self.nCol = self.obsList[0].nCol
print 'files opened'
#self.wvlBinWidth = params['wvlBinWidth'] #angstroms
self.energyBinWidth = self.params['energyBinWidth'] #eV
self.wvlStart = self.params['wvlStart'] #angstroms
self.wvlStop = self.params['wvlStop'] #angstroms
self.wvlBinEdges = ObsFile.makeWvlBins(self.energyBinWidth,self.wvlStart,self.wvlStop)
self.intTime = self.params['intTime']
self.countRateCutoff = self.params['countRateCutoff']
self.fractionOfChunksToTrim = self.params['fractionOfChunksToTrim']
#wvlBinEdges includes both lower and upper limits, so number of bins is 1 less than number of edges
self.nWvlBins = len(self.wvlBinEdges)-1
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False):
'''
Test run of obsFile.writePhotonList. fileName can be used
to specify the output file name. If not specified, default
name/location is used.
Now includes test for pixel remapping....
'''
#Details of example obs file to run test on.
# run = 'PAL2012'
# date = '20121207'
# tstamp = '20121208-074649'
# calTstamp='20121208-070505'
# fluxTstamp='20121208-133002'
run = 'PAL2013'
date = '20131209'
tstamp = '20131209-115553'
centroidTstamp = tstamp
calTstamp='20131209-132225'
fluxTstamp='ones'
flatTstamp='20131209'
if doPixRemap==True:
pixRemapFileName = FileName(run=run).pixRemap()
else:
pixRemapFileName = None
#Load up the obs file
obsFileName = FileName(run=run, date=date, tstamp=tstamp)
obsFile = ObsFile(obsFileName.obs())
obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())
HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask()
if not os.path.exists(HotPixFile): #check if hot pix file already exists
hp.findHotPixels(obsFileName.obs(),HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obsFile.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
#hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8''
centroid_RA = '12:13:56.42'
centroid_DEC = '32:56:30.8'
CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList()
#print "checking centroid file", CentFile
#if not os.path.exists(CentFile): #check if cent pix file already exists
cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10)
# print "Flux file centroid cal saved to %s"%(CentFile)
obsFile.loadCentroidListFile(CentFile)
print "Centroid File loaded %s"%(CentFile)
#Load up associated calibrations
#obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
#obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
#obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
#And write out the results....
obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
pixRemapFileName=pixRemapFileName)
#Read the results back in....
#photFile = photList.PhotFile(outputFilename)
print "Wrote photon list file: ", outputFileName
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) > 2:
fileNum = str(sys.argv[2])
else:
fileNum = "0"
# science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile, "r")
for line in f:
params.append(line)
f.close()
datadir = params[0].split("=")[1].strip()
flatdir = params[1].split("=")[1].strip()
wvldir = params[2].split("=")[1].strip()
obsfile = params[3].split("=")[1].strip()
skyfile = params[4].split("=")[1].strip()
flatfile = params[5].split("=")[1].strip()
wvlfile = params[6].split("=")[1].strip()
objectName = params[9].split("=")[1].strip()
# wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121205"
# objectName = "crabNight1"
if len(params) > 10:
xpix = int(params[10].split("=")[1].strip())
ypix = int(params[11].split("=")[1].strip())
apertureRadius = int(params[12].split("=")[1].strip())
startTime = int(params[13].split("=")[1].strip())
intTime = int(params[14].split("=")[1].strip())
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
obs = ObsFile(obsFileName)
# obs.loadWvlCalFile(wvlCalFileName)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatCalFileName)
print "analyzing file %s" % (obsFileName)
print "loaded data file and calibrations\n---------------------\n"
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
# wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
# nWvlBins=len(wvlBinEdges)-1
# print np.shape(obsSpectra)
# print nRow
# print nCol
# print nWvlBins
# Apply Hot pixel masking before getting dead time correction
# HotPixFile = getTimeMaskFileName(obsFileName)
HotPixFile = FileName(obsFile=obs).timeMask()
print "making hot pixel file ", HotPixFile
if not os.path.exists(HotPixFile): # check if hot pix file already exists
hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
print "Flux file pixel mask saved to %s" % (HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s" % (HotPixFile)
# GET RAW PIXEL COUNT IMAGE TO CALCULATE CORRECTION FACTORS
print "Making raw cube to get dead time correction"
cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=False)
cube = np.array(cubeDict["cube"], dtype=np.double)
wvlBinEdges = cubeDict["wvlBinEdges"]
effIntTime = cubeDict["effIntTime"]
print "median effective integration time = ", np.median(effIntTime)
nWvlBins = len(wvlBinEdges) - 1
print "cube shape ", np.shape(cube)
print "effIntTime shape ", np.shape(effIntTime)
# add third dimension to effIntTime for broadcasting
effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
# put cube into counts/s in each pixel
cube /= effIntTime
# CALCULATE DEADTIME CORRECTION
# NEED TOTAL COUNTS PER SECOND FOR EACH PIXEL TO DO PROPERLY
# ASSUMES SAME CORRECTION FACTOR APPLIED FOR EACH WAVELENGTH, MEANING NO WL DEPENDANCE ON DEAD TIME EFFECT
DTCorr = np.zeros((np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
for f in range(0, np.shape(cube)[2]):
print cube[:, :, f]
print "-----------------------"
DTCorr += cube[:, :, f]
print DTCorr
print "\n=====================\n"
# Correct for 100 us dead time
DTCorrNew = DTCorr / (1 - DTCorr * 100e-6)
CorrFactors = DTCorrNew / DTCorr # This is what the frames need to be multiplied by to get their true values
print "Dead time correction factors = "
print CorrFactors
# REMAKE CUBE WITH FLAT WEIGHTS AND APPLY DEAD TIME CORRECTION AS WELL
print "Making Weighted cube"
# load/generate hot pixel mask file
# HotPixFile = getTimeMaskFileName(obsFileName)
HotPixFile = FileName(obsFile=obs).timeMask()
if not os.path.exists(HotPixFile): # check if hot pix file already exists
hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
print "Flux file pixel mask saved to %s" % (HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s" % (HotPixFile)
cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=False)
# cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=True)
cube = np.array(cubeDict["cube"], dtype=np.double)
wvlBinEdges = cubeDict["wvlBinEdges"]
effIntTime = cubeDict["effIntTime"]
print "median effective integration time = ", np.median(effIntTime)
nWvlBins = len(wvlBinEdges) - 1
print "cube shape ", np.shape(cube)
print "effIntTime shape ", np.shape(effIntTime)
# add third dimension to effIntTime for broadcasting
effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
# put cube into counts/s in each pixel
cube /= effIntTime
# add third dimension to CorrFactors for broadcasting
CorrFactors = np.reshape(CorrFactors, np.shape(CorrFactors) + (1,))
# apply dead time correction factors
cube *= CorrFactors
# calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins), dtype=float)
for n in xrange(nWvlBins):
binsize = wvlBinEdges[n + 1] - wvlBinEdges[n]
wvls[n] = wvlBinEdges[n] + (binsize / 2.0)
print "wvls ", wvls
# reshape cube for makeMovie
movieCube = np.zeros((nWvlBins, np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
for i in xrange(nWvlBins):
movieCube[i, :, :] = cube[:, :, i]
# show individual frames as they are made to debug
# plt.matshow(movieCube[i],vmin = 0, vmax = 100)
# plt.show()
print "movieCube shape ", np.shape(movieCube)
print "wvls shape ", np.shape(wvls)
# print cube
# print "--------------------------"
# print movieCube
outdir = "/home/srmeeker/scratch/standards/"
np.savez(outdir + "%s_raw_%s.npz" % (objectName, fileNum), stack=movieCube, wvls=wvls)
utils.makeMovie(
movieCube,
frameTitles=wvls,
cbar=True,
outName=outdir + "%s_raw_%s.gif" % (objectName, fileNum),
normMin=0,
normMax=50,
)
"""
def hotPixelsTest(testFileName=FileName(run='PAL2012',date='20121208',tstamp='20121209-044636').obs()):
'''
Runs some basic checks for consistency between intermediate output
masks and the final 'bad time lists'.
To run
- from Python:
hotPixelsTest('someObsFile.h5')
- from command line:
python hotPixelsTest.py someObsFile.h5
(No parameter file need be supplied).
'''
workingDir = '/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/hotPixTest2/'
outputFile = workingDir + 'testoutput.h5'
paramFile = os.path.join(os.path.dirname(__file__),'../../params/hotPixels.dict') #/Users/vaneyken/UCSB/ARCONS/pipeline/github/ARCONS-pipeline/params/hotPixels.dict'
testStartTime = 2 #In seconds
testEndTime = 4 #In seconds
timeStep = 2 #In seconds (deliberately equal to start time - end time)
fwhm = 3.0
boxSize = 5
nSigmaHot = 2.5
nSigmaCold = 2.0
hp.findHotPixels(paramFile=paramFile, inputFileName=testFileName,
outputFileName=outputFile, timeStep=timeStep,
startTime=testStartTime, endTime=testEndTime,
fwhm=fwhm, boxSize=boxSize, nSigmaHot=nSigmaHot,
nSigmaCold=nSigmaCold, display=True)
intermediateOutput = hp.checkInterval(inputFileName=testFileName, display=True,
firstSec=testStartTime,
intTime=testEndTime - testStartTime,
fwhm=fwhm, boxSize=boxSize,
nSigmaHot=nSigmaHot, nSigmaCold=nSigmaCold)
hpOutput = hp.readHotPixels(outputFile)
intMask = intermediateOutput['mask'] > 0 #Make a Boolean mask - any code > 0 is bad for some reason.
intervals = hpOutput['intervals']
reasons = hpOutput['reasons']
#Find the number of entries for each pixel in both the 'intervals' and the
#'reasons' arrays.
nIntervals = np.reshape([len(x) for x in intervals.flat], np.shape(intervals))
nReasons = np.reshape([len(x) for x in reasons.flat], np.shape(reasons))
#Union the interval lists for each pixel to give an array of single (multi-component) interval objects:
uIntervals = np.reshape(np.array([interval.union(x) for x in intervals.flat],
dtype='object'), np.shape(intervals))
#Create a boolean mask that should be True for all bad (hot/cold/dead/other) pixels within the test time range
finalMask = np.reshape([(interval(testStartTime, testEndTime) in x)
for x in uIntervals.flat], np.shape(uIntervals))
assert np.all(np.equal(intMask, finalMask))
assert np.all(np.equal(nIntervals, nReasons))
print
print "All seems okay. The two plots shown should look identical."
def main():
obsSequence0="""
112633
112731
113234
113737
114240
114743
115246
115749
120252
120755
121258
121801
122304
"""
run = 'PAL2012'
obsSequences = [obsSequence0]
wvlCals = ['112506']
flatCals = ['20121211']
fluxCalDates = ['20121206']
fluxCals = ['20121207-072055']
#Row coordinate of center of crab pulsar for each obsSequence
centersRow = [30]
#Col coordinate of center of crab pulsar for each obsSequence
centersCol = [30]
obsUtcDates = ['20121208']
obsFileNames = []
obsFileNameTimestamps = []
wvlFileNames = []
flatFileNames = []
fluxFileNames = []
timeMaskFileNames = []
plFileNames = []
for iSeq in range(len(obsSequences)):
obsSequence = obsSequences[iSeq]
obsSequence = obsSequence.strip().split()
obsFileNameTimestamps.append(obsSequence)
obsUtcDate = obsUtcDates[iSeq]
sunsetDate = str(int(obsUtcDate)-1)
obsSequence = [obsUtcDates[iSeq]+'-'+ts for ts in obsSequence]
obsFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).obs() for ts in obsSequence])
timeMaskFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).timeMask() for ts in obsSequence])
wvlCalTstamp = obsUtcDate+'-'+wvlCals[iSeq]
wvlFileNames.append(FileName(run=run,date=sunsetDate,tstamp=wvlCalTstamp).calSoln())
fluxFileNames.append(FileName(run=run,date=fluxCalDates[iSeq],tstamp=fluxCals[iSeq]).fluxSoln())
flatFileNames.append(FileName(run=run,date=flatCals[iSeq],tstamp='').flatSoln())
plFileNames.append([FileName(run=run,date=sunsetDate,tstamp=ts).timedPhotonList() for ts in obsSequence])
for iSeq,obsSequence in enumerate(obsSequences):
obsSequence = obsSequence.strip().split()
print obsSequence
for iOb,obs in enumerate(obsSequence):
timeMaskFileName = timeMaskFileNames[iSeq][iOb]
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obs
hp.findHotPixels(obsFileNames[iSeq][iOb],timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
apertureRadius = 5
obLists = [[ObsFile(fn)for fn in seq ] for seq in obsFileNames]
tstampFormat = '%H:%M:%S'
#print 'fileName','headerUnix','headerUTC','logUnix','packetReceivedUnixTime'
for iSeq,obList in enumerate(obLists):
for iOb,ob in enumerate(obList):
print ob.fileName
centerRow = centersRow[iSeq]
centerCol = centersCol[iSeq]
circCol,circRow = circ(centerCol,centerRow,radius=apertureRadius)
ob.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
ob.loadWvlCalFile(wvlFileNames[iSeq])
ob.loadFlatCalFile(flatFileNames[iSeq])
ob.loadFluxCalFile(fluxFileNames[iSeq])
timeMaskFileName = timeMaskFileNames[iSeq][iOb]
ob.loadHotPixCalFile(timeMaskFileName)
ob.setWvlCutoffs(4000,11000)
photlist.writePhotonList(ob,pixelList=zip(circRow,circCol),filename=plFileNames[iSeq][iOb])
print 'wrote crab list'
def hotPixelsTest2(startTime=12.3, endTime=23.1, getRawCount=True):
'''
Runs a check of the bad pixel time masking. Checks:
- Number of photons removed from returned image is consistent
with time masks.
- That timestamps of all photons in the time-masked image are outside
the time intervals defined for each pixel in the hot pixel file.
Outputs a test hot pixel file in the current directory, and runs checks
on it.
INPUTS:
startTime - time from beginning of obs file at which to start the check.
endTime - time from beginning to obs file at which to end (both in seconds).
getRawCounts - if True, use raw, non-wavelength calibrated photon counts
with no wavelength cutoffs applied.
'''
#dir = '/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/hotPixTest2/'
run = 'PAL2012'
date = '20121208'
obsFileName = FileName(run=run,date=date,tstamp='20121209-044636').obs() #'obs_20121209-044636.h5'
wvlCalFileName = FileName(run=run, date=date, tstamp='20121209-060704').calSoln() #'calsol_20121209-060704.h5'
flatCalFileName = FileName(run=run, date='20121210').flatSoln() #'flatsol_20121210.h5'
hotPixFileName = os.path.abspath('test-hotPix_20121209-044636.h5')
paramFile = os.path.join(os.path.dirname(__file__),'../../params/hotPixels.dict')
startTime = float(startTime) #Force these values to floats to make sure
endTime = float(endTime) #that getPixelCountImage calls getPixelSpectrum
#and applies wavelength cutoffs consistently.
if not os.path.exists(hotPixFileName):
print 'Creating hot pixel file....'
hp.findHotPixels(paramFile=paramFile, inputFileName=obsFileName,
outputFileName=hotPixFileName, timeStep=1,
startTime=0, endTime=-1,
fwhm=3.0, boxSize=5, nSigmaHot=2.5,
nSigmaCold=2.5, display=True)
print 'Done creating hot pixel file.'
print
intTime = endTime - startTime
obsFile = of.ObsFile(obsFileName)
obsFile.loadWvlCalFile(wvlCalFileName)
obsFile.loadFlatCalFile(flatCalFileName)
print 'Loading hot pixel file into obsFile...'
obsFile.loadHotPixCalFile(hotPixFileName)
obsFile.setWvlCutoffs()
print 'Getting image with masking...'
imhp = obsFile.getPixelCountImage(startTime, intTime, weighted=False,
getRawCount=getRawCount)['image']
print 'Getting image without masking...'
obsFile.switchOffHotPixTimeMask()
im = obsFile.getPixelCountImage(startTime, intTime, weighted=False,
getRawCount=getRawCount)['image']
diffim = im - imhp #Should end up containing the total number of photons masked from each pixel.
print 'Displaying images...'
mpl.ion()
mpl.matshow(imhp)
mpl.title('Hot-pixel masked')
mpl.colorbar()
mpl.matshow(im)
mpl.title('Unmasked')
mpl.colorbar()
print 'Loading local version of hot pixel file for direct inspection...'
hotPix = hp.readHotPixels(hotPixFileName)
if True:
print 'Checking for consistency in number of photons removed....'
for iRow in range(np.shape(diffim)[0]):
for iCol in range(np.shape(diffim)[1]):
nMaskedPhotons = 0
for eachInter in hotPix['intervals'][iRow, iCol]:
#Make sure there is only one component per interval
assert len(eachInter) == 1
#If the interval overlaps with the time range in question then
#add the number of photons in the interval to our running tally.
if eachInter[0][0] < endTime and eachInter[0][1] > startTime:
firstSec = max(eachInter[0][0], startTime)
lastSec = min(eachInter[0][1], endTime)
nMaskedPhotons += obsFile.getPixelCount(iRow, iCol, firstSec=firstSec,
integrationTime=lastSec - firstSec,
weighted=False, fluxWeighted=False,
getRawCount=getRawCount)['counts']
assert nMaskedPhotons == diffim[iRow, iCol]
print 'Okay.'
print 'Checking timestamps of remaining photons for consistency with exposure masks'
obsFile.switchOnHotPixTimeMask() #Switch on hot pixel masking
for iRow in range(np.shape(diffim)[0]):
for iCol in range(np.shape(diffim)[1]):
timeStamps = obsFile.getTimedPacketList(iRow, iCol, firstSec=startTime, integrationTime=intTime)['timestamps']
#timeStamps = timeStamps[np.logical_and(timeStamps<=endTime, timeStamps>=startTime)]
badInterval = interval.union(hotPix['intervals'][iRow, iCol])
#The following check would be nice, but doesn't work because getTimedPacketList doesn't do a wavelength cut like getPixelCount does.
#assert len(timeStamps) == im[iRow,iCol] #Double check that the number of photons returned matches the number in the masked image
#
for eachTimestamp in timeStamps:
assert eachTimestamp not in badInterval #Check that none of those photons' timestamps are in the masked time range
print 'Okay.'
print
print 'Done. All looks good.'
for iSeq in range(len(seqs)):
timestampList = timestampLists[iSeq]
print timestampList
wfn = wvlCalFilenames[iSeq]
ffn = flatCalFilenames[iSeq]
sunsetDate = sunsetDates[iSeq]
for i,ts in enumerate(timestampList):
print 'loading',ts
obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
ob = ObsFile(obsFn)
ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
index1 = obsFn.find('_')
hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
if not os.path.exists(hotPixFn):
hp.findHotPixels(obsFn,hotPixFn)
print "Flux file pixel mask saved to %s"%(hotPixFn)
ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
ob.loadWvlCalFile(wfn)
ob.loadFlatCalFile(ffn)
ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)
bad_solution_mask=np.zeros((46,44))
bad_count=0;
for y in range(46):
for x in range(44):
if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff):
bad_solution_mask[y][x] = 1
unix = ob.getFromHeader('unixtime')
startJD = unix/86400.+2440587.5
def quantifyBadTime(inputFileName, startTime=0, endTime=-1,
defaultTimeMaskFileName='./testTimeMask.h5',
timeStep=1, fwhm=3.0, boxSize=5, nSigmaHot=3.0,
nSigmaCold=2.5,maxIter=5,binWidth=3,
dispToPickle=False, showHist=False, bkgdPercentile=50,
weighted=False,fluxWeighted=False, useRawCounts=True):
'''
Function to calculate various metrics for the degree of bad pixel behaviour in a raw
raw obs file. Calculates the mean total hot/cold/dead time per good pixel (i.e., per
pixel which is not permanently dead, hot, or cold).
Makes a couple of heat maps showing time spent bad in each way for each pixel,
as well as a histogram of times spent bad for the *temporarily* bad pixels.
JvE Nov 20 2013.
The parameters for the finding algorithm may need to be tuned a little,
but the defaults should basically work, in principle. nSigmaHot and nSigmaCold
are good places to start if things need playing around with.
e.g. call, in principle:
from hotpix import quantifyHotTime as qht
qht.quantifyHotTime('/Users/vaneyken/Data/UCSB/ARCONS/turkDataCopy/ScienceData/PAL2012/20121208/obs_20121209-120530.h5')
- should be all it needs....
INPUTS:
inputFileName - either a raw obs. filename or a time mask file. If the former,
runs a hot pixel search; otherwise uses time mask file instead.
startTime, endTime - start and end times within the obs file to
calculate the hot pixels for. (endTime =-1 means to end of file).
defaultTimeMaskFileName - use this filename to output new time mask to
(if inputFileName is an obs file)
binWidth - width of time bins for plotting the bad-time histogram (seconds)
dispToPickle - if not False, saves the data for the histogram plot to a pickle file.
If a string, then uses that as the name for the pickle file. Otherwise
saves to a default name. Saves a dictionary with four entries, the first
three of which are each a flat array of total times spent bad for every
pixel (in sec) ('hotTime','coldTime','deadTime'). The fourth, 'duration',
is the duration of the input time-mask in seconds.
showHist - if True, plot up a histogram of everything. Currently fails though
if there are no bad-flagged intervals in any of the type categories
(or their intersections, hot+cold, cold+dead).
Parameters passed on to findHotPixels routine if called (see also documentation
for that function):
timeStep #Check for hot pixels every timeStep seconds
fwhm #Expected full width half max of PSF in pixels. Any pixel
#with flux much tighter than this will be flagged as bad.
#Larger value => more sensitive hot pixel flagging.
boxSize #Compare flux in each pixel with median flux in a
#surrounding box of this size on a side.
nSigmaHot #Require flux in a pixel to be > nSigmaHot std. deviations
#above the max expected for a Gaussian PSF in order for it
#to be flagged as hot. Larger value => less sensitive flagging.
nSigmaCold #Require flux to be < nSigmaCold std. deviations below the median
#in a surrounding box in order to be flagged as cold (where std.
#deviation is estimated as the square root of the median flux).
maxIter #Max num. of iterations for the bad pixel detection algorithm.
bkdgPercentile, weighted, fluxWeighted - See findHotPixels() in hotpix.hotpixels.py
OUTPUTS:
A bunch of statistics on the different kinds of bad pixel behaviour, and a
'heat' plot showing how much time each pixel was bad in the array, for each
type of behaviour. In theory it can plot a histogram of everything too, but
currently it breaks if there are no bad intervals within any of the type
categories (hot only, hot and cold, cold only, cold and dead, dead only...)
'''
defaultPklFileName = 'badPixTimeHist.pickle'
#Check whether the input file is a time mask or a regular obs file.
absInputFileName = os.path.abspath(inputFileName) #To avoid weird issues with the way findHotPixels expands paths....
hdffile = tables.openFile(absInputFileName)
inputIsTimeMaskFile = '/timeMasks' in hdffile
hdffile.close()
#Decide whether to generate a new time mask file or not
if inputIsTimeMaskFile:
print 'Input file is a time mask file'
timeMaskFileName = absInputFileName
else:
print 'Assuming input file is an obs. file'
timeMaskFileName = os.path.abspath(defaultTimeMaskFileName)
if os.path.exists(timeMaskFileName):
response=''
while response != 'u' and response !='r':
response = raw_input(timeMaskFileName+' already exists - "u" to use this (default) or "r" to regenerate? ')
response = response.strip().lower()
if response == '': response = 'u'
else:
response = 'r' #If the file *didn't already exist, pretend the user entered 'r' despite not having been asked.
if response == 'r':
#Make/regenerate the hot pixel file.
print 'Making new hot pixel time mask file '+timeMaskFileName
hp.findHotPixels(inputFileName=absInputFileName,
outputFileName=timeMaskFileName,
startTime=startTime,
endTime=endTime,
timeStep=timeStep, fwhm=fwhm,
boxSize=boxSize, nSigmaHot=nSigmaHot, nSigmaCold=nSigmaCold,
display=True, dispToPickle=dispToPickle,
maxIter=maxIter, bkgdPercentile=bkgdPercentile,
weighted=weighted,fluxWeighted=fluxWeighted,useRawCounts=useRawCounts)
#Read in the time mask file and calculate hot, cold, and 'other' bad time per pixel.
timeMask = hp.readHotPixels(timeMaskFileName)
hotTime = np.zeros((timeMask.nRow,timeMask.nCol))
coldTime = np.zeros((timeMask.nRow,timeMask.nCol))
deadTime = np.zeros((timeMask.nRow,timeMask.nCol))
otherTime = np.zeros((timeMask.nRow,timeMask.nCol))
hotIntervals = np.array([])
coldIntervals = np.array([])
deadIntervals = np.array([])
otherIntervals = np.array([])
reasonStringMap = timeMask.reasonEnum
for iRow in range(timeMask.nRow):
for iCol in range(timeMask.nCol):
for eachInterval,eachReasonCode in zip(timeMask.intervals[iRow,iCol], timeMask.reasons[iRow,iCol]):
eachReasonString = reasonStringMap(eachReasonCode) #Convert integer code to human readable string
intSize = utils.intervalSize(eachInterval)
if eachReasonString == 'hot pixel':
hotTime[iRow,iCol] += intSize
hotIntervals = np.append(hotIntervals, intSize)
elif eachReasonString == 'cold pixel':
coldTime[iRow,iCol] += intSize
coldIntervals = np.append(coldIntervals, intSize)
elif eachReasonString == 'dead pixel':
deadTime[iRow,iCol] += intSize
deadIntervals = np.append(deadIntervals, intSize)
else:
otherTime[iRow,iCol] += intSize
otherIntervals = np.append(otherIntervals, intSize)
if np.size(hotIntervals) == 0: hotIntervals = np.array([-1])
if np.size(coldIntervals) == 0: coldIntervals = np.array([-1])
if np.size(deadIntervals) == 0: deadIntervals = np.array([-1])
if np.size(otherIntervals) == 0: otherIntervals = np.array([-1])
totBadTime = hotTime+coldTime+deadTime+otherTime
maskDuration = timeMask.endTime-timeMask.startTime
#Figure out which pixels are hot, cold, permanently hot, temporarily cold, etc. etc.
nPix = timeMask.nRow * timeMask.nCol
hotPix = hotTime > 0.1
coldPix = coldTime > 0.1
deadPix = deadTime > 0.1
otherPix = otherTime > 0.1
multiBehaviourPix = ( (np.array(hotPix,dtype=int)+np.array(coldPix,dtype=int)
+np.array(deadPix,dtype=int)+np.array(otherPix,dtype=int)) > 1)
#assert np.all(deadTime[deadPix] == maskDuration) #All dead pixels should be permanently dead....
tol = timeStep/10. #Tolerance for the comparison operators below.
permHotPix = hotTime >= maskDuration-tol
permColdPix = coldTime >= maskDuration-tol
permDeadPix = deadTime >= maskDuration-tol
permOtherPix = otherTime >= maskDuration-tol
permGoodPix = (hotTime+coldTime+deadTime+otherTime < tol)
permBadPix = permHotPix | permColdPix | permDeadPix | permOtherPix
tempHotPix = (hotTime < maskDuration-tol) & (hotTime > tol)
tempColdPix = (coldTime < maskDuration-tol) & (coldTime > tol)
tempDeadPix = (deadTime < maskDuration-tol) & (deadTime > tol)
tempOtherPix = (otherTime < maskDuration-tol) & (otherTime > tol)
tempGoodPix = tempHotPix | tempColdPix | tempDeadPix | tempOtherPix #Bitwise or should work okay with boolean arrays
tempBadPix = tempGoodPix #Just to be explicit about it....
nGoodPix = np.sum(permGoodPix | tempGoodPix) #A 'good pixel' is either temporarily or permanently good
#assert np.sum(tempDeadPix) == 0 #Shouldn't be any temporarily dead pixels. APPARENTLY THERE ARE....
assert np.all((tempHotPix & permHotPix)==False) #A pixel can't be permanently AND temporarily hot
assert np.all((tempColdPix & permColdPix)==False) #... etc.
assert np.all((tempOtherPix & permOtherPix)==False)
assert np.all((tempDeadPix & permDeadPix)==False)
#Print out the results
print '----------------------------------------------'
print
print '# pixels total: ', nPix
print 'Mask duration (sec): ', maskDuration
print
print '% hot pixels: ', float(np.sum(permHotPix+tempHotPix))/nPix * 100.
print '% cold pixels: ', float(np.sum(permColdPix+tempColdPix))/nPix * 100.
print '% dead pixels: ', float(np.sum(permDeadPix+tempDeadPix))/nPix * 100.
print '% other pixels: ', float(np.sum(permOtherPix+tempOtherPix))/nPix * 100.
print
print '% permanently hot pixels: ', float(np.sum(permHotPix))/nPix * 100.
print '% permanently cold pixels: ', float(np.sum(permColdPix))/nPix * 100.
print '% permanently dead pixels: ', float(np.sum(permDeadPix))/nPix * 100.
print '% permanently "other" bad pixels: ', float(np.sum(permOtherPix))/nPix * 100.
print
print '% temporarily hot pixels: ', float(np.sum(tempHotPix))/nPix * 100.
print '% temporarily cold pixels: ', float(np.sum(tempColdPix))/nPix * 100.
print '% temporarily dead pixels: ', float(np.sum(tempDeadPix))/nPix * 100.
print '% temporarily "other" bad pixels: ', float(np.sum(tempOtherPix))/nPix * 100.
print
print '% pixels showing multiple bad behaviours: ', float(np.sum(multiBehaviourPix))/nPix * 100.
print
print '% permanently bad pixels: ', float(np.sum(permBadPix))/nPix * 100.
print '% temporarily bad pixels: ', float(np.sum(tempGoodPix))/nPix * 100. #Temp. good == temp. bad!
print '% permanently good pixels: ', float(np.sum(permGoodPix))/nPix * 100.
print
print 'Mean temp. hot pixel time per good pixel: ', np.sum(hotTime[tempHotPix])/nGoodPix
print 'Mean temp. hot pixel time per temp. hot pixel: ', np.sum(hotTime[tempHotPix])/np.sum(tempHotPix)
print
print 'Mean temp. cold pixel time per good pixel: ', np.sum(coldTime[tempColdPix])/nGoodPix
print 'Mean temp. cold pixel time per temp. cold pixel: ', np.sum(coldTime[tempColdPix])/np.sum(tempColdPix)
print
print 'Mean temp. "other" bad pixel time per good pixel: ', np.sum(otherTime[tempOtherPix])/nGoodPix
print 'Mean temp. "other" bad pixel time per temp. "other" pixel: ', np.sum(otherTime[tempOtherPix])/np.sum(tempOtherPix)
print
print '(All times in seconds)'
print
print 'Done.'
print
if np.sum(tempOtherPix) > 0 or np.sum(permOtherPix) > 0:
print '--------------------------------------------------------'
print 'WARNING: Pixels flagged for "other" reasons detected - '
print 'Histogram plot will not account for these!!'
print '--------------------------------------------------------'
#Display contour plots of the array of total bad times for each pixel for each type of behaviour
utils.plotArray(hotTime, plotTitle='Hot time per pixel (sec)', fignum=None, cbar=True)
utils.plotArray(coldTime, plotTitle='Cold time per pixel (sec)', fignum=None, cbar=True)
utils.plotArray(otherTime, plotTitle='Other bad time per pixel (sec)', fignum=None, cbar=True)
utils.plotArray(deadTime, plotTitle='Dead time per pixel (sec)', fignum=None, cbar=True)
#Make histogram of time spent 'bad' for the temporarily bad pixels.
#Ignore pixels flagged as bad for 'other' reasons (other than hot/cold/dead),
#of which there should be none at the moment.
assert np.all(otherPix == False)
#Find total bad time for pixels which go only one of hot, cold, or dead
onlyHotBadTime = totBadTime[hotPix & ~coldPix & ~deadPix]
onlyColdBadTime = totBadTime[~hotPix & coldPix & ~deadPix]
onlyDeadBadTime = totBadTime[~hotPix & ~coldPix & deadPix]
#Find total bad time for pixels which alternate between more than one bad state
hotAndColdBadTime = totBadTime[hotPix & coldPix & ~deadPix]
hotAndDeadBadTime = totBadTime[hotPix & ~coldPix & deadPix]
coldAndDeadBadTime = totBadTime[~hotPix & coldPix & deadPix]
hotAndColdAndDeadBadTime = totBadTime[hotPix & coldPix & deadPix]
allGoodBadTime = totBadTime[~hotPix & ~coldPix & ~deadPix]
assert np.sum(allGoodBadTime) == 0
if dispToPickle is not False:
#Save to pickle file to feed into a separate plotting script, primarily for
#the pipeline paper.
if type(dispToPickle) is str:
pklFileName = dispToPickle
else:
pklFileName = defaultPklFileName
pDict = {'hotTime':hotTime,
'coldTime':coldTime,
'deadTime':deadTime,
'onlyHotBadTime':onlyHotBadTime,
'onlyColdBadTime':onlyColdBadTime,
'onlyDeadBadTime':onlyDeadBadTime,
'hotAndColdBadTime':hotAndColdBadTime,
'hotAndDeadBadTime':hotAndDeadBadTime,
'coldAndDeadBadTime':coldAndDeadBadTime,
'hotAndColdAndDeadBadTime':hotAndColdAndDeadBadTime,
'maskDuration':maskDuration}
#pDict = {"hotTime":hotTime.ravel(),"coldTime":coldTime.ravel(),"deadTime":deadTime.ravel(),
# "duration":maskDuration}
print 'Saving to file: ',pklFileName
output = open(pklFileName, 'wb')
pickle.dump(pDict,output)
output.close()
assert np.size(hotTime)==nPix and np.size(coldTime)==nPix and np.size(deadTime)==nPix
assert (len(onlyHotBadTime)+len(onlyColdBadTime)+len(onlyDeadBadTime)+len(hotAndColdBadTime)
+len(coldAndDeadBadTime)+len(hotAndDeadBadTime)+len(hotAndColdAndDeadBadTime)
+len(allGoodBadTime))==nPix
if showHist is True:
#Be sure it's okay to leave hot+dead pixels out, and hot+cold+dead pixels.
#assert len(hotAndDeadBadTime)==0 and len(hotAndColdAndDeadBadTime)==0
mpl.figure()
norm = 1. #1./np.size(hotTime)*100.
dataList = [onlyHotBadTime,hotAndColdBadTime,onlyColdBadTime,coldAndDeadBadTime,onlyDeadBadTime]
dataList2 = [x if np.size(x)>0 else np.array([-1.0]) for x in dataList] #-1 is a dummy value for empty arrays, so that pyplot.hist doesn't barf.
weights = [np.ones_like(x)*norm if np.size(x)>0 else np.array([0]) for x in dataList]
mpl.hist(dataList2, range=None, #[-0.1,maskDuration+0.001], #Eliminate data at 0sec and maskDuration sec. (always good or permanently bad)
weights=weights,
label=['Hot only','Hot/cold','Cold only','Cold/dead','Dead only'], #,'Hot/dead','Hot/cold/dead'],
color=['red','pink','lightgray','lightblue','blue'], #,'green','black'],
bins=maskDuration/binWidth,histtype='stepfilled',stacked=True,log=False)
mpl.title('Duration of Bad Pixel Behaviour - '+os.path.basename(inputFileName))
mpl.xlabel('Total time "bad" (sec)')
mpl.ylabel('Percantage of pixels')
mpl.legend()
mpl.figure()
mpl.hist(hotIntervals, bins=maskDuration/binWidth)
print 'Median hot interval: ', np.median(hotIntervals)
mpl.xlabel('Duration of hot intervals')
mpl.ylabel('Number of intervals')
mpl.figure()
mpl.hist(coldIntervals, bins=maskDuration/binWidth)
print 'Median cold interval: ', np.median(coldIntervals)
mpl.xlabel('Duration of cold intervals')
mpl.ylabel('Number of intervals')
mpl.figure()
mpl.hist(deadIntervals, bins=maskDuration/binWidth)
print 'Median dead interval: ', np.median(deadIntervals)
mpl.xlabel('Duration of dead intervals')
mpl.ylabel('Number of intervals')
mpl.figure()
mpl.hist(otherIntervals, bins=maskDuration/binWidth)
print 'Median "other" interval: ', np.median(otherIntervals)
mpl.xlabel('Duration of "other" intervals')
mpl.ylabel('Number of intervals')
print 'Mask duration (s): ',maskDuration
print 'Number of pixels: ',nPix
print 'Fraction at 0s (hot,cold,dead): ', np.array([np.sum(hotTime<tol),np.sum(coldTime<tol),
np.sum(deadTime<tol)]) / float(nPix)
print 'Fraction at '+str(maskDuration)+'s (hot,cold,dead): ', np.array([np.sum(hotTime>maskDuration-tol),
np.sum(coldTime>maskDuration-tol),
np.sum(deadTime>maskDuration-tol)])/float(nPix)
def hotPixPlot():
if 1==1:
#File #1 - 2012, SDSS J0926
run = 'PAL2012'
date = '20121208'
tstamp='20121209-120530'
inputFile = fn.FileName(run=run,date=date,tstamp=tstamp).obs()
timeMaskFile = fn.FileName(run=run,date=date,tstamp=tstamp).timeMask()
timeAdjFileName = fn.FileName(run='PAL2012').timeAdjustments()
#wvlCalFileName = fn.FileName(run=run, date=date, tstamp='20121209-060704').calSoln() #'calsol_20121209-060704.h5'
#flatCalFileName = fn.FileName(run=run, date='20121210').flatSoln() #'flatsol_20121210.h5'
#fluxCalFileName = fn.FileName(run=run, date='20121211', tstamp='absolute_021727').fluxSoln()
obsFile = ObsFile.ObsFile(inputFile)
print inputFile
obsFile.loadTimeAdjustmentFile(timeAdjFileName)
print timeAdjFileName
obsFile.loadBestWvlCalFile()
print obsFile.wvlCalFileName
#print flatCalFileName
#obsFile.loadFlatCalFile(flatCalFileName)
#print fluxCalFileName
#obsFile.loadFluxCalFile(fluxCalFileName)
obsFile.setWvlCutoffs() #Use default wavelength cutoffs
hp.findHotPixels(obsFile=obsFile, outputFileName='badPix1.h5', startTime=54,
endTime=55, display=True, dispMinPerc=0, dispMaxPerc=98, maxIter=10,
nSigmaHot=4.0, bkgdPercentile=50, fwhm=2.5, diagnosticPlots=False,
boxSize=5, weighted=False,fluxWeighted=False, useRawCounts=True,
dispToPickle='hotPixData-'+tstamp+'.pkl')
mpl.title(obsFile.getFromHeader('target'))
#Get the stats for the same image
qbt.quantifyBadTime(inputFile,startTime=0,endTime=-1,maxIter=10,nSigmaHot=4.0,
bkgdPercentile=50,fwhm=2.0,boxSize=5,useRawCounts=True,
defaultTimeMaskFileName=timeMaskFile)
#File 2
run = 'PAL2014'
date = '20141021' #'20141022'
tstamp = '20141022-092629' #'20141022-032331' # '20141023-050637'
inputFile = fn.FileName(run=run,date=date,tstamp=tstamp).obs()
timeMaskFile = fn.FileName(run=run,date=date,tstamp=tstamp).timeMask()
#Time adjustment file not needed
#flatCalFileName = fn.FileName(run=run, date=date).flatSoln()
#Ignore flux cal for this one (shouldn't make much difference, and don't know if new one is set up yet)
#fluxCalFileName = fn.FileName(run=run, date=', tstamp='absolute_021727').fluxSoln()
obsFile = ObsFile.ObsFile(inputFile)
print inputFile
obsFile.loadBestWvlCalFile()
print obsFile.wvlCalFileName
#print flatCalFileName
#obsFile.loadFlatCalFile(flatCalFileName)
obsFile.setWvlCutoffs() #Use default wavelength cutoffs
hp.findHotPixels(obsFile=obsFile, outputFileName='badPix2.h5',
startTime=24, endTime=25, display=True, dispMinPerc=0, dispMaxPerc=99, maxIter=10,
nSigmaHot=4.0, bkgdPercentile=50, fwhm=2.0, diagnosticPlots=False,
boxSize=5, weighted=False,fluxWeighted=False, useRawCounts=True,
dispToPickle='hotPixData-'+tstamp+'.pkl')
mpl.title(obsFile.getFromHeader('target'))
#Get the stats for the same image
qbt.quantifyBadTime(inputFile,startTime=0,endTime=-1,maxIter=10,nSigmaHot=4.0,
bkgdPercentile=50,fwhm=2.0,boxSize=5,useRawCounts=True,
defaultTimeMaskFileName=timeMaskFile)
return obsFile
# plt.show()
startJD = ob.getFromHeader('jd')
nSecInFile = ob.getFromHeader('exptime')
deadMask = ob.getDeadPixels()
for sec in range(0,nSecInFile,integrationTime):
jd = startJD + sec/(24.*3600.)#add seconds offset to julian date
print count,jd
count+=1
times.append(jd)
titles.append('%.6f'%jd)
#make 30s flat-fielded,hot pixel masked image
#hotPixMaskRaw = hotPixels.findHotPixels(obsFile=ob,firstSec=sec,intTime=integrationTime)['badflag']
#hotPixMask = hotPixels.findHotPixels(obsFile=ob,firstSec=sec,intTime=integrationTime,weighted=True)['badflag']
frame = ob.getPixelCountImage(firstSec=sec,integrationTime=integrationTime,weighted=True)
hotPixMask = hotPixels.findHotPixels(image=frame,firstSec=sec,intTime=integrationTime,weighted=True)['badflag']
# plt.show()
showFrame = np.array(frame)
# utils.plotArray(showFrame,cbar=True,normMax=np.mean(showFrame)+3*np.std(showFrame))
# showFrame[hotPixMaskRaw == 1] = 0
# utils.plotArray(showFrame,cbar=True)
showFrame[hotPixMask == 1] = 0
# utils.plotArray(showFrame,cbar=True)
showframes.append(showFrame)
# frame[hotPixMaskRaw == 1] = np.nan
frame[hotPixMask == 1] = np.nan
frame[deadMask == 0] = np.nan
frames.append(frame)