def generateObsObjectList(obsFNs,wvlLowerLimit=3000, wvlUpperLimit=13000,beammapFileName = None,loadHotPix=True,loadWvlCal=True,loadFlatCal=True,loadSpectralCal=True):
obsFiles = []
for obsFN in obsFNs:
if type(obsFN) == type(''): #Full path to obs file
obs = ObsFile(obsFN)
else: #FileName object
obs = ObsFile(obsFN.obs())
if beammapFileName is not None and os.path.isfile(beammapFileName):
obs.loadBeammapFile(beammapFileName)
obs.setWvlCutoffs(wvlLowerLimit=wvlLowerLimit, wvlUpperLimit=wvlUpperLimit)
if loadHotPix:
if not os.path.isfile(FileName(obsFile=obs).timeMask()):
print "generating hp file ", FileName(obsFile=obs).timeMask()
hp.findHotPixels(obsFile=obs,outputFileName=FileName(obsFile=obs).timeMask())
obs.loadHotPixCalFile(FileName(obsFile=obs).timeMask(),reasons=['hot pixel','dead pixel'])
if loadWvlCal:
obs.loadBestWvlCalFile()
if loadFlatCal:
#obs.loadFlatCalFile(FileName(obsFile=obs).flatSoln())
obs.loadFlatCalFile('/Scratch/flatCalSolnFiles/flatsol_1s.h5')
if loadSpectralCal:
pass
obsFiles.append(obs)
return obsFiles
def main():
np.set_printoptions(threshold=np.nan)
testPixelRow = 24
testPixelCol = 17
#obs_20120919-131142.h5,obs_20120919-131346.h5
#create a cal file from a twilight flat
cal = FlatCal('../../params/flatCal.dict')
#open another twilight flat as an observation and apply a wavelength cal and the new flat cal
# run='LICK2012'
# obsFileName = FileName(run=run,date='20120918',tstamp='20120919-131142').flat()
# flatCalFileName = FileName(run=run,date='20120918',tstamp='20120919-131448').flatSoln()
# wvlCalFileName = FileName(run=run,date='20120916',tstamp='20120917-072537').calSoln()
run = 'PAL2012'
obsFileName = FileName(run=run,date='20121211',tstamp='20121212-140003').obs()
flatCalFileName = FileName(run=run,date='20121210',tstamp='').flatSoln()
wvlCalFileName = FileName(run=run,date='20121210',tstamp='20121211-133056').calSoln()
flatCalPath = os.path.dirname(flatCalFileName)
ob = ObsFile(obsFileName)#('obs_20120919-131142.h5')
ob.loadWvlCalFile(wvlCalFileName)#('calsol_20120917-072537.h5')
ob.loadFlatCalFile(flatCalFileName)#('flatsol_20120919-131142.h5')
#plot some uncalibrated and calibrated spectra for one pixel
fig = plt.figure()
ax = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
print ob.getPixelCount(testPixelRow,testPixelCol)
#flatSpectrum,wvlBinEdges = ob.getPixelSpectrum(testPixelRow,testPixelCol,weighted=False)
spectrum,wvlBinEdges = ob.getPixelSpectrum(testPixelRow,testPixelCol,wvlStart=cal.wvlStart,wvlStop=cal.wvlStop,wvlBinWidth=cal.wvlBinWidth,weighted=False,firstSec=0,integrationTime=-1)
weightedSpectrum,wvlBinEdges = ob.getPixelSpectrum(testPixelRow,testPixelCol,weighted=True)
#flatSpectrum,wvlBinEdges = cal.flatFile.getPixelSpectrum(testPixelRow,testPixelCol,wvlStart=cal.wvlStart,wvlStop=cal.wvlStop,wvlBinWidth=cal.wvlBinWidth,weighted=False,firstSec=0,integrationTime=-1)
flatSpectrum = cal.spectra[testPixelRow,testPixelCol]
x = wvlBinEdges[0:-1]
ax.plot(x,cal.wvlMedians,label='median spectrum',alpha=.5)
ax2.plot(x,cal.flatFactors[testPixelRow,testPixelCol,:],label='pixel weights',alpha=.5)
ax2.set_title('flat weights for pixel %d,%d'%(testPixelRow,testPixelCol))
ax.plot(x,spectrum+20,label='unweighted spectrum for pixel %d,%d'%(testPixelRow,testPixelCol),alpha=.5)
ax.plot(x,weightedSpectrum+10,label='weighted %d,%d'%(testPixelRow,testPixelCol),alpha=.5)
ax.plot(x,flatSpectrum+30,label='flatFile %d,%d'%(testPixelRow,testPixelCol),alpha=.5)
ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.3),fancybox=True,ncol=3)
plt.show()
#display a time-flattened image of the twilight flat as it is and after using itself as it's flat cal
#cal.flatFile.loadFlatCalFile(flatCalFileName)#('flatsol_20120919-131142.h5')
#cal.flatFile.displaySec(weighted=True,integrationTime=-1)
#ob.displaySec(integrationTime=-1)
#ob.displaySec(weighted=True,integrationTime=-1)
for idx in range(0,100,20):
factors10 = cal.flatFactors[:,:,idx]
plt.matshow(factors10,vmax=np.mean(factors10)+1.5*np.std(factors10))
plt.title('Flat weights at %d'%cal.wvlBinEdges[idx])
plt.colorbar()
plt.savefig('plots/factors%d.png'%idx)
plt.show()
def loadObs(self,stackLabel):
timestampList = self.params[stackLabel+'Sequence']
run = self.params['run']
sunsetDate = self.params[stackLabel+'SunsetDate']
utcDate = self.params[stackLabel+'UtcDate']
intTime = self.params[stackLabel+'IntTime']
wvlLowerCutoff = self.params[stackLabel+'WvlLowerCutoff']
wvlUpperCutoff = self.params[stackLabel+'WvlUpperCutoff']
calTimestamp = self.params[stackLabel+'WvlTimestamp']
print stackLabel,calTimestamp
wvlSolnFileName = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln()
wvlCalFileName = FileName(run=run,date=self.params[stackLabel+'WvlSunsetDate'],tstamp=calTimestamp).cal()
flatSolnFileName = FileName(run=run,date=self.params[stackLabel+'FlatCalSunsetDate'],tstamp=self.params[stackLabel+'FlatCalTimestamp']).flatSoln()
obsFileNames = [FileName(run=run,date=sunsetDate,tstamp=timestamp).obs() for timestamp in timestampList]
obList = [ObsFile(obsFn) for obsFn in obsFileNames]
for ob in obList:
ob.loadWvlCalFile(wvlSolnFileName)
ob.loadFlatCalFile(flatSolnFileName)
self.stackObsFileLists[stackLabel] = obList
cal = ObsFile(wvlCalFileName)
cal.loadWvlCalFile(wvlSolnFileName)
cal.loadFlatCalFile(flatSolnFileName)
self.stackWvlCals[stackLabel] = cal
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess,hotPath,flatPath):
obs = ObsFile(obsPath)
print obsPath,obs.getFromHeader('exptime'),obs
if not os.path.exists(hotPath):
hp.findHotPixels(obsFile=obs,outputFileName=hotPath)
obs.loadHotPixCalFile(hotPath,switchOnMask=False)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatPath)
obs.setWvlCutoffs(3000,8000)
cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath)
print 'done centroid',centroidPath
del obs
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=True):
'''
Test run of obsFile.writePhotonList. fileName can be used
to specify the output file name. If not specified, default
name/location is used.
Now includes test for pixel remapping....
'''
#Details of example obs file to run test on.
# run = 'PAL2012'
# date = '20121207'
# tstamp = '20121208-074649'
# calTstamp='20121208-070505'
# fluxTstamp='20121208-133002'
run = 'PAL2012'
date = '20121208'
tstamp = '20121209-120530'
centroidTstamp = '20121209-120530'
calTstamp='20121209-131132'
fluxTstamp='20121209-020416'
flatTstamp='20121209-021036'
if doPixRemap==True:
pixRemapFileName = FileName(run=run).pixRemap()
else:
pixRemapFileName = None
#Load up the obs file
obsFileName = FileName(run=run, date=date, tstamp=tstamp)
obsFile = ObsFile(obsFileName.obs())
#Load up associated calibrations
obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())
obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
#And write out the results....
obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
pixRemapFileName=pixRemapFileName)
from util.ObsFile import ObsFile from util.FileName import FileName run = "PAL2014" date = "20141022" timeStamp = '20141023-033821' fn = FileName(run,date,timeStamp) of = ObsFile(fn.obs()) of.loadBeammapFile(fn.beammap()) of.loadBestWvlCalFile() print "wvlCalFileName=",of.wvlCalFileName fn2 = FileName(run,date,"") of.loadFlatCalFile(fn2.flatSoln()) row = 4 col = 4 firstSec = 72 integrationTime = 1 spec = of.getPixelSpectrum(row,col,firstSec,integrationTime) print "spec=",spec del of
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) >3:
filenum = str('_'+sys.argv[3])
else:
filenum = '_0'
#science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
wvldir = params[2].split('=')[1].strip()
obsfile = params[3].split('=')[1].strip()
skyfile = params[4].split('=')[1].strip()
flatfile = params[5].split('=')[1].strip()
wvlfile = params[6].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
if len(params)>10:
xpix = params[10].split('=')[1].strip()
ypix = params[11].split('=')[1].strip()
apertureRadius = params[12].split('=')[1].strip()
#flux cal object parameter file
params2 = []
param2file = sys.argv[2]
f = open(param2file,'r')
for line in f:
params2.append(line)
f.close()
fluxdir = params2[7].split('=')[1].strip()
fluxfile = params2[8].split('=')[1].strip()
fluxCalObject = params2[9].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
print "obsfile = ",obsFileName
print "skyfile = ",skyFileName
print "wvlcal = ", wvlCalFileName
print "flatcal = ", flatCalFileName
print "fluxcal = ", fluxCalFileName
print "object = ", objectName
print "flux cal object = ", fluxCalObject
print "\n---------------------\n"
obs = ObsFile(obsFileName)
obs.loadWvlCalFile(wvlCalFileName)
obs.loadFlatCalFile(flatCalFileName)
obs.loadFluxCalFile(fluxCalFileName)
print "loaded data file and calibrations\n---------------------\n"
HotPixFile = getTimeMaskFileName(obsFileName)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFileName,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
wvlBinEdges,obsSpectra,obsIntTime = loadSpectra(obs,nCol,nRow)
nWvlBins=len(wvlBinEdges)-1
#divide spectrum by bin widths
binWidths = np.empty(nWvlBins)
for i in xrange(nWvlBins):
binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i]
obsSpectra = obsSpectra/binWidths
#divide by effective integration time
for iRow in xrange(nRow):
for iCol in xrange(nCol):
obsSpectra[iRow][iCol] = obsSpectra[iRow][iCol] / obsIntTime[iRow][iCol]
#print np.shape(obsSpectra)
#print nRow
#print nCol
#print nWvlBins
medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
print "target spectrum loaded\n---------------------\n"
if skyfile != "None":
sky = ObsFile(skyFileName)
sky.loadWvlCalFile(wvlCalFileName)
sky.loadFlatCalFile(flatCalFileName)
sky.loadFluxCalFile(fluxCalFileName)
skyTime = sky.getFromHeader("exptime")
skyHotPixFile = getTimeMaskFileName(skyFileName)
if not os.path.exists(skyHotPixFile):
hp.findHotPixels(skyFileName,skyHotPixFile)
print "Flux file pixel mask saved to %s"%(skyHotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(skyHotPixFile)
wvlBinEdges,skySpectra,skyIntTime = loadSpectra(sky,nCol,nRow)
skySpectra = skySpectra/binWidths
#divide by effective integration time
for iRow in xrange(nRow):
for iCol in xrange(nCol):
skySpectra[iRow][iCol] = skySpectra[iRow][iCol] / skyIntTime[iRow][iCol]
skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
#skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
print "sky spectrum loaded\n---------------------\n"
else:
#if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
skySpectrum = medianObsSpectrum
print "sky spectrum estimated as median of target file spectrum\n---------------------\n"
#subtract sky spectrum from every pixel
allSkySpectrum = obsSpectra-skySpectrum
#set any negative values to 0 after sky subtraction
allSkySpectrum[allSkySpectrum<0]=0
#take median of remaining sky subtracted spectra to get median object spectrum
finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
#load std spectrum for comparison
realSpectra = loadStd(objectName,wvlBinEdges)
print "real std spectrum loaded for reference\n---------------------\n"
#create plots
plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
pp = PdfPages(fullFluxPlotFileName)
matplotlib.rcParams['font.size']=6
#calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins),dtype=float)
for n in xrange(nWvlBins):
binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
wvls[n] = (wvlBinEdges[n]+(binsize/2.0))
plt.figure()
ax1 = plt.subplot(231)
ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
plt.plot(wvls,medianObsSpectrum)
#plt.show()
ax2 = plt.subplot(232)
ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
plt.plot(wvls,skySpectrum)
#plt.show()
ax5 = plt.subplot(233)
ax5.set_title('Sensitivity Spectrum')
plt.plot(wvls,obs.fluxWeights)
ax3 = plt.subplot(234)
ax3.set_title('MKID data minus sky in counts')
plt.plot(wvls,finalSpectrum/max(finalSpectrum))
ax4 = plt.subplot(235)
ax4.set_title('Rebinned Std Spectrum of %s'%(objectName))
plt.plot(wvls,realSpectra)
#ax7 = plt.subplot(337)
#ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
#plt.plot(wvls,fluxFactors*subtractedSpectra)
pp.savefig()
pp.close()
#del obs
#del sky
print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)
txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
fullFluxTxtFileName = os.path.join(txtDir,txtFileName)
outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
outarr[:,0]=wvls
outarr[:,1]=medianObsSpectrum
#save sensitivity spectrum to file
np.savetxt(fullFluxTxtFileName, outarr)
print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
tstamp = '20140925-112538'
obsFN = FileName(run=run,date=date,tstamp=tstamp)
obsPath = obsFN.obs()
flatPath = FileName(run=run,date=date).flatSoln()
hotPath = obsFN.timeMask()
centroidPath = obsFN.centroidList()
fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln()
obs = ObsFile(obsPath)
if not os.path.exists(hotPath):
hp.findHotPixels(obsPath,hotPath)
obs.loadHotPixCalFile(hotPath,switchOnMask=False)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatPath)
obs.loadFluxCalFile(fluxPath)
obs.loadCentroidListFile(centroidPath)
obs.setWvlCutoffs(3000,11000)
centroidRa = '03:37:43.826'
centroidDec = '14:15:14.828'
haOffset = 150.
# imgDict = obs.getPixelCountImage(integrationTime=60,scaleByEffInt=True)
# plotArray(imgDict['image'])
xGuess = 18 #col
yGuess = 15 #row
class DisplayStack(QMainWindow):
def __init__(self):
# 0) Start up gui
QWidget.__init__(self, parent=None)
self.ui = Ui_DisplayStack_gui()
self.ui.setupUi(self)
# Initialize Variables
self.initializeVariables()
# Initialize run and target, load param file
self.chooseRun()
'''
# Browse wavecal button
self.ui.wavelengthCalibrationSolutionButton.clicked.connect(self.chooseWavelengthFile)
# Browse flatcal button
self.ui.flatCalibrationSolutionButton.clicked.connect(self.chooseFlatFile)
'''
# Use wavelength calibration checkbox
self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal)
# Use flat calibration checkbox
self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal)
# Buttons for obs list creation
self.ui.addButton.clicked.connect(self.addObservation)
self.ui.removeButton.clicked.connect(self.removeObservation)
self.ui.clearButton.clicked.connect(self.createObsList)
# Start process button
self.ui.stackButton.clicked.connect(self.stackProcess)
# Load image stack button
self.ui.loadStackButton.clicked.connect(self.chooseStack)
# 1) Initialize Variables
def initializeVariables(self):
# Define path names
self.displayStackPath = '/Scratch/DisplayStack/'
self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/'
self.defaultFlatPath = '/Scratch/flatCalSolnFiles/'
# Arrays with run names and buttons
self.runNames = ['LICK2012','PAL2012','PAL2013']
self.runButtons = [self.ui.lick2012Button, self.ui.pal2012Button, self.ui.pal2013Button]
# Arrays with target names and buttons
self.lick2012TargetNames = ['LICK2012_Object']
self.lick2012TargetButtons = [self.ui.lick2012ObjectButton]
self.pal2012TargetNames = ['SDSS_J0651', 'SDSS_J0926']
self.pal2012TargetButtons = [self.ui.sdss0651Button,self.ui.sdss0926Button]
self.pal2013TargetNames = ['PAL2013_Object']
self.pal2013TargetButtons = [self.ui.pal2013ObjectButton]
self.targetNames = [self.lick2012TargetNames, self.pal2012TargetNames, self.pal2013TargetNames]
self.targetButtons = [self.lick2012TargetButtons, self.pal2012TargetButtons, self.pal2013TargetButtons]
# Activate run and target buttons
for iRun in range(len(self.runButtons)):
self.runButtons[iRun].clicked.connect(self.chooseRun)
for iTarget in range(len(self.targetButtons[iRun])):
self.targetButtons[iRun][iTarget].clicked.connect(self.chooseTarget)
self.ui.sunsetList.itemClicked.connect(self.createObsList)
def testFunction(self):
print 'test'
# 2) Choose Run
def chooseRun(self):
for iRun in range(len(self.runButtons)):
for iTarget in range(len(self.targetButtons[iRun])):
self.targetButtons[iRun][iTarget].setVisible(False)
if self.runButtons[iRun].isChecked():
self.runNumber = iRun
for iTarget in range(len(self.targetButtons[iRun])):
self.targetButtons[iRun][iTarget].setVisible(True)
self.run = self.runNames[self.runNumber]
self.targetNumber = 0
self.targetButtons[self.runNumber][self.targetNumber].setChecked(True)
self.target = self.targetNames[self.runNumber][self.targetNumber]
self.loadParamFile()
# 3) Choose Target
def chooseTarget(self):
for iTarget in range(len(self.targetButtons[self.runNumber])):
if self.targetButtons[self.runNumber][iTarget].isChecked():
self.targetNumber = iTarget
self.target = self.targetNames[self.runNumber][self.targetNumber]
self.loadParamFile()
# 4) Load Param File
def loadParamFile(self):
try:
self.paramData = readDict()
self.paramData.read_from_file(self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict')
self.obsTimes = np.array(self.paramData['obsTimes'])
self.utcDates = self.paramData['utcDates']
self.sunsetDates = self.paramData['sunsetDates']
self.calTimestamps = self.paramData['calTimestamps']
self.flatCalDates = self.paramData['flatCalDates']
print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.createSunsetList()
#self.createObsList()
self.createWavelengthList()
self.createFlatList()
self.paramFileExists = True
except IOError:
print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.ui.sunsetList.clear()
self.ui.obsList.clear()
self.ui.inputList.clear()
self.ui.wavelengthList.clear()
self.ui.flatList.clear()
self.paramFileExists = False
# 5) Choose Obs File
# Create list of available sunset dates
def createSunsetList(self):
self.ui.sunsetList.clear()
for iDate in range(len(self.sunsetDates)):
self.ui.sunsetList.addItem(self.sunsetDates[iDate])
# Create Initial Obs file list
def createObsList(self):
self.ui.obsList.clear()
self.ui.inputList.clear()
self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
for iObs in range(len(self.singleDayObservations)):
self.ui.obsList.addItem(self.singleDayObservations[iObs])
# Add observation to input list
def addObservation(self):
self.selectedObs = self.ui.obsList.currentItem()
self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs))
self.ui.inputList.addItem(self.selectedObs)
self.ui.inputList.sortItems()
# Remove observation from input list
def removeObservation(self):
self.removedObs = self.ui.inputList.currentItem()
self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs))
self.ui.obsList.addItem(self.removedObs)
self.ui.obsList.sortItems()
# 6) Load settings
def loadSettings(self):
self.validSettings = True
# Run and target information
self.run = self.runNames[self.runNumber]
self.target = self.targetNames[self.runNumber][self.targetNumber]
# General settings
self.integrationTime = int(self.ui.integrationTimeLine.text())
self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked()
self.useHotPixelMasking = self.ui.hotPixelBox.isChecked()
# Wavelength calibration settings
self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked()
self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text())
self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text())
# Flat calibration settings
self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked()
self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked()
self.fileCount = self.ui.inputList.count()
self.weighted = self.useFlatCalibration
self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration)
self.scaleByEffInt = self.useHotPixelMasking
if self.ui.sunsetList.currentItem() != None:
if self.fileCount == 0:
print 'Please select files to process...'
self.validSettings = False
if self.useWavelengthCalibration:
if self.ui.wavelengthList.currentItem() == None:
print 'Please select wavelength calibration...'
self.validSettings = False
else:
self.selectedWvlCal = self.ui.wavelengthList.currentItem().text()
self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln())
if self.useFlatCalibration:
if self.ui.flatList.currentItem() == None:
print 'Please select flat calibration...'
self.validSettings = False
else:
self.flatCalNight = self.ui.flatList.currentItem().text()
self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln())
else:
print 'Please select sunset night...'
self.validSettings = False
# 7) Load hot pixel mask
def loadHotMask(self):
self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5')
if not os.path.exists(self.hotPixelFilename):
hp.findHotPixels(self.obsFn,self.hotPixelFilename)
print "Hot pixel mask saved to %s"%(self.hotPixelFilename)
self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True)
# 8) Create wavelength cal file list
def createWavelengthList(self):
self.ui.wavelengthList.clear()
for iCal in range(len(self.calTimestamps)):
self.ui.wavelengthList.addItem(self.calTimestamps[iCal])
# Enable/disable wavecal options
def useWaveCal(self):
if self.ui.wavelengthCalibrationBox.isChecked():
self.ui.lowerWavelengthCutoffLine.setEnabled(True)
self.ui.lowerWavelengthCutoffLabel.setEnabled(True)
self.ui.upperWavelengthCutoffLine.setEnabled(True)
self.ui.upperWavelengthCutoffLabel.setEnabled(True)
self.ui.wavelengthList.setEnabled(True)
self.ui.flatCalibrationBox.setEnabled(True)
self.ui.flatCalibrationBox.setChecked(True)
self.ui.deadPixelBox.setEnabled(True)
self.ui.deadPixelBox.setChecked(True)
self.ui.flatList.setEnabled(True)
else:
self.ui.lowerWavelengthCutoffLine.setEnabled(False)
self.ui.lowerWavelengthCutoffLabel.setEnabled(False)
self.ui.upperWavelengthCutoffLine.setEnabled(False)
self.ui.upperWavelengthCutoffLabel.setEnabled(False)
self.ui.wavelengthList.setEnabled(False)
self.ui.flatCalibrationBox.setEnabled(False)
self.ui.flatCalibrationBox.setChecked(False)
self.ui.deadPixelBox.setChecked(False)
self.ui.deadPixelBox.setEnabled(False)
self.ui.flatList.setEnabled(False)
# 9) Create flat cal file list
def createFlatList(self):
self.ui.flatList.clear()
for iCal in range(len(self.flatCalDates)):
self.ui.flatList.addItem(self.flatCalDates[iCal])
# Enable/disable flatcal options
def useFlatCal(self):
if self.ui.flatCalibrationBox.isChecked():
self.ui.deadPixelBox.setEnabled(True)
self.ui.deadPixelBox.setChecked(True)
self.ui.flatList.setEnabled(True)
else:
self.ui.deadPixelBox.setChecked(False)
self.ui.deadPixelBox.setEnabled(False)
self.ui.flatList.setEnabled(False)
# 10) Load dead pixel mask
def loadDeadMask(self):
self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz')
if not os.path.exists(self.deadPixelFilename):
self.deadMask = self.ob.getDeadPixels()
np.savez(self.deadPixelFilename,deadMask = self.deadMask)
print "Dead pixel mask saved to %s"%(self.deadPixelFilename)
else:
self.deadFile = np.load(self.deadPixelFilename)
self.deadMask = self.deadFile['deadMask']
# 12) Create output name
def createOutputName(self):
self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's')
if self.useWavelengthCalibration and self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.npz')
elif self.useWavelengthCalibration and not self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.npz')
elif not self.waveLengthCalibration and self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_hp.npz')
else:
self.outputFilename = str(self.rawName + '.npz')
# Start process for creating image stacks
def stackProcess(self):
# Check for valid params file
if self.paramFileExists:
# 6) Load settings choosen from gui
self.loadSettings()
if self.validSettings:
# Loop through all files in input list
for iFile in range(self.fileCount):
# Create ObsFile instance
self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text()
self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs())
print 'Processing file ' + self.obsFn + '...'
self.ob = ObsFile(self.obsFn)
# Load time adjustment file
if self.useTimeAdjustment:
print 'Loading time adjustment file...'
self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments())
# 7) Load hot pixel mask
if self.useHotPixelMasking:
print 'Loading hot pixel mask...'
self.loadHotMask()
# 8) Load wave cal solution
if self.useWavelengthCalibration:
print 'Loading wavelength calibration...'
self.ob.loadWvlCalFile(self.wvlCalFilename)
# 9) Load flatcal solution
if self.useFlatCalibration:
print 'Loading flat calibration...'
self.ob.loadFlatCalFile(self.flatCalFilename)
# 10) Load dead pixel mask
if self.useDeadPixelMasking:
print 'Loading dead pixel mask...'
self.loadDeadMask()
# 11) Set wavelength cutoffs
if self.useWavelengthCalibration:
print 'Setting wavelength cutoffs...'
self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff)
# Timing
self.unix = self.ob.getFromHeader('unixtime')
self.startJD = self.unix/86400.+2440587.5
self.exptime = self.ob.getFromHeader('exptime')
self.times = []
self.frames = []
# 11) Create Image Stack
print 'Stacking images...'
for iSec in np.arange(0,self.exptime,self.integrationTime):
#add seconds offset to julian date, move jd to center of bin
self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.)
'Creating frame for time ' + self.jd
self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt)
self.frame = self.frameData['image']
if self.useDeadPixelMasking:
self.frame[self.deadMask == 0] = np.nan
self.frames.append(self.frame)
self.cube = np.dstack(self.frames)
self.times = np.array(self.times)
# 12) Create output file
self.createOutputName()
print 'Saving image stack to ' + self.outputFilename
np.savez(self.outputFilename, stack=self.cube, jd=self.times)
# Invalid params file
else:
print 'Invalid parameter file...'
# Choose an image stack
def chooseStack(self):
self.defaultLoadStackDirectory = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks')
self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("NPZ (*.npz)"))
if self.stackName == '':
print 'No file chosen'
else:
self.loadStack()
# Load that image stack for viewing
def loadStack(self):
hp.findHotPixels(inputFileName=obsFileName,outputFileName=timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs = ObsFile(obsFileName)
exptime = obs.getFromHeader('exptime')
#obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
if wvlCal != '':
obs.loadWvlCalFile(wvlFileName)
else:
obs.loadBestWvlCalFile()
obs.loadHotPixCalFile(timeMaskFileName)
obs.setWvlCutoffs(lowerWvlCut,upperWvlCut)
obs.loadFlatCalFile(flatFileName)
#get image before and after flat cal
#print 'getting images'
rawCubeDict = obs.getSpectralCube(weighted=False)
rawCube = np.array(rawCubeDict['cube'],dtype=np.double)
effIntTime = rawCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
rawCube *= maxIntTime / effIntTime3d
rawImg = np.sum(rawCube,axis=-1)
rawImg[rawImg == 0] = np.nan
rawImg[~np.isfinite(rawImg)] = np.nan
#print 'finished raw cube'
flatCubeDict = obs.getSpectralCube(weighted=True)
tic = time()
print 'Loading obs file and performing calibrations ...'
obsFn = FileName(run=run,date=sunsetDate,tstamp=obsTimestamp).obs()
ob = ObsFile(obsFn)
# Load roach time delay corrections.
ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
# Search for time mask for given observation file. If the time mask does not exist, create it.
index1 = obsFn.find('_')
hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
if not os.path.exists(hotPixFn):
hp.findHotPixels(obsFn,hotPixFn)
print "Flux file pixel mask saved to %s"%(hotPixFn)
# Load time mask, wavelength calibration, and flat calibration and set wavelenth cutoffs.
ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
ob.loadWvlCalFile(wvlCalFilename)
ob.loadFlatCalFile(flatCalFilename)
ob.setWvlCutoffs(3000,5000)
print 'Total load time: ' + str(time()-tic) + 's'
tic = time()
print 'Retrieving photon timestamps...'
# Define the area of an observation containing the object.
appMask = aperture(15,8,8)
yValues,xValues = np.where(appMask==0)
# Get a list of timestamps with photons from each pixel in the aperture.
timestamps=[]
for j in range(len(xValues)):
x=ob.getPixelWvlList(iRow=yValues[j],iCol=xValues[j])
timestamps = np.append(timestamps,x['timestamps'])
print 'Total retrieval time: ' + str(time()-tic) + 's'
class AppForm(QMainWindow):
def __init__(self, parent=None):
QMainWindow.__init__(self, parent)
self.setWindowTitle('Pixel Explorer')
paramFile = sys.argv[1]
self.params = readDict()
self.params.read_from_file(paramFile)
self.createMainFrame()
self.createStatusBar()
def createMainFrame(self):
self.main_frame = QWidget()
# Create the mpl Figure and FigCanvas objects.
self.dpi = 100
self.fig = Figure((7.0, 7.0), dpi=self.dpi)
self.canvas = FigureCanvas(self.fig)
self.canvas.setParent(self.main_frame)
self.axes0 = self.fig.add_subplot(111)
cid=self.canvas.mpl_connect('button_press_event', self.clickCanvas)
# Create the navigation toolbar, tied to the canvas
self.mpl_toolbar = NavigationToolbar(self.canvas, self.main_frame)
vbox = QVBoxLayout()
vbox.addWidget(self.canvas)
vbox.addWidget(self.mpl_toolbar)
self.main_frame.setLayout(vbox)
self.setCentralWidget(self.main_frame)
def createStatusBar(self):
self.status_text = QLabel("Awaiting orders.")
self.statusBar().addWidget(self.status_text, 1)
def openImage(self):
timestampList = [self.params['obsUtcDate']+'-'+ts for ts in self.params['obsSequence']]
run = self.params['run']
sunsetDate = self.params['obsSunsetDate']
utcDate = self.params['obsUtcDate']
self.intTime = self.params['intTime']
wvlLowerCutoff = self.params['wvlLowerCutoff']
wvlUpperCutoff = self.params['wvlUpperCutoff']
calTimestamp = self.params['wvlTimestamp']
wfn = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln()
calfn = FileName(run=run,date=self.params['wvlSunsetDate'],tstamp=calTimestamp).cal()
ffn = FileName(run=run,date=self.params['flatCalSunsetDate'],tstamp='').flatSoln()
obsFns = [FileName(run=run,date=sunsetDate,tstamp=timestamp).obs() for timestamp in timestampList]
self.obList = [ObsFile(obsFn) for obsFn in obsFns]
for ob in self.obList:
print 'Loading ',ob.fullFileName
ob.loadWvlCalFile(wfn)
ob.loadFlatCalFile(ffn)
self.cal = ObsFile(calfn)
self.cal.loadWvlCalFile(wfn)
self.cal.loadFlatCalFile(ffn)
self.loadSpectra()
def loadSpectra(self):
fileName = self.params['outFileName']
if os.path.exists(fileName):
data = np.load(fileName)
self.spectra = data['cube']
self.frame = data['frame']
self.frameValues = self.frame[~np.isnan(self.frame)]
self.wvlBinEdges = np.array(self.obList[0].flatCalWvlBins)
self.frameIntTime = 300*6
else:
self.spectra,self.wvlBinEdges = self.obList[0].getSpectralCube(weighted=True)
self.frameIntTime = 0
for ob in self.obList[1:]:
print ob.fileName
cube,wvlBinEdges = ob.getSpectralCube(weighted=True)
self.spectra += cube
self.frameIntTime += ob.getFromHeader('exptime')
self.spectra = np.array(self.spectra,dtype=np.float64)
self.frame = np.sum(self.spectra,axis=2)
hotPixMask = hotPixels.findHotPixels(image=self.frame,nsigma=2)['badflag']
self.frame[hotPixMask != 0] = np.nan
self.frame[self.frame == 0] = np.nan
self.frameValues = self.frame[~np.isnan(self.frame)]
np.savez(fileName,cube=self.spectra,frame=self.frame)
def plotWeightedImage(self):
self.showFrame = np.array(self.frame)
self.showFrame[np.isnan(self.frame)] = 0
handleMatshow = self.axes0.matshow(self.showFrame,cmap=matplotlib.cm.gnuplot2,origin='lower',vmax=np.mean(self.showFrame)+3*np.std(self.showFrame))
self.fig.colorbar(handleMatshow)
def clickCanvas(self,event):
self.showLaserSpectrum = True
self.showPixelSpectrum = True
self.showWvlLightCurves = True
self.showWvlLightCurveHists = False
self.showStdVsIntTime = True
self.showNormStdVsIntTime = True
col = round(event.xdata)
row = round(event.ydata)
if self.showPixelSpectrum:
#next plot the integrated spectrum for this pixel in the total image
spectrum = self.spectra[row,col]
print sum(spectrum),' counts in broadband spectrum'
def plotFunc(fig,axes):
axes.plot(self.wvlBinEdges[:-1],spectrum)
axes.set_xlabel(r'$\lambda$ ($\AA$)')
axes.set_ylabel(r'total counts')
popup = PopUp(parent=self,plotFunc=plotFunc,title='spectrum, pixel %d,%d (intTime=%d)'%(row,col,self.frameIntTime))
rebinSpecBins = 5
firstAfterConvolve = rebinSpecBins//2
rebinnedWvlEdges = self.wvlBinEdges[::rebinSpecBins]
if self.showLaserSpectrum:
#First plot the laser cal spectrum for this pixel to see if it's good
laserSpectrum,binEdges = self.cal.getPixelSpectrum(row,col,weighted=True)
def plotFunc(fig,axes):
axes.plot(binEdges[:-1],laserSpectrum)
axes.set_xlabel(r'$\lambda$ ($\AA$)')
axes.set_ylabel(r'total counts')
popup = PopUp(parent=self,plotFunc=plotFunc,title='Laser Cal Spectrum, pixel %d,%d'%(row,col))
if self.showWvlLightCurves:
spectrumInTime = []
for iOb,ob in enumerate(self.obList):
for sec in range(0,ob.getFromHeader('exptime'),self.intTime):
spectrum,binEdges = ob.getPixelSpectrum(pixelRow=row,pixelCol=col,firstSec=sec,integrationTime=self.intTime,weighted=True)
spectrum = np.convolve(spectrum,np.ones(rebinSpecBins),'same')[firstAfterConvolve::rebinSpecBins]
spectrumInTime.append(spectrum)
spectrumInTime = np.array(spectrumInTime)
nBins = np.shape(spectrumInTime)[1]
def plotFunc(fig,axes):
#plot counts vs time for each wavelength bin
t=np.arange(len(spectrumInTime[:,0]))*self.intTime
for iBin in xrange(nBins):
axes.plot(t,1.0*spectrumInTime[:,iBin]/self.intTime,
c=cm.jet((iBin+1.)/nBins),
label=r'%d-%d $\AA$'%(rebinnedWvlEdges[iBin],
rebinnedWvlEdges[iBin+1]))
axes.set_xlabel('time (s)')
axes.set_ylabel('cps')
#plot counts vs time summed over all wavelengths
axes.plot(t,np.sum(spectrumInTime,axis=1)/self.intTime,c='k',
label=r'%d-%d $\AA$'%(rebinnedWvlEdges[0],rebinnedWvlEdges[-1]))
#axes.legend(loc='center right')
popup = PopUp(parent=self,plotFunc=plotFunc,title='Light Curve by Band, Pixel %d,%d'%(row,col))
if self.showWvlLightCurveHists or self.showStdVsIntTime or self.showNormStdVsIntTime:
intTimes = [1,2,3,5,10,15,30]
spectrumVsIntTimeVsTime = []
for intTime in intTimes:
spectrumInTime = []
for iOb,ob in enumerate(self.obList):
for sec in range(0,ob.getFromHeader('exptime'),intTime):
spectrum,binEdges = ob.getPixelSpectrum(pixelRow=row,pixelCol=col,firstSec=sec,integrationTime=intTime,weighted=True)
spectrum = np.convolve(spectrum,np.ones(rebinSpecBins),'same')[firstAfterConvolve::rebinSpecBins]
spectrumInTime.append(spectrum)
spectrumInTime = np.array(spectrumInTime)
spectrumVsIntTimeVsTime.append(spectrumInTime)
#resulting array indexed as
#spectrumVsIntTimeVsTime[iIntTime][iTimeChunk][iWvlBin]
#sum over wavelength for total counts
countsVsIntTimeVsTime = [np.sum(spectrumInTime,axis=1) for spectrumInTime in spectrumVsIntTimeVsTime]
#countsVsIntTimeVsTime[iIntTime][iTimeChunk]
if self.showWvlLightCurveHists:
for iIntTime,countsVsTime in enumerate(countsVsIntTimeVsTime):
def plotFunc(fig,axes):
axes.hist(countsVsTime,bins=20)
axes.set_xlabel('counts per intTime %d s'%intTimes[iIntTime])
popup = PopUp(parent=self,plotFunc=plotFunc,title='Int Time %d s, pixel %d,%d'%(intTimes[iIntTime],row,col))
countStds = [np.std(countsVsTime) for countsVsTime in countsVsIntTimeVsTime]
countStds = np.array(countStds)
countSqrts = [np.sqrt(np.median(countsVsTime)) for countsVsTime in countsVsIntTimeVsTime]
countSqrts = np.array(countSqrts)
spectrumStds = [np.std(spectrumVsTime,axis=0) for spectrumVsTime in spectrumVsIntTimeVsTime]
spectrumSqrts = [np.sqrt(np.median(spectrumVsTime,axis=0)) for spectrumVsTime in spectrumVsIntTimeVsTime]
spectrumStds = np.array(spectrumStds)
spectrumSqrts = np.array(spectrumSqrts)
if self.showStdVsIntTime:
def plotFunc(fig,axes):
axes.set_xlabel('integration time (s)')
axes.plot(intTimes,countStds,'k',label=r'total $\sigma$')
axes.plot(intTimes,countSqrts,'k--',label=r'$\sqrt{med(N)}$')
nBins = np.shape(spectrumStds)[1]
for iBin in xrange(nBins):
axes.plot(intTimes,spectrumStds[:,iBin],
c=cm.jet((iBin+1.)/nBins),
label=r'%d-%d $\AA$ $\sigma$'%(rebinnedWvlEdges[iBin],
rebinnedWvlEdges[iBin+1]))
axes.plot(intTimes,spectrumSqrts[:,iBin],
c=cm.jet((iBin+1.)/nBins),linestyle='--')
axes.legend(loc='upper left')
popup = PopUp(parent=self,plotFunc=plotFunc,
title=r'$\sigma$ vs Integration Time, Pixel %d,%d'%(row,col))
if self.showNormStdVsIntTime:
def plotFunc(fig,axes):
axes.set_xlabel('integration time (s)')
axes.set_ylabel('normalized $\sigma$')
axes.plot(intTimes,countSqrts/np.max(countSqrts),'k--',
label=r'$\sqrt{N}$')
axes.plot(intTimes,countStds/np.max(countSqrts),'k',
label=r'%d-%d $\AA$'%(rebinnedWvlEdges[0],rebinnedWvlEdges[-1]))
nBins = np.shape(spectrumStds)[1]
for iBin in xrange(nBins):
axes.plot(intTimes,
spectrumStds[:,iBin]/np.max(spectrumSqrts[:,iBin]),
c=cm.jet((iBin+1.0)/nBins),
label=r'%d-%d $\AA$'%(rebinnedWvlEdges[iBin],
rebinnedWvlEdges[iBin+1]))
axes.legend(loc='upper left')
popup = PopUp(parent=self,plotFunc=plotFunc,
title='Normalized Standard Deviation vs IntegrationTime, Pixel %d,%d'%(row,col))
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) >3:
filenum = str('_'+sys.argv[3])
else:
filenum = '_0'
#science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
wvldir = params[2].split('=')[1].strip()
obsfile = params[3].split('=')[1].strip()
skyfile = params[4].split('=')[1].strip()
flatfile = params[5].split('=')[1].strip()
wvlfile = params[6].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
if len(params)>10:
xpix = int(params[10].split('=')[1].strip())
ypix = int(params[11].split('=')[1].strip())
apertureRadius = int(params[12].split('=')[1].strip())
#flux cal object parameter file
params2 = []
param2file = sys.argv[2]
f = open(param2file,'r')
for line in f:
params2.append(line)
f.close()
fluxdir = params2[7].split('=')[1].strip()
fluxfile = params2[8].split('=')[1].strip()
fluxCalObject = params2[9].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
print "obsfile = ",obsFileName
print "skyfile = ",skyFileName
print "wvlcal = ", wvlCalFileName
print "flatcal = ", flatCalFileName
print "fluxcal = ", fluxCalFileName
print "object = ", objectName
print "flux cal object = ", fluxCalObject
print "\n---------------------\n"
obs = ObsFile(obsFileName)
obs.loadWvlCalFile(wvlCalFileName)
obs.loadFlatCalFile(flatCalFileName)
obs.loadFluxCalFile(fluxCalFileName)
print "loaded data file and calibrations\n---------------------\n"
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
#wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
#nWvlBins=len(wvlBinEdges)-1
#print np.shape(obsSpectra)
#print nRow
#print nCol
#print nWvlBins
"""
medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
print "target spectrum loaded\n---------------------\n"
if skyfile != "None":
sky = ObsFile(skyFileName)
sky.loadWvlCalFile(wvlCalFileName)
sky.loadFlatCalFile(flatCalFileName)
sky.loadFluxCalFile(fluxCalFileName)
skyTime = sky.getFromHeader("exptime")
wvlBinEdges,skySpectra = loadSpectra(sky,nCol,nRow)
skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
print "sky spectrum loaded\n---------------------\n"
else:
#if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
skySpectrum = calculateMedian(obsSpectra, nCol, nRow, nWvlBins)
print "sky spectrum estimated as median of target file spectrum\n---------------------\n"
#subtract sky spectrum from every pixel
allSkySpectrum = obsSpectra-skySpectrum
#set any negative values to 0 after sky subtraction
allSkySpectrum[allSkySpectrum<0]=0
#take median of remaining sky subtracted spectra to get median object spectrum
finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
"""
#load/generate hot pixel mask file
HotPixFile = getTimeMaskFileName(obsFileName)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFileName,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
print "Making spectrum with Aperture Spectrum in ObsFile"
#use Aperture Spectrum from obsfile
medianObsSpectrum, wvlBinEdges = obs.getApertureSpectrum(pixelCol=ypix,pixelRow=xpix,radius1=apertureRadius, radius2 = apertureRadius*2.0,weighted=True, fluxWeighted=True, lowCut=3000, highCut=7000)
nWvlBins=len(wvlBinEdges)-1
#load std spectrum for comparison
try:
realSpectra = loadStd(objectName,wvlBinEdges)
print "real std spectrum loaded for reference\n---------------------\n"
stdTitle = "Rebinned Std Spectrum of %s"%(objectName)
except KeyError:
print "Key Error loading MKIDStd"
realSpectra = np.ones(nWvlBins)
stdTitle = "No MKIDStd spectrum available for %s"%(objectName)
#create plots
plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
pp = PdfPages(fullFluxPlotFileName)
matplotlib.rcParams['font.size']=6
#calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins),dtype=float)
for n in xrange(nWvlBins):
binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
wvls[n] = (wvlBinEdges[n]+(binsize/2.0))
plt.figure()
ax1 = plt.subplot(221)
ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
ax1.set_xlim((4000,11000))
ax1.set_ylim((min(medianObsSpectrum[(wvls>4000) & (wvls<11000)]),max(medianObsSpectrum[(wvls>4000) & (wvls<8000)])))
plt.plot(wvls,medianObsSpectrum)
#plt.show()
#ax2 = plt.subplot(232)
#ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
#plt.plot(wvls,skySpectrum)
#plt.show()
ax5 = plt.subplot(223)
ax5.set_title('Sensitivity Spectrum')
ax5.set_xlim((3000,13000))
ax5.set_ylim((0,5))
plt.plot(wvls,obs.fluxWeights)
#ax3 = plt.subplot(234)
#ax3.set_title('MKID data minus sky in counts')
#plt.plot(wvls,finalSpectrum/max(finalSpectrum))
ax4 = plt.subplot(222)
ax4.set_title(stdTitle)
plt.plot(wvls,realSpectra)
#ax7 = plt.subplot(337)
#ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
#plt.plot(wvls,fluxFactors*subtractedSpectra)
pp.savefig()
pp.close()
#del obs
#del sky
print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)
txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
fullFluxTxtFileName = os.path.join(txtDir,txtFileName)
outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
outarr[:,0]=wvls
outarr[:,1]=medianObsSpectrum
#save sensitivity spectrum to file
np.savetxt(fullFluxTxtFileName, outarr)
print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
class DisplayStack(QMainWindow):
def __init__(self):
# Start up gui
QWidget.__init__(self, parent=None)
self.ui = Ui_DisplayStack_gui()
self.ui.setupUi(self)
# Initialize Variables
self.initializeVariables()
# Lists and buttons used for specifying run and target information.
# Click item in runList to select a run. Run number corresponds to array index. Load up corresponding target list.
self.ui.runList.itemClicked.connect(self.selectRun)
# Click item in targetList to select a target.
self.ui.targetList.itemClicked.connect(self.selectTarget)
# Click button in
self.ui.targetButton.clicked.connect(self.loadTarget)
self.ui.sunsetList.itemClicked.connect(self.createObsList)
# Use wavelength calibration checkbox
self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal)
# Use flat calibration checkbox
self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal)
# Buttons for obs list creation
self.ui.addButton.clicked.connect(self.addObservation)
self.ui.removeButton.clicked.connect(self.removeObservation)
self.ui.clearButton.clicked.connect(self.createObsList)
# Start process button
self.ui.stackButton.clicked.connect(self.stackProcess)
# Load image stack button
self.ui.loadStackButton.clicked.connect(self.chooseStack)
# Initialize Variables
def initializeVariables(self):
# Define path names
self.displayStackPath = '/Scratch/DisplayStack/'
self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/PAL2012/master_cals/'
self.defaultFlatPath = '/Scratch/flatCalSolnFiles/'
# Load and display list of run names from /Scratch/DisplayStack/runList.dict
self.loadRunData()
# Load list of target names from /Scratch/DisplayStack/runName/runName.dict, for all runs
self.loadTargetData()
# Function to load list of run names. Runs on initialization.
def loadRunData(self):
# Load run data from /Scratch/DisplayStack/runList.dict
self.runData = readDict()
self.runData.read_from_file(self.displayStackPath + '/runList.dict')
self.runNames = np.array(self.runData['runs'])
# Populate runList table with run names.
for iRun in range(len(self.runNames)):
self.ui.runList.addItem(self.runNames[iRun])
# Function to load a table of target names.
def loadTargetData(self):
self.targetNames = []
# Cycle through runs and extract target name information from various dictionaries.
for iRun in range(len(self.runNames)):
self.targetData = readDict()
self.targetData.read_from_file(self.displayStackPath + self.runNames[iRun] + '/' + self.runNames[iRun] + '.dict')
self.iTargets = np.array(self.targetData['targets'])
self.targetNames.append(self.iTargets)
# Function to select a run and populate the target list for that particular run.
def selectRun(self):
# Clear list of target information for previously selected run.
self.ui.targetList.clear()
# Define a run number by the index of the selected run.
self.runNumber = self.ui.runList.row(self.ui.runList.currentItem())
# Populate targetList table with target names for selected run.
for iTarget in range(len(self.targetNames[self.runNumber])):
self.ui.targetList.addItem(self.targetNames[self.runNumber][iTarget])
def selectTarget(self):
self.targetNumber = self.ui.targetList.row(self.ui.targetList.currentItem())
def loadTarget(self):
self.run = self.runNames[self.runNumber]
self.target = self.targetNames[self.runNumber][self.targetNumber]
try:
self.paramName = self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.paramData = readDict()
self.paramData.read_from_file(self.paramName)
self.obsTimes = np.array(self.paramData['obsTimes'])
self.utcDates = self.paramData['utcDates']
self.sunsetDates = self.paramData['sunsetDates']
self.calTimestamps = self.paramData['calTimestamps']
self.flatCalDates = self.paramData['flatCalDates']
self.RA = self.paramData['RA']
self.Dec = self.paramData['Dec']
self.hourAngleOffset = self.paramData['HA_offset']
print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.createSunsetList()
#self.createObsList()
self.createWavelengthList()
self.createFlatList()
self.paramFileExists = True
except IOError:
print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.ui.sunsetList.clear()
self.ui.obsList.clear()
self.ui.inputList.clear()
self.ui.wavelengthList.clear()
self.ui.flatList.clear()
self.paramFileExists = False
# Choose Obs File
# Create list of available sunset dates
def createSunsetList(self):
self.ui.sunsetList.clear()
for iDate in range(len(self.sunsetDates)):
self.ui.sunsetList.addItem(self.sunsetDates[iDate])
# Create Initial Obs file list
def createObsList(self):
self.ui.obsList.clear()
self.ui.inputList.clear()
self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
for iObs in range(len(self.singleDayObservations)):
self.ui.obsList.addItem(self.singleDayObservations[iObs])
# Add observation to input list
def addObservation(self):
self.selectedObs = self.ui.obsList.currentItem()
self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs))
self.ui.inputList.addItem(self.selectedObs)
self.ui.inputList.sortItems()
# Remove observation from input list
def removeObservation(self):
self.removedObs = self.ui.inputList.currentItem()
self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs))
self.ui.obsList.addItem(self.removedObs)
self.ui.obsList.sortItems()
# Load settings
def loadSettings(self):
self.validSettings = True
# Run and target information
self.run = self.runNames[self.runNumber]
self.target = self.targetNames[self.runNumber][self.targetNumber]
# General settings
self.integrationTime = int(self.ui.integrationTimeLine.text())
self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked()
self.useHotPixelMasking = self.ui.hotPixelBox.isChecked()
# Wavelength calibration settings
self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked()
self.useBestWavelengthCalibration = self.ui.bestWavelengthCalibrationBox.isChecked()
self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text())
self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text())
# Flat calibration settings
self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked()
self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked()
self.fileCount = self.ui.inputList.count()
self.weighted = self.useFlatCalibration
self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration)
self.scaleByEffInt = self.useHotPixelMasking
if self.ui.sunsetList.currentItem() != None:
if self.fileCount == 0:
print 'Please select files to process...'
self.validSettings = False
if self.useWavelengthCalibration:
if self.useBestWavelengthCalibration:
print 'Using best wavelength calibration...'
elif self.ui.wavelengthList.currentItem() == None:
print 'Please select wavelength calibration...'
self.validSettings = False
else:
self.selectedWvlCal = self.ui.wavelengthList.currentItem().text()
print "Chose wvl cal: ", str(self.defaultWavelengthPath+self.selectedWvlCal)
self.wvlCalFilename = str(self.defaultWavelengthPath+self.selectedWvlCal)
#self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln())
if self.useFlatCalibration:
if self.ui.flatList.currentItem() == None:
print 'Please select flat calibration...'
self.validSettings = False
else:
self.flatCalNight = self.ui.flatList.currentItem().text()
self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln())
else:
print 'Please select sunset night...'
self.validSettings = False
# Load hot pixel mask
def loadHotMask(self):
#self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5')
self.hotPixelFilename = str(FileName(obsFile = self.ob).timeMask())
if not os.path.exists(self.hotPixelFilename):
hp.findHotPixels(obsFile=self.ob,outputFileName=self.hotPixelFilename)
print "Hot pixel mask saved to %s"%(self.hotPixelFilename)
self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True)
# Create wavelength cal file list
def createWavelengthList(self):
self.ui.wavelengthList.clear()
for iCal in range(len(self.calTimestamps)):
self.ui.wavelengthList.addItem(self.calTimestamps[iCal])
# Enable/disable wavecal options
def useWaveCal(self):
if self.ui.wavelengthCalibrationBox.isChecked():
self.ui.lowerWavelengthCutoffLine.setEnabled(True)
self.ui.lowerWavelengthCutoffLabel.setEnabled(True)
self.ui.upperWavelengthCutoffLine.setEnabled(True)
self.ui.upperWavelengthCutoffLabel.setEnabled(True)
self.ui.wavelengthList.setEnabled(True)
self.ui.flatCalibrationBox.setEnabled(True)
self.ui.flatCalibrationBox.setChecked(True)
self.ui.deadPixelBox.setEnabled(True)
self.ui.deadPixelBox.setChecked(True)
self.ui.flatList.setEnabled(True)
self.ui.bestWavelengthCalibrationBox.setEnabled(True)
self.ui.bestWavelengthCalibrationBox.setChecked(True)
else:
self.ui.lowerWavelengthCutoffLine.setEnabled(False)
self.ui.lowerWavelengthCutoffLabel.setEnabled(False)
self.ui.upperWavelengthCutoffLine.setEnabled(False)
self.ui.upperWavelengthCutoffLabel.setEnabled(False)
self.ui.wavelengthList.setEnabled(False)
self.ui.flatCalibrationBox.setEnabled(False)
self.ui.flatCalibrationBox.setChecked(False)
self.ui.deadPixelBox.setChecked(False)
self.ui.deadPixelBox.setEnabled(False)
self.ui.flatList.setEnabled(False)
self.ui.bestWavelengthCalibrationBox.setEnabled(False)
self.ui.bestWavelengthCalibrationBox.setChecked(False)
# Create flat cal file list
def createFlatList(self):
self.ui.flatList.clear()
for iCal in range(len(self.flatCalDates)):
self.ui.flatList.addItem(self.flatCalDates[iCal])
# Enable/disable flatcal options
def useFlatCal(self):
if self.ui.flatCalibrationBox.isChecked():
self.ui.deadPixelBox.setEnabled(True)
self.ui.deadPixelBox.setChecked(True)
self.ui.flatList.setEnabled(True)
else:
self.ui.deadPixelBox.setChecked(False)
self.ui.deadPixelBox.setEnabled(False)
self.ui.flatList.setEnabled(False)
# Load dead pixel mask
def loadDeadMask(self):
self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz')
if not os.path.exists(self.deadPixelFilename):
self.deadMask = self.ob.getDeadPixels()
np.savez(self.deadPixelFilename,deadMask = self.deadMask)
print "Dead pixel mask saved to %s"%(self.deadPixelFilename)
else:
self.deadFile = np.load(self.deadPixelFilename)
self.deadMask = self.deadFile['deadMask']
# Describe the structure of the header row
class headerDescription(tables.IsDescription):
targetName = tables.StringCol(100, dflt='')
run = tables.StringCol(100, dflt='')
obsFileName = tables.StringCol(100, dflt='')
wvlCalFileName = tables.StringCol(100, dflt=np.nan)
flatCalFileName = tables.StringCol(100, dflt='')
deadPixFileName = tables.StringCol(100, dflt='')
hotPixFileName = tables.StringCol(100, dflt='')
nCol = tables.UInt32Col(dflt=-1)
nRow = tables.UInt32Col(dflt=-1)
lowWvlCutoff = tables.Float64Col(dflt=np.nan)
highWvlCutoff = tables.Float64Col(dflt=np.nan)
exptime = tables.Float64Col(dflt=np.nan)
lst = tables.StringCol(100, dflt='')
integrationTime = tables.Float64Col(dflt=np.nan)
RA = tables.StringCol(100, dflt='')
Dec = tables.StringCol(100, dflt='')
HA_offset = tables.Float64Col(dflt=0.0)
# Create output name
def createOutputName(self):
self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's')
if self.useWavelengthCalibration and self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.h5')
elif self.useWavelengthCalibration and not self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.h5')
elif not self.useWavelengthCalibration and self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_hp.h5')
else:
self.outputFilename = str(self.rawName + '.h5')
def createH5File(self):
# Create header and data group and table names
headerGroupName = 'header'
headerTableName = 'header'
dataGroupName = 'stack'
dataTableName = 'stack'
timeTableName = 'time'
# Create lookup names for header information
runColName = 'run'
targetColName = 'targetName'
obsFileColName = 'obsFileName'
wvlCalFileColName = 'wvlCalFileName'
flatCalFileColName = 'flatCalFileName'
nRowColName = 'nRow'
nColColName = 'nCol'
RAColName = 'RA'
DecColName = 'Dec'
deadPixColName = 'deadPixFileName'
hotPixColName = 'hotPixFileName'
lowWvlColName = 'lowWvlCutoff'
highWvlColName = 'highWvlCutoff'
expTimeColName = 'exptime'
lstColName = 'lst'
integrationTimeColName = 'integrationTime'
HA_offsetColName = 'HA_offset'
# Create and h5 output file, create header and data groups
fileh = tables.openFile(self.outputFilename, mode='w')
headerGroup = fileh.createGroup("/", headerGroupName, 'Header')
stackGroup = fileh.createGroup("/", dataGroupName, 'Image Stack')
# Create row for header information
headerTable = fileh.createTable(headerGroup, headerTableName, self.headerDescription,
'Header Info')
header = headerTable.row
# Fill in the header with possibly useful information.
header[runColName] = self.run
header[targetColName] = self.target
header[obsFileColName] = self.obsFn
header[nColColName] = self.numberCols
header[nRowColName] = self.numberRows
header[RAColName] = self.RA
header[DecColName] = self.Dec
header[expTimeColName] = self.exptime
header[lstColName] = self.lst
header[integrationTimeColName] = self.integrationTime
header[HA_offsetColName] = self.hourAngleOffset
if self.useDeadPixelMasking:
header[deadPixColName] = self.deadPixelFilename
if self.useHotPixelMasking:
header[hotPixColName] = self.hotPixelFilename
if self.useWavelengthCalibration:
header[wvlCalFileColName] = self.ob.wvlCalFileName
header[lowWvlColName] = self.lowerWavelengthCutoff
header[highWvlColName] = self.upperWavelengthCutoff
if self.useFlatCalibration:
header[flatCalFileColName] = self.flatCalFilename
header.append()
# Create an h5 array for the midtime of each frame in the image cube.
timeTable = fileh.createCArray(stackGroup, timeTableName, Float64Atom(), (1,len(self.times)))
timeTable[:] = self.times
# Create an h5 table for the image cube.
stackTable = fileh.createCArray(stackGroup, dataTableName, Float64Atom(), (self.numberRows,self.numberCols, self.cube.shape[2]))
stackTable[:] = self.cube
# Flush the h5 output file
fileh.flush()
fileh.close()
# Start process for creating image stacks
def stackProcess(self):
# Check for valid params file
if self.paramFileExists:
# Load settings choosen from gui
self.loadSettings()
if self.validSettings:
# Loop through all files in input list
for iFile in range(self.fileCount):
# Create ObsFile instance
self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text()
self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs())
print 'Processing file ' + self.obsFn + '...'
self.ob = ObsFile(self.obsFn)
self.numberRows = self.ob.nRow
self.numberCols = self.ob.nCol
# Load time adjustment file
if self.useTimeAdjustment:
print 'Loading time adjustment file...'
self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments())
# Load hot pixel mask
if self.useHotPixelMasking:
print 'Loading hot pixel mask...'
self.loadHotMask()
# Load wave cal solution
if self.useWavelengthCalibration:
if self.useBestWavelengthCalibration:
print 'Loading best wavelength calibration...'
self.ob.loadBestWvlCalFile()
else:
print 'Loading selected wavelength calibration...'
self.ob.loadWvlCalFile(self.wvlCalFilename)
# Load flatcal solution
if self.useFlatCalibration:
print 'Loading flat calibration...'
self.ob.loadFlatCalFile(self.flatCalFilename)
# Load dead pixel mask
if self.useDeadPixelMasking:
print 'Loading dead pixel mask...'
self.loadDeadMask()
# Set wavelength cutoffs
if self.useWavelengthCalibration:
print 'Setting wavelength cutoffs...'
self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff)
# Timing
self.unix = self.ob.getFromHeader('unixtime')
self.startJD = self.unix/86400.+2440587.5
self.exptime = self.ob.getFromHeader('exptime')
self.lst = self.ob.getFromHeader('lst')
self.times = []
self.frames = []
# Create Image Stack
print 'Stacking images...'
for iSec in np.arange(0,self.exptime,self.integrationTime):
#add seconds offset to julian date, move jd to center of bin
self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.)
self.times.append(self.jd)
print 'Creating frame for time ' + str(self.jd)
self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt)
self.frame = self.frameData['image']
if self.ui.verticalFlipBox.isChecked():
self.frame = np.flipud(self.frame)
if self.useDeadPixelMasking:
self.frame[self.deadMask == 0] = np.nan
self.frames.append(self.frame)
self.cube = np.dstack(self.frames)
self.times = np.array(self.times)
# Create output file
self.createOutputName()
print 'Saving image stack to ' + self.outputFilename
self.createH5File()
# Invalid params file
else:
print 'Invalid parameter file...'
# Choose an image stack
def chooseStack(self):
self.defaultLoadStackDirectory = str(self.displayStackPath)
self.stackName = ''
self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("H5 (*.h5)"))
if self.stackName == '':
print 'No file chosen'
else:
loadStackApp = LoadImageStack.LoadImageStack(stackName = self.stackName)
loadStackApp.show()
loadStackApp.exec_()
def main():
#open the sky file for hr9087
run = 'PAL2012'
date = '20121210'
wvlCal = '20121211-052230'
obsTimestamp = '20121211-051650'
flatCalDate = '20121211'
flatCalTstamp = '20121212-074700'
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
timeMaskFileName = obsFN.timeMask()
wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obsFileName
hp.findHotPixels(obsFileName,timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs = ObsFile(obsFileName)
obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
obs.loadWvlCalFile(wvlFileName)
obs.loadFlatCalFile(flatFileName)
obs.loadHotPixCalFile(timeMaskFileName)
#obs.setWvlCutoffs(4000,8000)
#get image before and after flat cal
print 'getting images'
beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True)
rawCubeDict = obs.getSpectralCube(weighted=False)
rawCube = np.array(rawCubeDict['cube'],dtype=np.double)
effIntTime = rawCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
rawCube *= maxIntTime / effIntTime3d
rawCube[np.isnan(rawCube)] = 0
rawCube[rawCube == np.inf] = 0
beforeImg = np.sum(rawCube,axis=-1)
print 'finished first cube'
flatCubeDict = obs.getSpectralCube(weighted=True)
flatCube = np.array(flatCubeDict['cube'],dtype=np.double)
effIntTime = flatCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
flatCube *= maxIntTime / effIntTime3d
flatCube[np.isnan(flatCube)] = 0
flatCube[flatCube == np.inf] = 0
afterImg = np.sum(flatCube,axis=-1)
plotArray(title='before flatcal',image=beforeImg)
plotArray(title='after flatcal',image=afterImg)
print 'before sdev',np.std(beforeImg[afterImg!=0])
print 'after sdev',np.std(afterImg[afterImg!=0])
np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) > 2:
fileNum = str(sys.argv[2])
else:
fileNum = "0"
# science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile, "r")
for line in f:
params.append(line)
f.close()
datadir = params[0].split("=")[1].strip()
flatdir = params[1].split("=")[1].strip()
wvldir = params[2].split("=")[1].strip()
obsfile = params[3].split("=")[1].strip()
skyfile = params[4].split("=")[1].strip()
flatfile = params[5].split("=")[1].strip()
wvlfile = params[6].split("=")[1].strip()
objectName = params[9].split("=")[1].strip()
# wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121205"
# objectName = "crabNight1"
if len(params) > 10:
xpix = int(params[10].split("=")[1].strip())
ypix = int(params[11].split("=")[1].strip())
apertureRadius = int(params[12].split("=")[1].strip())
startTime = int(params[13].split("=")[1].strip())
intTime = int(params[14].split("=")[1].strip())
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
obs = ObsFile(obsFileName)
# obs.loadWvlCalFile(wvlCalFileName)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatCalFileName)
print "analyzing file %s" % (obsFileName)
print "loaded data file and calibrations\n---------------------\n"
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
# wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
# nWvlBins=len(wvlBinEdges)-1
# print np.shape(obsSpectra)
# print nRow
# print nCol
# print nWvlBins
# Apply Hot pixel masking before getting dead time correction
# HotPixFile = getTimeMaskFileName(obsFileName)
HotPixFile = FileName(obsFile=obs).timeMask()
print "making hot pixel file ", HotPixFile
if not os.path.exists(HotPixFile): # check if hot pix file already exists
hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
print "Flux file pixel mask saved to %s" % (HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s" % (HotPixFile)
# GET RAW PIXEL COUNT IMAGE TO CALCULATE CORRECTION FACTORS
print "Making raw cube to get dead time correction"
cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=False)
cube = np.array(cubeDict["cube"], dtype=np.double)
wvlBinEdges = cubeDict["wvlBinEdges"]
effIntTime = cubeDict["effIntTime"]
print "median effective integration time = ", np.median(effIntTime)
nWvlBins = len(wvlBinEdges) - 1
print "cube shape ", np.shape(cube)
print "effIntTime shape ", np.shape(effIntTime)
# add third dimension to effIntTime for broadcasting
effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
# put cube into counts/s in each pixel
cube /= effIntTime
# CALCULATE DEADTIME CORRECTION
# NEED TOTAL COUNTS PER SECOND FOR EACH PIXEL TO DO PROPERLY
# ASSUMES SAME CORRECTION FACTOR APPLIED FOR EACH WAVELENGTH, MEANING NO WL DEPENDANCE ON DEAD TIME EFFECT
DTCorr = np.zeros((np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
for f in range(0, np.shape(cube)[2]):
print cube[:, :, f]
print "-----------------------"
DTCorr += cube[:, :, f]
print DTCorr
print "\n=====================\n"
# Correct for 100 us dead time
DTCorrNew = DTCorr / (1 - DTCorr * 100e-6)
CorrFactors = DTCorrNew / DTCorr # This is what the frames need to be multiplied by to get their true values
print "Dead time correction factors = "
print CorrFactors
# REMAKE CUBE WITH FLAT WEIGHTS AND APPLY DEAD TIME CORRECTION AS WELL
print "Making Weighted cube"
# load/generate hot pixel mask file
# HotPixFile = getTimeMaskFileName(obsFileName)
HotPixFile = FileName(obsFile=obs).timeMask()
if not os.path.exists(HotPixFile): # check if hot pix file already exists
hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
print "Flux file pixel mask saved to %s" % (HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s" % (HotPixFile)
cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=False)
# cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=True)
cube = np.array(cubeDict["cube"], dtype=np.double)
wvlBinEdges = cubeDict["wvlBinEdges"]
effIntTime = cubeDict["effIntTime"]
print "median effective integration time = ", np.median(effIntTime)
nWvlBins = len(wvlBinEdges) - 1
print "cube shape ", np.shape(cube)
print "effIntTime shape ", np.shape(effIntTime)
# add third dimension to effIntTime for broadcasting
effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
# put cube into counts/s in each pixel
cube /= effIntTime
# add third dimension to CorrFactors for broadcasting
CorrFactors = np.reshape(CorrFactors, np.shape(CorrFactors) + (1,))
# apply dead time correction factors
cube *= CorrFactors
# calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins), dtype=float)
for n in xrange(nWvlBins):
binsize = wvlBinEdges[n + 1] - wvlBinEdges[n]
wvls[n] = wvlBinEdges[n] + (binsize / 2.0)
print "wvls ", wvls
# reshape cube for makeMovie
movieCube = np.zeros((nWvlBins, np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
for i in xrange(nWvlBins):
movieCube[i, :, :] = cube[:, :, i]
# show individual frames as they are made to debug
# plt.matshow(movieCube[i],vmin = 0, vmax = 100)
# plt.show()
print "movieCube shape ", np.shape(movieCube)
print "wvls shape ", np.shape(wvls)
# print cube
# print "--------------------------"
# print movieCube
outdir = "/home/srmeeker/scratch/standards/"
np.savez(outdir + "%s_raw_%s.npz" % (objectName, fileNum), stack=movieCube, wvls=wvls)
utils.makeMovie(
movieCube,
frameTitles=wvls,
cbar=True,
outName=outdir + "%s_raw_%s.gif" % (objectName, fileNum),
normMin=0,
normMax=50,
)
"""
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) >2:
fileNum = str(sys.argv[2])
else:
fileNum = '0'
#science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
wvldir = params[2].split('=')[1].strip()
obsfile = params[3].split('=')[1].strip()
skyfile = params[4].split('=')[1].strip()
flatfile = params[5].split('=')[1].strip()
wvlfile = params[6].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121206"
if len(params)>10:
xpix = int(params[10].split('=')[1].strip())
ypix = int(params[11].split('=')[1].strip())
apertureRadius = int(params[12].split('=')[1].strip())
startTime = int(params[13].split('=')[1].strip())
intTime =int(params[14].split('=')[1].strip())
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
obs = ObsFile(obsFileName)
obs.loadWvlCalFile(wvlCalFileName)
obs.loadFlatCalFile(flatCalFileName)
print "analyzing file %s"%(obsFileName)
print "loaded data file and calibrations\n---------------------\n"
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
#wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
#nWvlBins=len(wvlBinEdges)-1
#print np.shape(obsSpectra)
#print nRow
#print nCol
#print nWvlBins
#load/generate hot pixel mask file
HotPixFile = getTimeMaskFileName(obsFileName)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFileName,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
#######
#EVERYTHING BEFORE HERE IS STANDARD FILE/CALFILE LOADING
startWvl = 3000
#stopWvl = 7000 #for V-band
stopWvl = 9000 #for R-band
print "Making spectral cube"
#for pg0220 first sec should be 80 since object is moving around before this
#for pg0220A first sec should be 70, integration time is 140
#for landolt 9542 first sec should be 20, int time is -1
cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, wvlStart = startWvl, wvlStop = stopWvl, wvlBinEdges = [startWvl,stopWvl], weighted=False)
cube= np.array(cubeDict['cube'], dtype=np.double)
wvlBinEdges= cubeDict['wvlBinEdges']
effIntTime= cubeDict['effIntTime']
print "median effective integration time = ", np.median(effIntTime)
nWvlBins=len(wvlBinEdges)-1
print "cube shape ", np.shape(cube)
print "effIntTime shape ", np.shape(effIntTime)
#add third dimension to effIntTime for broadcasting
effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
cube /= effIntTime #put cube into counts/s
#calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins),dtype=float)
for n in xrange(nWvlBins):
binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
wvls[n] = (wvlBinEdges[n]+(binsize/2.0))
print "wvls ",wvls
#reshape cube for makeMovie
movieCube = np.zeros((nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float)
for i in xrange(nWvlBins):
movieCube[i,:,:] = cube[:,:,i]
#show individual frames as they are made to debug
#plt.matshow(movieCube[i],vmin = 0, vmax = 100)
#plt.show()
print "movieCube shape ", np.shape(movieCube)
print "wvls shape ", np.shape(wvls)
#print cube
#print "--------------------------"
#print movieCube
print "adding frames with wvl below ", stopWvl
finalframe = np.zeros((1,np.shape(movieCube)[1],np.shape(movieCube)[2]))
for f in xrange(len(wvls[wvls<stopWvl])):
print wvls[f]
finalframe[0]+=movieCube[f]
plt.matshow(movieCube[f],vmin=0,vmax = 40)
plt.show()
movieCube = finalframe
np.savez('%s_%s.npz'%(objectName,fileNum),stack=movieCube,wvls=wvls)
print "Saved frame to .npz file"
plt.matshow(movieCube[0],vmin=0,vmax = 40)
plt.show()
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False):
'''
Test run of obsFile.writePhotonList. fileName can be used
to specify the output file name. If not specified, default
name/location is used.
Now includes test for pixel remapping....
'''
#Details of example obs file to run test on.
# run = 'PAL2012'
# date = '20121207'
# tstamp = '20121208-074649'
# calTstamp='20121208-070505'
# fluxTstamp='20121208-133002'
run = 'PAL2013'
date = '20131209'
tstamp = '20131209-115553'
centroidTstamp = tstamp
calTstamp='20131209-132225'
fluxTstamp='ones'
flatTstamp='20131209'
if doPixRemap==True:
pixRemapFileName = FileName(run=run).pixRemap()
else:
pixRemapFileName = None
#Load up the obs file
obsFileName = FileName(run=run, date=date, tstamp=tstamp)
obsFile = ObsFile(obsFileName.obs())
obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())
HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask()
if not os.path.exists(HotPixFile): #check if hot pix file already exists
hp.findHotPixels(obsFileName.obs(),HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obsFile.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
#hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8''
centroid_RA = '12:13:56.42'
centroid_DEC = '32:56:30.8'
CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList()
#print "checking centroid file", CentFile
#if not os.path.exists(CentFile): #check if cent pix file already exists
cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10)
# print "Flux file centroid cal saved to %s"%(CentFile)
obsFile.loadCentroidListFile(CentFile)
print "Centroid File loaded %s"%(CentFile)
#Load up associated calibrations
#obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
#obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
#obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
#And write out the results....
obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
pixRemapFileName=pixRemapFileName)
#Read the results back in....
#photFile = photList.PhotFile(outputFilename)
print "Wrote photon list file: ", outputFileName
ffn = flatCalFilenames[iSeq]
sunsetDate = sunsetDates[iSeq]
for i,ts in enumerate(timestampList):
print 'loading',ts
obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
ob = ObsFile(obsFn)
ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments())
index1 = obsFn.find('_')
hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
if not os.path.exists(hotPixFn):
hp.findHotPixels(obsFn,hotPixFn)
print "Flux file pixel mask saved to %s"%(hotPixFn)
ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
ob.loadWvlCalFile(wfn)
ob.loadFlatCalFile(ffn)
ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)
bad_solution_mask=np.zeros((46,44))
bad_count=0;
for y in range(46):
for x in range(44):
if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff):
bad_solution_mask[y][x] = 1
unix = ob.getFromHeader('unixtime')
startJD = unix/86400.+2440587.5
nSecInFile = ob.getFromHeader('exptime')
#tic = time()
deadMask = ob.getDeadPixels()
timestampLists = [[utcDate+'-'+str(ts) for ts in seq] for utcDate,seq in zip(utcDates,seqs)]
wvlCalFilenames = [FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln() for sunsetDate,calTimestamp in zip(sunsetDates,calTimestamps)]
flatCalFilenames = [FileName(run=run,date=sunsetDate,tstamp=calTimestamp).flatSoln() for sunsetDate,calTimestamp in zip(sunsetDates,calTimestamps)]
#wvlCalFilenames[0] = '/Scratch/waveCalSolnFiles/20121210/calsol_20121211-074031.h5'
#wvlCalFilenames[1] = '/home/danica/optimusP/testing/forMatt/calsol_20121211-044853.h5'
flatCalFilenames[0] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5'
flatCalFilenames[1] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5'
fluxCalFileNames = ['/Scratch/fluxCalSolnFiles/20121206/fluxsol_20121207-124034.h5']
obsFn = FileName(run=run,date=sunsetDates[0],tstamp='20121209-120530').obs()
ob = ObsFile(obsFn)
print 'Loading wavelength calibration solution: ' + wvlCalFilenames[0]
ob.loadWvlCalFile(wvlCalFilenames[0])
print 'Loading flat calibration solution: ' + flatCalFilenames[0]
ob.loadFlatCalFile(flatCalFilenames[0])
ob.loadFluxCalFile(fluxCalFileNames[0])
#load/generate hot pixel mask file
HotPixFile = getTimeMaskFileName(obsFn)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFn,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
ob.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
frame = ob.getPixelCountImage(firstSec=0,integrationTime=300,weighted=True)
#hotPixMask = hotPixels.checkInterval(image=frame, firstSec=0, intTime=300, weighted=True, display=False)['mask']
#summed_array,bin_edges=ob.getApertureSpectrum(pixelCol=14,pixelRow=8,radius=7)
ob.plotApertureSpectrum(pixelCol=14,pixelRow=8,radius=7,weighted = True,fluxWeighted=True,lowCut=3000,highCut=9000)
