def generateObsObjectList(obsFNs,wvlLowerLimit=3000, wvlUpperLimit=13000,beammapFileName = None,loadHotPix=True,loadWvlCal=True,loadFlatCal=True,loadSpectralCal=True): obsFiles = [] for obsFN in obsFNs: if type(obsFN) == type(''): #Full path to obs file obs = ObsFile(obsFN) else: #FileName object obs = ObsFile(obsFN.obs()) if beammapFileName is not None and os.path.isfile(beammapFileName): obs.loadBeammapFile(beammapFileName) obs.setWvlCutoffs(wvlLowerLimit=wvlLowerLimit, wvlUpperLimit=wvlUpperLimit) if loadHotPix: if not os.path.isfile(FileName(obsFile=obs).timeMask()): print "generating hp file ", FileName(obsFile=obs).timeMask() hp.findHotPixels(obsFile=obs,outputFileName=FileName(obsFile=obs).timeMask()) obs.loadHotPixCalFile(FileName(obsFile=obs).timeMask(),reasons=['hot pixel','dead pixel']) if loadWvlCal: obs.loadBestWvlCalFile() if loadFlatCal: #obs.loadFlatCalFile(FileName(obsFile=obs).flatSoln()) obs.loadFlatCalFile('/Scratch/flatCalSolnFiles/flatsol_1s.h5') if loadSpectralCal: pass obsFiles.append(obs) return obsFiles
def main(): np.set_printoptions(threshold=np.nan) testPixelRow = 24 testPixelCol = 17 #obs_20120919-131142.h5,obs_20120919-131346.h5 #create a cal file from a twilight flat cal = FlatCal('../../params/flatCal.dict') #open another twilight flat as an observation and apply a wavelength cal and the new flat cal # run='LICK2012' # obsFileName = FileName(run=run,date='20120918',tstamp='20120919-131142').flat() # flatCalFileName = FileName(run=run,date='20120918',tstamp='20120919-131448').flatSoln() # wvlCalFileName = FileName(run=run,date='20120916',tstamp='20120917-072537').calSoln() run = 'PAL2012' obsFileName = FileName(run=run,date='20121211',tstamp='20121212-140003').obs() flatCalFileName = FileName(run=run,date='20121210',tstamp='').flatSoln() wvlCalFileName = FileName(run=run,date='20121210',tstamp='20121211-133056').calSoln() flatCalPath = os.path.dirname(flatCalFileName) ob = ObsFile(obsFileName)#('obs_20120919-131142.h5') ob.loadWvlCalFile(wvlCalFileName)#('calsol_20120917-072537.h5') ob.loadFlatCalFile(flatCalFileName)#('flatsol_20120919-131142.h5') #plot some uncalibrated and calibrated spectra for one pixel fig = plt.figure() ax = fig.add_subplot(211) ax2 = fig.add_subplot(212) print ob.getPixelCount(testPixelRow,testPixelCol) #flatSpectrum,wvlBinEdges = ob.getPixelSpectrum(testPixelRow,testPixelCol,weighted=False) spectrum,wvlBinEdges = ob.getPixelSpectrum(testPixelRow,testPixelCol,wvlStart=cal.wvlStart,wvlStop=cal.wvlStop,wvlBinWidth=cal.wvlBinWidth,weighted=False,firstSec=0,integrationTime=-1) weightedSpectrum,wvlBinEdges = ob.getPixelSpectrum(testPixelRow,testPixelCol,weighted=True) #flatSpectrum,wvlBinEdges = cal.flatFile.getPixelSpectrum(testPixelRow,testPixelCol,wvlStart=cal.wvlStart,wvlStop=cal.wvlStop,wvlBinWidth=cal.wvlBinWidth,weighted=False,firstSec=0,integrationTime=-1) flatSpectrum = cal.spectra[testPixelRow,testPixelCol] x = wvlBinEdges[0:-1] ax.plot(x,cal.wvlMedians,label='median spectrum',alpha=.5) ax2.plot(x,cal.flatFactors[testPixelRow,testPixelCol,:],label='pixel weights',alpha=.5) ax2.set_title('flat weights for pixel %d,%d'%(testPixelRow,testPixelCol)) ax.plot(x,spectrum+20,label='unweighted spectrum for pixel %d,%d'%(testPixelRow,testPixelCol),alpha=.5) ax.plot(x,weightedSpectrum+10,label='weighted %d,%d'%(testPixelRow,testPixelCol),alpha=.5) ax.plot(x,flatSpectrum+30,label='flatFile %d,%d'%(testPixelRow,testPixelCol),alpha=.5) ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.3),fancybox=True,ncol=3) plt.show() #display a time-flattened image of the twilight flat as it is and after using itself as it's flat cal #cal.flatFile.loadFlatCalFile(flatCalFileName)#('flatsol_20120919-131142.h5') #cal.flatFile.displaySec(weighted=True,integrationTime=-1) #ob.displaySec(integrationTime=-1) #ob.displaySec(weighted=True,integrationTime=-1) for idx in range(0,100,20): factors10 = cal.flatFactors[:,:,idx] plt.matshow(factors10,vmax=np.mean(factors10)+1.5*np.std(factors10)) plt.title('Flat weights at %d'%cal.wvlBinEdges[idx]) plt.colorbar() plt.savefig('plots/factors%d.png'%idx) plt.show()
def loadObs(self,stackLabel): timestampList = self.params[stackLabel+'Sequence'] run = self.params['run'] sunsetDate = self.params[stackLabel+'SunsetDate'] utcDate = self.params[stackLabel+'UtcDate'] intTime = self.params[stackLabel+'IntTime'] wvlLowerCutoff = self.params[stackLabel+'WvlLowerCutoff'] wvlUpperCutoff = self.params[stackLabel+'WvlUpperCutoff'] calTimestamp = self.params[stackLabel+'WvlTimestamp'] print stackLabel,calTimestamp wvlSolnFileName = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln() wvlCalFileName = FileName(run=run,date=self.params[stackLabel+'WvlSunsetDate'],tstamp=calTimestamp).cal() flatSolnFileName = FileName(run=run,date=self.params[stackLabel+'FlatCalSunsetDate'],tstamp=self.params[stackLabel+'FlatCalTimestamp']).flatSoln() obsFileNames = [FileName(run=run,date=sunsetDate,tstamp=timestamp).obs() for timestamp in timestampList] obList = [ObsFile(obsFn) for obsFn in obsFileNames] for ob in obList: ob.loadWvlCalFile(wvlSolnFileName) ob.loadFlatCalFile(flatSolnFileName) self.stackObsFileLists[stackLabel] = obList cal = ObsFile(wvlCalFileName) cal.loadWvlCalFile(wvlSolnFileName) cal.loadFlatCalFile(flatSolnFileName) self.stackWvlCals[stackLabel] = cal
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess,hotPath,flatPath): obs = ObsFile(obsPath) print obsPath,obs.getFromHeader('exptime'),obs if not os.path.exists(hotPath): hp.findHotPixels(obsFile=obs,outputFileName=hotPath) obs.loadHotPixCalFile(hotPath,switchOnMask=False) obs.loadBestWvlCalFile() obs.loadFlatCalFile(flatPath) obs.setWvlCutoffs(3000,8000) cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath) print 'done centroid',centroidPath del obs
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=True): ''' Test run of obsFile.writePhotonList. fileName can be used to specify the output file name. If not specified, default name/location is used. Now includes test for pixel remapping.... ''' #Details of example obs file to run test on. # run = 'PAL2012' # date = '20121207' # tstamp = '20121208-074649' # calTstamp='20121208-070505' # fluxTstamp='20121208-133002' run = 'PAL2012' date = '20121208' tstamp = '20121209-120530' centroidTstamp = '20121209-120530' calTstamp='20121209-131132' fluxTstamp='20121209-020416' flatTstamp='20121209-021036' if doPixRemap==True: pixRemapFileName = FileName(run=run).pixRemap() else: pixRemapFileName = None #Load up the obs file obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) #Load up associated calibrations obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln()) obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln()) obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln()) obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments()) obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask()) obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList()) #And write out the results.... obsFile.writePhotonList(outputFileName,firstSec,integrationTime, pixRemapFileName=pixRemapFileName)
from util.ObsFile import ObsFile from util.FileName import FileName run = "PAL2014" date = "20141022" timeStamp = '20141023-033821' fn = FileName(run,date,timeStamp) of = ObsFile(fn.obs()) of.loadBeammapFile(fn.beammap()) of.loadBestWvlCalFile() print "wvlCalFileName=",of.wvlCalFileName fn2 = FileName(run,date,"") of.loadFlatCalFile(fn2.flatSoln()) row = 4 col = 4 firstSec = 72 integrationTime = 1 spec = of.getPixelSpectrum(row,col,firstSec,integrationTime) print "spec=",spec del of
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) >3: filenum = str('_'+sys.argv[3]) else: filenum = '_0' #science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() wvldir = params[2].split('=')[1].strip() obsfile = params[3].split('=')[1].strip() skyfile = params[4].split('=')[1].strip() flatfile = params[5].split('=')[1].strip() wvlfile = params[6].split('=')[1].strip() objectName = params[9].split('=')[1].strip() if len(params)>10: xpix = params[10].split('=')[1].strip() ypix = params[11].split('=')[1].strip() apertureRadius = params[12].split('=')[1].strip() #flux cal object parameter file params2 = [] param2file = sys.argv[2] f = open(param2file,'r') for line in f: params2.append(line) f.close() fluxdir = params2[7].split('=')[1].strip() fluxfile = params2[8].split('=')[1].strip() fluxCalObject = params2[9].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) print "obsfile = ",obsFileName print "skyfile = ",skyFileName print "wvlcal = ", wvlCalFileName print "flatcal = ", flatCalFileName print "fluxcal = ", fluxCalFileName print "object = ", objectName print "flux cal object = ", fluxCalObject print "\n---------------------\n" obs = ObsFile(obsFileName) obs.loadWvlCalFile(wvlCalFileName) obs.loadFlatCalFile(flatCalFileName) obs.loadFluxCalFile(fluxCalFileName) print "loaded data file and calibrations\n---------------------\n" HotPixFile = getTimeMaskFileName(obsFileName) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFileName,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") wvlBinEdges,obsSpectra,obsIntTime = loadSpectra(obs,nCol,nRow) nWvlBins=len(wvlBinEdges)-1 #divide spectrum by bin widths binWidths = np.empty(nWvlBins) for i in xrange(nWvlBins): binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i] obsSpectra = obsSpectra/binWidths #divide by effective integration time for iRow in xrange(nRow): for iCol in xrange(nCol): obsSpectra[iRow][iCol] = obsSpectra[iRow][iCol] / obsIntTime[iRow][iCol] #print np.shape(obsSpectra) #print nRow #print nCol #print nWvlBins medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins) print "target spectrum loaded\n---------------------\n" if skyfile != "None": sky = ObsFile(skyFileName) sky.loadWvlCalFile(wvlCalFileName) sky.loadFlatCalFile(flatCalFileName) sky.loadFluxCalFile(fluxCalFileName) skyTime = sky.getFromHeader("exptime") skyHotPixFile = getTimeMaskFileName(skyFileName) if not os.path.exists(skyHotPixFile): hp.findHotPixels(skyFileName,skyHotPixFile) print "Flux file pixel mask saved to %s"%(skyHotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(skyHotPixFile) wvlBinEdges,skySpectra,skyIntTime = loadSpectra(sky,nCol,nRow) skySpectra = skySpectra/binWidths #divide by effective integration time for iRow in xrange(nRow): for iCol in xrange(nCol): skySpectra[iRow][iCol] = skySpectra[iRow][iCol] / skyIntTime[iRow][iCol] skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins) #skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time print "sky spectrum loaded\n---------------------\n" else: #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny skySpectrum = medianObsSpectrum print "sky spectrum estimated as median of target file spectrum\n---------------------\n" #subtract sky spectrum from every pixel allSkySpectrum = obsSpectra-skySpectrum #set any negative values to 0 after sky subtraction allSkySpectrum[allSkySpectrum<0]=0 #take median of remaining sky subtracted spectra to get median object spectrum finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins) #load std spectrum for comparison realSpectra = loadStd(objectName,wvlBinEdges) print "real std spectrum loaded for reference\n---------------------\n" #create plots plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots" plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum) fullFluxPlotFileName = os.path.join(plotDir,plotFileName) pp = PdfPages(fullFluxPlotFileName) matplotlib.rcParams['font.size']=6 #calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins),dtype=float) for n in xrange(nWvlBins): binsize=wvlBinEdges[n+1]-wvlBinEdges[n] wvls[n] = (wvlBinEdges[n]+(binsize/2.0)) plt.figure() ax1 = plt.subplot(231) ax1.set_title('ARCONS median flat/flux cal\'d obs in counts') plt.plot(wvls,medianObsSpectrum) #plt.show() ax2 = plt.subplot(232) ax2.set_title('ARCONS median flat/flux cal\'d sky in counts') plt.plot(wvls,skySpectrum) #plt.show() ax5 = plt.subplot(233) ax5.set_title('Sensitivity Spectrum') plt.plot(wvls,obs.fluxWeights) ax3 = plt.subplot(234) ax3.set_title('MKID data minus sky in counts') plt.plot(wvls,finalSpectrum/max(finalSpectrum)) ax4 = plt.subplot(235) ax4.set_title('Rebinned Std Spectrum of %s'%(objectName)) plt.plot(wvls,realSpectra) #ax7 = plt.subplot(337) #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std') #plt.plot(wvls,fluxFactors*subtractedSpectra) pp.savefig() pp.close() #del obs #del sky print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName) txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt" txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum) fullFluxTxtFileName = os.path.join(txtDir,txtFileName) outarr = np.empty((len(medianObsSpectrum),2),dtype=float) outarr[:,0]=wvls outarr[:,1]=medianObsSpectrum #save sensitivity spectrum to file np.savetxt(fullFluxTxtFileName, outarr) print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
tstamp = '20140925-112538' obsFN = FileName(run=run,date=date,tstamp=tstamp) obsPath = obsFN.obs() flatPath = FileName(run=run,date=date).flatSoln() hotPath = obsFN.timeMask() centroidPath = obsFN.centroidList() fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln() obs = ObsFile(obsPath) if not os.path.exists(hotPath): hp.findHotPixels(obsPath,hotPath) obs.loadHotPixCalFile(hotPath,switchOnMask=False) obs.loadBestWvlCalFile() obs.loadFlatCalFile(flatPath) obs.loadFluxCalFile(fluxPath) obs.loadCentroidListFile(centroidPath) obs.setWvlCutoffs(3000,11000) centroidRa = '03:37:43.826' centroidDec = '14:15:14.828' haOffset = 150. # imgDict = obs.getPixelCountImage(integrationTime=60,scaleByEffInt=True) # plotArray(imgDict['image']) xGuess = 18 #col yGuess = 15 #row
class DisplayStack(QMainWindow): def __init__(self): # 0) Start up gui QWidget.__init__(self, parent=None) self.ui = Ui_DisplayStack_gui() self.ui.setupUi(self) # Initialize Variables self.initializeVariables() # Initialize run and target, load param file self.chooseRun() ''' # Browse wavecal button self.ui.wavelengthCalibrationSolutionButton.clicked.connect(self.chooseWavelengthFile) # Browse flatcal button self.ui.flatCalibrationSolutionButton.clicked.connect(self.chooseFlatFile) ''' # Use wavelength calibration checkbox self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal) # Use flat calibration checkbox self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal) # Buttons for obs list creation self.ui.addButton.clicked.connect(self.addObservation) self.ui.removeButton.clicked.connect(self.removeObservation) self.ui.clearButton.clicked.connect(self.createObsList) # Start process button self.ui.stackButton.clicked.connect(self.stackProcess) # Load image stack button self.ui.loadStackButton.clicked.connect(self.chooseStack) # 1) Initialize Variables def initializeVariables(self): # Define path names self.displayStackPath = '/Scratch/DisplayStack/' self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/' self.defaultFlatPath = '/Scratch/flatCalSolnFiles/' # Arrays with run names and buttons self.runNames = ['LICK2012','PAL2012','PAL2013'] self.runButtons = [self.ui.lick2012Button, self.ui.pal2012Button, self.ui.pal2013Button] # Arrays with target names and buttons self.lick2012TargetNames = ['LICK2012_Object'] self.lick2012TargetButtons = [self.ui.lick2012ObjectButton] self.pal2012TargetNames = ['SDSS_J0651', 'SDSS_J0926'] self.pal2012TargetButtons = [self.ui.sdss0651Button,self.ui.sdss0926Button] self.pal2013TargetNames = ['PAL2013_Object'] self.pal2013TargetButtons = [self.ui.pal2013ObjectButton] self.targetNames = [self.lick2012TargetNames, self.pal2012TargetNames, self.pal2013TargetNames] self.targetButtons = [self.lick2012TargetButtons, self.pal2012TargetButtons, self.pal2013TargetButtons] # Activate run and target buttons for iRun in range(len(self.runButtons)): self.runButtons[iRun].clicked.connect(self.chooseRun) for iTarget in range(len(self.targetButtons[iRun])): self.targetButtons[iRun][iTarget].clicked.connect(self.chooseTarget) self.ui.sunsetList.itemClicked.connect(self.createObsList) def testFunction(self): print 'test' # 2) Choose Run def chooseRun(self): for iRun in range(len(self.runButtons)): for iTarget in range(len(self.targetButtons[iRun])): self.targetButtons[iRun][iTarget].setVisible(False) if self.runButtons[iRun].isChecked(): self.runNumber = iRun for iTarget in range(len(self.targetButtons[iRun])): self.targetButtons[iRun][iTarget].setVisible(True) self.run = self.runNames[self.runNumber] self.targetNumber = 0 self.targetButtons[self.runNumber][self.targetNumber].setChecked(True) self.target = self.targetNames[self.runNumber][self.targetNumber] self.loadParamFile() # 3) Choose Target def chooseTarget(self): for iTarget in range(len(self.targetButtons[self.runNumber])): if self.targetButtons[self.runNumber][iTarget].isChecked(): self.targetNumber = iTarget self.target = self.targetNames[self.runNumber][self.targetNumber] self.loadParamFile() # 4) Load Param File def loadParamFile(self): try: self.paramData = readDict() self.paramData.read_from_file(self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict') self.obsTimes = np.array(self.paramData['obsTimes']) self.utcDates = self.paramData['utcDates'] self.sunsetDates = self.paramData['sunsetDates'] self.calTimestamps = self.paramData['calTimestamps'] self.flatCalDates = self.paramData['flatCalDates'] print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.createSunsetList() #self.createObsList() self.createWavelengthList() self.createFlatList() self.paramFileExists = True except IOError: print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.ui.sunsetList.clear() self.ui.obsList.clear() self.ui.inputList.clear() self.ui.wavelengthList.clear() self.ui.flatList.clear() self.paramFileExists = False # 5) Choose Obs File # Create list of available sunset dates def createSunsetList(self): self.ui.sunsetList.clear() for iDate in range(len(self.sunsetDates)): self.ui.sunsetList.addItem(self.sunsetDates[iDate]) # Create Initial Obs file list def createObsList(self): self.ui.obsList.clear() self.ui.inputList.clear() self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] for iObs in range(len(self.singleDayObservations)): self.ui.obsList.addItem(self.singleDayObservations[iObs]) # Add observation to input list def addObservation(self): self.selectedObs = self.ui.obsList.currentItem() self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs)) self.ui.inputList.addItem(self.selectedObs) self.ui.inputList.sortItems() # Remove observation from input list def removeObservation(self): self.removedObs = self.ui.inputList.currentItem() self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs)) self.ui.obsList.addItem(self.removedObs) self.ui.obsList.sortItems() # 6) Load settings def loadSettings(self): self.validSettings = True # Run and target information self.run = self.runNames[self.runNumber] self.target = self.targetNames[self.runNumber][self.targetNumber] # General settings self.integrationTime = int(self.ui.integrationTimeLine.text()) self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked() self.useHotPixelMasking = self.ui.hotPixelBox.isChecked() # Wavelength calibration settings self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked() self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text()) self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text()) # Flat calibration settings self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked() self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked() self.fileCount = self.ui.inputList.count() self.weighted = self.useFlatCalibration self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration) self.scaleByEffInt = self.useHotPixelMasking if self.ui.sunsetList.currentItem() != None: if self.fileCount == 0: print 'Please select files to process...' self.validSettings = False if self.useWavelengthCalibration: if self.ui.wavelengthList.currentItem() == None: print 'Please select wavelength calibration...' self.validSettings = False else: self.selectedWvlCal = self.ui.wavelengthList.currentItem().text() self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln()) if self.useFlatCalibration: if self.ui.flatList.currentItem() == None: print 'Please select flat calibration...' self.validSettings = False else: self.flatCalNight = self.ui.flatList.currentItem().text() self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln()) else: print 'Please select sunset night...' self.validSettings = False # 7) Load hot pixel mask def loadHotMask(self): self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5') if not os.path.exists(self.hotPixelFilename): hp.findHotPixels(self.obsFn,self.hotPixelFilename) print "Hot pixel mask saved to %s"%(self.hotPixelFilename) self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True) # 8) Create wavelength cal file list def createWavelengthList(self): self.ui.wavelengthList.clear() for iCal in range(len(self.calTimestamps)): self.ui.wavelengthList.addItem(self.calTimestamps[iCal]) # Enable/disable wavecal options def useWaveCal(self): if self.ui.wavelengthCalibrationBox.isChecked(): self.ui.lowerWavelengthCutoffLine.setEnabled(True) self.ui.lowerWavelengthCutoffLabel.setEnabled(True) self.ui.upperWavelengthCutoffLine.setEnabled(True) self.ui.upperWavelengthCutoffLabel.setEnabled(True) self.ui.wavelengthList.setEnabled(True) self.ui.flatCalibrationBox.setEnabled(True) self.ui.flatCalibrationBox.setChecked(True) self.ui.deadPixelBox.setEnabled(True) self.ui.deadPixelBox.setChecked(True) self.ui.flatList.setEnabled(True) else: self.ui.lowerWavelengthCutoffLine.setEnabled(False) self.ui.lowerWavelengthCutoffLabel.setEnabled(False) self.ui.upperWavelengthCutoffLine.setEnabled(False) self.ui.upperWavelengthCutoffLabel.setEnabled(False) self.ui.wavelengthList.setEnabled(False) self.ui.flatCalibrationBox.setEnabled(False) self.ui.flatCalibrationBox.setChecked(False) self.ui.deadPixelBox.setChecked(False) self.ui.deadPixelBox.setEnabled(False) self.ui.flatList.setEnabled(False) # 9) Create flat cal file list def createFlatList(self): self.ui.flatList.clear() for iCal in range(len(self.flatCalDates)): self.ui.flatList.addItem(self.flatCalDates[iCal]) # Enable/disable flatcal options def useFlatCal(self): if self.ui.flatCalibrationBox.isChecked(): self.ui.deadPixelBox.setEnabled(True) self.ui.deadPixelBox.setChecked(True) self.ui.flatList.setEnabled(True) else: self.ui.deadPixelBox.setChecked(False) self.ui.deadPixelBox.setEnabled(False) self.ui.flatList.setEnabled(False) # 10) Load dead pixel mask def loadDeadMask(self): self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz') if not os.path.exists(self.deadPixelFilename): self.deadMask = self.ob.getDeadPixels() np.savez(self.deadPixelFilename,deadMask = self.deadMask) print "Dead pixel mask saved to %s"%(self.deadPixelFilename) else: self.deadFile = np.load(self.deadPixelFilename) self.deadMask = self.deadFile['deadMask'] # 12) Create output name def createOutputName(self): self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's') if self.useWavelengthCalibration and self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.npz') elif self.useWavelengthCalibration and not self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.npz') elif not self.waveLengthCalibration and self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_hp.npz') else: self.outputFilename = str(self.rawName + '.npz') # Start process for creating image stacks def stackProcess(self): # Check for valid params file if self.paramFileExists: # 6) Load settings choosen from gui self.loadSettings() if self.validSettings: # Loop through all files in input list for iFile in range(self.fileCount): # Create ObsFile instance self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text() self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs()) print 'Processing file ' + self.obsFn + '...' self.ob = ObsFile(self.obsFn) # Load time adjustment file if self.useTimeAdjustment: print 'Loading time adjustment file...' self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments()) # 7) Load hot pixel mask if self.useHotPixelMasking: print 'Loading hot pixel mask...' self.loadHotMask() # 8) Load wave cal solution if self.useWavelengthCalibration: print 'Loading wavelength calibration...' self.ob.loadWvlCalFile(self.wvlCalFilename) # 9) Load flatcal solution if self.useFlatCalibration: print 'Loading flat calibration...' self.ob.loadFlatCalFile(self.flatCalFilename) # 10) Load dead pixel mask if self.useDeadPixelMasking: print 'Loading dead pixel mask...' self.loadDeadMask() # 11) Set wavelength cutoffs if self.useWavelengthCalibration: print 'Setting wavelength cutoffs...' self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff) # Timing self.unix = self.ob.getFromHeader('unixtime') self.startJD = self.unix/86400.+2440587.5 self.exptime = self.ob.getFromHeader('exptime') self.times = [] self.frames = [] # 11) Create Image Stack print 'Stacking images...' for iSec in np.arange(0,self.exptime,self.integrationTime): #add seconds offset to julian date, move jd to center of bin self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.) 'Creating frame for time ' + self.jd self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt) self.frame = self.frameData['image'] if self.useDeadPixelMasking: self.frame[self.deadMask == 0] = np.nan self.frames.append(self.frame) self.cube = np.dstack(self.frames) self.times = np.array(self.times) # 12) Create output file self.createOutputName() print 'Saving image stack to ' + self.outputFilename np.savez(self.outputFilename, stack=self.cube, jd=self.times) # Invalid params file else: print 'Invalid parameter file...' # Choose an image stack def chooseStack(self): self.defaultLoadStackDirectory = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks') self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("NPZ (*.npz)")) if self.stackName == '': print 'No file chosen' else: self.loadStack() # Load that image stack for viewing def loadStack(self):
hp.findHotPixels(inputFileName=obsFileName,outputFileName=timeMaskFileName) print "Flux file pixel mask saved to %s"%(timeMaskFileName) obs = ObsFile(obsFileName) exptime = obs.getFromHeader('exptime') #obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments()) if wvlCal != '': obs.loadWvlCalFile(wvlFileName) else: obs.loadBestWvlCalFile() obs.loadHotPixCalFile(timeMaskFileName) obs.setWvlCutoffs(lowerWvlCut,upperWvlCut) obs.loadFlatCalFile(flatFileName) #get image before and after flat cal #print 'getting images' rawCubeDict = obs.getSpectralCube(weighted=False) rawCube = np.array(rawCubeDict['cube'],dtype=np.double) effIntTime = rawCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) rawCube *= maxIntTime / effIntTime3d rawImg = np.sum(rawCube,axis=-1) rawImg[rawImg == 0] = np.nan rawImg[~np.isfinite(rawImg)] = np.nan #print 'finished raw cube' flatCubeDict = obs.getSpectralCube(weighted=True)
tic = time() print 'Loading obs file and performing calibrations ...' obsFn = FileName(run=run,date=sunsetDate,tstamp=obsTimestamp).obs() ob = ObsFile(obsFn) # Load roach time delay corrections. ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments()) # Search for time mask for given observation file. If the time mask does not exist, create it. index1 = obsFn.find('_') hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:] if not os.path.exists(hotPixFn): hp.findHotPixels(obsFn,hotPixFn) print "Flux file pixel mask saved to %s"%(hotPixFn) # Load time mask, wavelength calibration, and flat calibration and set wavelenth cutoffs. ob.loadHotPixCalFile(hotPixFn,switchOnMask=True) ob.loadWvlCalFile(wvlCalFilename) ob.loadFlatCalFile(flatCalFilename) ob.setWvlCutoffs(3000,5000) print 'Total load time: ' + str(time()-tic) + 's' tic = time() print 'Retrieving photon timestamps...' # Define the area of an observation containing the object. appMask = aperture(15,8,8) yValues,xValues = np.where(appMask==0) # Get a list of timestamps with photons from each pixel in the aperture. timestamps=[] for j in range(len(xValues)): x=ob.getPixelWvlList(iRow=yValues[j],iCol=xValues[j]) timestamps = np.append(timestamps,x['timestamps']) print 'Total retrieval time: ' + str(time()-tic) + 's'
class AppForm(QMainWindow): def __init__(self, parent=None): QMainWindow.__init__(self, parent) self.setWindowTitle('Pixel Explorer') paramFile = sys.argv[1] self.params = readDict() self.params.read_from_file(paramFile) self.createMainFrame() self.createStatusBar() def createMainFrame(self): self.main_frame = QWidget() # Create the mpl Figure and FigCanvas objects. self.dpi = 100 self.fig = Figure((7.0, 7.0), dpi=self.dpi) self.canvas = FigureCanvas(self.fig) self.canvas.setParent(self.main_frame) self.axes0 = self.fig.add_subplot(111) cid=self.canvas.mpl_connect('button_press_event', self.clickCanvas) # Create the navigation toolbar, tied to the canvas self.mpl_toolbar = NavigationToolbar(self.canvas, self.main_frame) vbox = QVBoxLayout() vbox.addWidget(self.canvas) vbox.addWidget(self.mpl_toolbar) self.main_frame.setLayout(vbox) self.setCentralWidget(self.main_frame) def createStatusBar(self): self.status_text = QLabel("Awaiting orders.") self.statusBar().addWidget(self.status_text, 1) def openImage(self): timestampList = [self.params['obsUtcDate']+'-'+ts for ts in self.params['obsSequence']] run = self.params['run'] sunsetDate = self.params['obsSunsetDate'] utcDate = self.params['obsUtcDate'] self.intTime = self.params['intTime'] wvlLowerCutoff = self.params['wvlLowerCutoff'] wvlUpperCutoff = self.params['wvlUpperCutoff'] calTimestamp = self.params['wvlTimestamp'] wfn = FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln() calfn = FileName(run=run,date=self.params['wvlSunsetDate'],tstamp=calTimestamp).cal() ffn = FileName(run=run,date=self.params['flatCalSunsetDate'],tstamp='').flatSoln() obsFns = [FileName(run=run,date=sunsetDate,tstamp=timestamp).obs() for timestamp in timestampList] self.obList = [ObsFile(obsFn) for obsFn in obsFns] for ob in self.obList: print 'Loading ',ob.fullFileName ob.loadWvlCalFile(wfn) ob.loadFlatCalFile(ffn) self.cal = ObsFile(calfn) self.cal.loadWvlCalFile(wfn) self.cal.loadFlatCalFile(ffn) self.loadSpectra() def loadSpectra(self): fileName = self.params['outFileName'] if os.path.exists(fileName): data = np.load(fileName) self.spectra = data['cube'] self.frame = data['frame'] self.frameValues = self.frame[~np.isnan(self.frame)] self.wvlBinEdges = np.array(self.obList[0].flatCalWvlBins) self.frameIntTime = 300*6 else: self.spectra,self.wvlBinEdges = self.obList[0].getSpectralCube(weighted=True) self.frameIntTime = 0 for ob in self.obList[1:]: print ob.fileName cube,wvlBinEdges = ob.getSpectralCube(weighted=True) self.spectra += cube self.frameIntTime += ob.getFromHeader('exptime') self.spectra = np.array(self.spectra,dtype=np.float64) self.frame = np.sum(self.spectra,axis=2) hotPixMask = hotPixels.findHotPixels(image=self.frame,nsigma=2)['badflag'] self.frame[hotPixMask != 0] = np.nan self.frame[self.frame == 0] = np.nan self.frameValues = self.frame[~np.isnan(self.frame)] np.savez(fileName,cube=self.spectra,frame=self.frame) def plotWeightedImage(self): self.showFrame = np.array(self.frame) self.showFrame[np.isnan(self.frame)] = 0 handleMatshow = self.axes0.matshow(self.showFrame,cmap=matplotlib.cm.gnuplot2,origin='lower',vmax=np.mean(self.showFrame)+3*np.std(self.showFrame)) self.fig.colorbar(handleMatshow) def clickCanvas(self,event): self.showLaserSpectrum = True self.showPixelSpectrum = True self.showWvlLightCurves = True self.showWvlLightCurveHists = False self.showStdVsIntTime = True self.showNormStdVsIntTime = True col = round(event.xdata) row = round(event.ydata) if self.showPixelSpectrum: #next plot the integrated spectrum for this pixel in the total image spectrum = self.spectra[row,col] print sum(spectrum),' counts in broadband spectrum' def plotFunc(fig,axes): axes.plot(self.wvlBinEdges[:-1],spectrum) axes.set_xlabel(r'$\lambda$ ($\AA$)') axes.set_ylabel(r'total counts') popup = PopUp(parent=self,plotFunc=plotFunc,title='spectrum, pixel %d,%d (intTime=%d)'%(row,col,self.frameIntTime)) rebinSpecBins = 5 firstAfterConvolve = rebinSpecBins//2 rebinnedWvlEdges = self.wvlBinEdges[::rebinSpecBins] if self.showLaserSpectrum: #First plot the laser cal spectrum for this pixel to see if it's good laserSpectrum,binEdges = self.cal.getPixelSpectrum(row,col,weighted=True) def plotFunc(fig,axes): axes.plot(binEdges[:-1],laserSpectrum) axes.set_xlabel(r'$\lambda$ ($\AA$)') axes.set_ylabel(r'total counts') popup = PopUp(parent=self,plotFunc=plotFunc,title='Laser Cal Spectrum, pixel %d,%d'%(row,col)) if self.showWvlLightCurves: spectrumInTime = [] for iOb,ob in enumerate(self.obList): for sec in range(0,ob.getFromHeader('exptime'),self.intTime): spectrum,binEdges = ob.getPixelSpectrum(pixelRow=row,pixelCol=col,firstSec=sec,integrationTime=self.intTime,weighted=True) spectrum = np.convolve(spectrum,np.ones(rebinSpecBins),'same')[firstAfterConvolve::rebinSpecBins] spectrumInTime.append(spectrum) spectrumInTime = np.array(spectrumInTime) nBins = np.shape(spectrumInTime)[1] def plotFunc(fig,axes): #plot counts vs time for each wavelength bin t=np.arange(len(spectrumInTime[:,0]))*self.intTime for iBin in xrange(nBins): axes.plot(t,1.0*spectrumInTime[:,iBin]/self.intTime, c=cm.jet((iBin+1.)/nBins), label=r'%d-%d $\AA$'%(rebinnedWvlEdges[iBin], rebinnedWvlEdges[iBin+1])) axes.set_xlabel('time (s)') axes.set_ylabel('cps') #plot counts vs time summed over all wavelengths axes.plot(t,np.sum(spectrumInTime,axis=1)/self.intTime,c='k', label=r'%d-%d $\AA$'%(rebinnedWvlEdges[0],rebinnedWvlEdges[-1])) #axes.legend(loc='center right') popup = PopUp(parent=self,plotFunc=plotFunc,title='Light Curve by Band, Pixel %d,%d'%(row,col)) if self.showWvlLightCurveHists or self.showStdVsIntTime or self.showNormStdVsIntTime: intTimes = [1,2,3,5,10,15,30] spectrumVsIntTimeVsTime = [] for intTime in intTimes: spectrumInTime = [] for iOb,ob in enumerate(self.obList): for sec in range(0,ob.getFromHeader('exptime'),intTime): spectrum,binEdges = ob.getPixelSpectrum(pixelRow=row,pixelCol=col,firstSec=sec,integrationTime=intTime,weighted=True) spectrum = np.convolve(spectrum,np.ones(rebinSpecBins),'same')[firstAfterConvolve::rebinSpecBins] spectrumInTime.append(spectrum) spectrumInTime = np.array(spectrumInTime) spectrumVsIntTimeVsTime.append(spectrumInTime) #resulting array indexed as #spectrumVsIntTimeVsTime[iIntTime][iTimeChunk][iWvlBin] #sum over wavelength for total counts countsVsIntTimeVsTime = [np.sum(spectrumInTime,axis=1) for spectrumInTime in spectrumVsIntTimeVsTime] #countsVsIntTimeVsTime[iIntTime][iTimeChunk] if self.showWvlLightCurveHists: for iIntTime,countsVsTime in enumerate(countsVsIntTimeVsTime): def plotFunc(fig,axes): axes.hist(countsVsTime,bins=20) axes.set_xlabel('counts per intTime %d s'%intTimes[iIntTime]) popup = PopUp(parent=self,plotFunc=plotFunc,title='Int Time %d s, pixel %d,%d'%(intTimes[iIntTime],row,col)) countStds = [np.std(countsVsTime) for countsVsTime in countsVsIntTimeVsTime] countStds = np.array(countStds) countSqrts = [np.sqrt(np.median(countsVsTime)) for countsVsTime in countsVsIntTimeVsTime] countSqrts = np.array(countSqrts) spectrumStds = [np.std(spectrumVsTime,axis=0) for spectrumVsTime in spectrumVsIntTimeVsTime] spectrumSqrts = [np.sqrt(np.median(spectrumVsTime,axis=0)) for spectrumVsTime in spectrumVsIntTimeVsTime] spectrumStds = np.array(spectrumStds) spectrumSqrts = np.array(spectrumSqrts) if self.showStdVsIntTime: def plotFunc(fig,axes): axes.set_xlabel('integration time (s)') axes.plot(intTimes,countStds,'k',label=r'total $\sigma$') axes.plot(intTimes,countSqrts,'k--',label=r'$\sqrt{med(N)}$') nBins = np.shape(spectrumStds)[1] for iBin in xrange(nBins): axes.plot(intTimes,spectrumStds[:,iBin], c=cm.jet((iBin+1.)/nBins), label=r'%d-%d $\AA$ $\sigma$'%(rebinnedWvlEdges[iBin], rebinnedWvlEdges[iBin+1])) axes.plot(intTimes,spectrumSqrts[:,iBin], c=cm.jet((iBin+1.)/nBins),linestyle='--') axes.legend(loc='upper left') popup = PopUp(parent=self,plotFunc=plotFunc, title=r'$\sigma$ vs Integration Time, Pixel %d,%d'%(row,col)) if self.showNormStdVsIntTime: def plotFunc(fig,axes): axes.set_xlabel('integration time (s)') axes.set_ylabel('normalized $\sigma$') axes.plot(intTimes,countSqrts/np.max(countSqrts),'k--', label=r'$\sqrt{N}$') axes.plot(intTimes,countStds/np.max(countSqrts),'k', label=r'%d-%d $\AA$'%(rebinnedWvlEdges[0],rebinnedWvlEdges[-1])) nBins = np.shape(spectrumStds)[1] for iBin in xrange(nBins): axes.plot(intTimes, spectrumStds[:,iBin]/np.max(spectrumSqrts[:,iBin]), c=cm.jet((iBin+1.0)/nBins), label=r'%d-%d $\AA$'%(rebinnedWvlEdges[iBin], rebinnedWvlEdges[iBin+1])) axes.legend(loc='upper left') popup = PopUp(parent=self,plotFunc=plotFunc, title='Normalized Standard Deviation vs IntegrationTime, Pixel %d,%d'%(row,col))
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) >3: filenum = str('_'+sys.argv[3]) else: filenum = '_0' #science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() wvldir = params[2].split('=')[1].strip() obsfile = params[3].split('=')[1].strip() skyfile = params[4].split('=')[1].strip() flatfile = params[5].split('=')[1].strip() wvlfile = params[6].split('=')[1].strip() objectName = params[9].split('=')[1].strip() if len(params)>10: xpix = int(params[10].split('=')[1].strip()) ypix = int(params[11].split('=')[1].strip()) apertureRadius = int(params[12].split('=')[1].strip()) #flux cal object parameter file params2 = [] param2file = sys.argv[2] f = open(param2file,'r') for line in f: params2.append(line) f.close() fluxdir = params2[7].split('=')[1].strip() fluxfile = params2[8].split('=')[1].strip() fluxCalObject = params2[9].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) print "obsfile = ",obsFileName print "skyfile = ",skyFileName print "wvlcal = ", wvlCalFileName print "flatcal = ", flatCalFileName print "fluxcal = ", fluxCalFileName print "object = ", objectName print "flux cal object = ", fluxCalObject print "\n---------------------\n" obs = ObsFile(obsFileName) obs.loadWvlCalFile(wvlCalFileName) obs.loadFlatCalFile(flatCalFileName) obs.loadFluxCalFile(fluxCalFileName) print "loaded data file and calibrations\n---------------------\n" nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow) #nWvlBins=len(wvlBinEdges)-1 #print np.shape(obsSpectra) #print nRow #print nCol #print nWvlBins """ medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins) print "target spectrum loaded\n---------------------\n" if skyfile != "None": sky = ObsFile(skyFileName) sky.loadWvlCalFile(wvlCalFileName) sky.loadFlatCalFile(flatCalFileName) sky.loadFluxCalFile(fluxCalFileName) skyTime = sky.getFromHeader("exptime") wvlBinEdges,skySpectra = loadSpectra(sky,nCol,nRow) skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins) skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time print "sky spectrum loaded\n---------------------\n" else: #if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny skySpectrum = calculateMedian(obsSpectra, nCol, nRow, nWvlBins) print "sky spectrum estimated as median of target file spectrum\n---------------------\n" #subtract sky spectrum from every pixel allSkySpectrum = obsSpectra-skySpectrum #set any negative values to 0 after sky subtraction allSkySpectrum[allSkySpectrum<0]=0 #take median of remaining sky subtracted spectra to get median object spectrum finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins) """ #load/generate hot pixel mask file HotPixFile = getTimeMaskFileName(obsFileName) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFileName,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) print "Making spectrum with Aperture Spectrum in ObsFile" #use Aperture Spectrum from obsfile medianObsSpectrum, wvlBinEdges = obs.getApertureSpectrum(pixelCol=ypix,pixelRow=xpix,radius1=apertureRadius, radius2 = apertureRadius*2.0,weighted=True, fluxWeighted=True, lowCut=3000, highCut=7000) nWvlBins=len(wvlBinEdges)-1 #load std spectrum for comparison try: realSpectra = loadStd(objectName,wvlBinEdges) print "real std spectrum loaded for reference\n---------------------\n" stdTitle = "Rebinned Std Spectrum of %s"%(objectName) except KeyError: print "Key Error loading MKIDStd" realSpectra = np.ones(nWvlBins) stdTitle = "No MKIDStd spectrum available for %s"%(objectName) #create plots plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots" plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum) fullFluxPlotFileName = os.path.join(plotDir,plotFileName) pp = PdfPages(fullFluxPlotFileName) matplotlib.rcParams['font.size']=6 #calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins),dtype=float) for n in xrange(nWvlBins): binsize=wvlBinEdges[n+1]-wvlBinEdges[n] wvls[n] = (wvlBinEdges[n]+(binsize/2.0)) plt.figure() ax1 = plt.subplot(221) ax1.set_title('ARCONS median flat/flux cal\'d obs in counts') ax1.set_xlim((4000,11000)) ax1.set_ylim((min(medianObsSpectrum[(wvls>4000) & (wvls<11000)]),max(medianObsSpectrum[(wvls>4000) & (wvls<8000)]))) plt.plot(wvls,medianObsSpectrum) #plt.show() #ax2 = plt.subplot(232) #ax2.set_title('ARCONS median flat/flux cal\'d sky in counts') #plt.plot(wvls,skySpectrum) #plt.show() ax5 = plt.subplot(223) ax5.set_title('Sensitivity Spectrum') ax5.set_xlim((3000,13000)) ax5.set_ylim((0,5)) plt.plot(wvls,obs.fluxWeights) #ax3 = plt.subplot(234) #ax3.set_title('MKID data minus sky in counts') #plt.plot(wvls,finalSpectrum/max(finalSpectrum)) ax4 = plt.subplot(222) ax4.set_title(stdTitle) plt.plot(wvls,realSpectra) #ax7 = plt.subplot(337) #ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std') #plt.plot(wvls,fluxFactors*subtractedSpectra) pp.savefig() pp.close() #del obs #del sky print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName) txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt" txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum) fullFluxTxtFileName = os.path.join(txtDir,txtFileName) outarr = np.empty((len(medianObsSpectrum),2),dtype=float) outarr[:,0]=wvls outarr[:,1]=medianObsSpectrum #save sensitivity spectrum to file np.savetxt(fullFluxTxtFileName, outarr) print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
class DisplayStack(QMainWindow): def __init__(self): # Start up gui QWidget.__init__(self, parent=None) self.ui = Ui_DisplayStack_gui() self.ui.setupUi(self) # Initialize Variables self.initializeVariables() # Lists and buttons used for specifying run and target information. # Click item in runList to select a run. Run number corresponds to array index. Load up corresponding target list. self.ui.runList.itemClicked.connect(self.selectRun) # Click item in targetList to select a target. self.ui.targetList.itemClicked.connect(self.selectTarget) # Click button in self.ui.targetButton.clicked.connect(self.loadTarget) self.ui.sunsetList.itemClicked.connect(self.createObsList) # Use wavelength calibration checkbox self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal) # Use flat calibration checkbox self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal) # Buttons for obs list creation self.ui.addButton.clicked.connect(self.addObservation) self.ui.removeButton.clicked.connect(self.removeObservation) self.ui.clearButton.clicked.connect(self.createObsList) # Start process button self.ui.stackButton.clicked.connect(self.stackProcess) # Load image stack button self.ui.loadStackButton.clicked.connect(self.chooseStack) # Initialize Variables def initializeVariables(self): # Define path names self.displayStackPath = '/Scratch/DisplayStack/' self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/PAL2012/master_cals/' self.defaultFlatPath = '/Scratch/flatCalSolnFiles/' # Load and display list of run names from /Scratch/DisplayStack/runList.dict self.loadRunData() # Load list of target names from /Scratch/DisplayStack/runName/runName.dict, for all runs self.loadTargetData() # Function to load list of run names. Runs on initialization. def loadRunData(self): # Load run data from /Scratch/DisplayStack/runList.dict self.runData = readDict() self.runData.read_from_file(self.displayStackPath + '/runList.dict') self.runNames = np.array(self.runData['runs']) # Populate runList table with run names. for iRun in range(len(self.runNames)): self.ui.runList.addItem(self.runNames[iRun]) # Function to load a table of target names. def loadTargetData(self): self.targetNames = [] # Cycle through runs and extract target name information from various dictionaries. for iRun in range(len(self.runNames)): self.targetData = readDict() self.targetData.read_from_file(self.displayStackPath + self.runNames[iRun] + '/' + self.runNames[iRun] + '.dict') self.iTargets = np.array(self.targetData['targets']) self.targetNames.append(self.iTargets) # Function to select a run and populate the target list for that particular run. def selectRun(self): # Clear list of target information for previously selected run. self.ui.targetList.clear() # Define a run number by the index of the selected run. self.runNumber = self.ui.runList.row(self.ui.runList.currentItem()) # Populate targetList table with target names for selected run. for iTarget in range(len(self.targetNames[self.runNumber])): self.ui.targetList.addItem(self.targetNames[self.runNumber][iTarget]) def selectTarget(self): self.targetNumber = self.ui.targetList.row(self.ui.targetList.currentItem()) def loadTarget(self): self.run = self.runNames[self.runNumber] self.target = self.targetNames[self.runNumber][self.targetNumber] try: self.paramName = self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.paramData = readDict() self.paramData.read_from_file(self.paramName) self.obsTimes = np.array(self.paramData['obsTimes']) self.utcDates = self.paramData['utcDates'] self.sunsetDates = self.paramData['sunsetDates'] self.calTimestamps = self.paramData['calTimestamps'] self.flatCalDates = self.paramData['flatCalDates'] self.RA = self.paramData['RA'] self.Dec = self.paramData['Dec'] self.hourAngleOffset = self.paramData['HA_offset'] print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.createSunsetList() #self.createObsList() self.createWavelengthList() self.createFlatList() self.paramFileExists = True except IOError: print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict' self.ui.sunsetList.clear() self.ui.obsList.clear() self.ui.inputList.clear() self.ui.wavelengthList.clear() self.ui.flatList.clear() self.paramFileExists = False # Choose Obs File # Create list of available sunset dates def createSunsetList(self): self.ui.sunsetList.clear() for iDate in range(len(self.sunsetDates)): self.ui.sunsetList.addItem(self.sunsetDates[iDate]) # Create Initial Obs file list def createObsList(self): self.ui.obsList.clear() self.ui.inputList.clear() self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())] for iObs in range(len(self.singleDayObservations)): self.ui.obsList.addItem(self.singleDayObservations[iObs]) # Add observation to input list def addObservation(self): self.selectedObs = self.ui.obsList.currentItem() self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs)) self.ui.inputList.addItem(self.selectedObs) self.ui.inputList.sortItems() # Remove observation from input list def removeObservation(self): self.removedObs = self.ui.inputList.currentItem() self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs)) self.ui.obsList.addItem(self.removedObs) self.ui.obsList.sortItems() # Load settings def loadSettings(self): self.validSettings = True # Run and target information self.run = self.runNames[self.runNumber] self.target = self.targetNames[self.runNumber][self.targetNumber] # General settings self.integrationTime = int(self.ui.integrationTimeLine.text()) self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked() self.useHotPixelMasking = self.ui.hotPixelBox.isChecked() # Wavelength calibration settings self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked() self.useBestWavelengthCalibration = self.ui.bestWavelengthCalibrationBox.isChecked() self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text()) self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text()) # Flat calibration settings self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked() self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked() self.fileCount = self.ui.inputList.count() self.weighted = self.useFlatCalibration self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration) self.scaleByEffInt = self.useHotPixelMasking if self.ui.sunsetList.currentItem() != None: if self.fileCount == 0: print 'Please select files to process...' self.validSettings = False if self.useWavelengthCalibration: if self.useBestWavelengthCalibration: print 'Using best wavelength calibration...' elif self.ui.wavelengthList.currentItem() == None: print 'Please select wavelength calibration...' self.validSettings = False else: self.selectedWvlCal = self.ui.wavelengthList.currentItem().text() print "Chose wvl cal: ", str(self.defaultWavelengthPath+self.selectedWvlCal) self.wvlCalFilename = str(self.defaultWavelengthPath+self.selectedWvlCal) #self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln()) if self.useFlatCalibration: if self.ui.flatList.currentItem() == None: print 'Please select flat calibration...' self.validSettings = False else: self.flatCalNight = self.ui.flatList.currentItem().text() self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln()) else: print 'Please select sunset night...' self.validSettings = False # Load hot pixel mask def loadHotMask(self): #self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5') self.hotPixelFilename = str(FileName(obsFile = self.ob).timeMask()) if not os.path.exists(self.hotPixelFilename): hp.findHotPixels(obsFile=self.ob,outputFileName=self.hotPixelFilename) print "Hot pixel mask saved to %s"%(self.hotPixelFilename) self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True) # Create wavelength cal file list def createWavelengthList(self): self.ui.wavelengthList.clear() for iCal in range(len(self.calTimestamps)): self.ui.wavelengthList.addItem(self.calTimestamps[iCal]) # Enable/disable wavecal options def useWaveCal(self): if self.ui.wavelengthCalibrationBox.isChecked(): self.ui.lowerWavelengthCutoffLine.setEnabled(True) self.ui.lowerWavelengthCutoffLabel.setEnabled(True) self.ui.upperWavelengthCutoffLine.setEnabled(True) self.ui.upperWavelengthCutoffLabel.setEnabled(True) self.ui.wavelengthList.setEnabled(True) self.ui.flatCalibrationBox.setEnabled(True) self.ui.flatCalibrationBox.setChecked(True) self.ui.deadPixelBox.setEnabled(True) self.ui.deadPixelBox.setChecked(True) self.ui.flatList.setEnabled(True) self.ui.bestWavelengthCalibrationBox.setEnabled(True) self.ui.bestWavelengthCalibrationBox.setChecked(True) else: self.ui.lowerWavelengthCutoffLine.setEnabled(False) self.ui.lowerWavelengthCutoffLabel.setEnabled(False) self.ui.upperWavelengthCutoffLine.setEnabled(False) self.ui.upperWavelengthCutoffLabel.setEnabled(False) self.ui.wavelengthList.setEnabled(False) self.ui.flatCalibrationBox.setEnabled(False) self.ui.flatCalibrationBox.setChecked(False) self.ui.deadPixelBox.setChecked(False) self.ui.deadPixelBox.setEnabled(False) self.ui.flatList.setEnabled(False) self.ui.bestWavelengthCalibrationBox.setEnabled(False) self.ui.bestWavelengthCalibrationBox.setChecked(False) # Create flat cal file list def createFlatList(self): self.ui.flatList.clear() for iCal in range(len(self.flatCalDates)): self.ui.flatList.addItem(self.flatCalDates[iCal]) # Enable/disable flatcal options def useFlatCal(self): if self.ui.flatCalibrationBox.isChecked(): self.ui.deadPixelBox.setEnabled(True) self.ui.deadPixelBox.setChecked(True) self.ui.flatList.setEnabled(True) else: self.ui.deadPixelBox.setChecked(False) self.ui.deadPixelBox.setEnabled(False) self.ui.flatList.setEnabled(False) # Load dead pixel mask def loadDeadMask(self): self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz') if not os.path.exists(self.deadPixelFilename): self.deadMask = self.ob.getDeadPixels() np.savez(self.deadPixelFilename,deadMask = self.deadMask) print "Dead pixel mask saved to %s"%(self.deadPixelFilename) else: self.deadFile = np.load(self.deadPixelFilename) self.deadMask = self.deadFile['deadMask'] # Describe the structure of the header row class headerDescription(tables.IsDescription): targetName = tables.StringCol(100, dflt='') run = tables.StringCol(100, dflt='') obsFileName = tables.StringCol(100, dflt='') wvlCalFileName = tables.StringCol(100, dflt=np.nan) flatCalFileName = tables.StringCol(100, dflt='') deadPixFileName = tables.StringCol(100, dflt='') hotPixFileName = tables.StringCol(100, dflt='') nCol = tables.UInt32Col(dflt=-1) nRow = tables.UInt32Col(dflt=-1) lowWvlCutoff = tables.Float64Col(dflt=np.nan) highWvlCutoff = tables.Float64Col(dflt=np.nan) exptime = tables.Float64Col(dflt=np.nan) lst = tables.StringCol(100, dflt='') integrationTime = tables.Float64Col(dflt=np.nan) RA = tables.StringCol(100, dflt='') Dec = tables.StringCol(100, dflt='') HA_offset = tables.Float64Col(dflt=0.0) # Create output name def createOutputName(self): self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's') if self.useWavelengthCalibration and self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.h5') elif self.useWavelengthCalibration and not self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.h5') elif not self.useWavelengthCalibration and self.useHotPixelMasking: self.outputFilename = str(self.rawName + '_hp.h5') else: self.outputFilename = str(self.rawName + '.h5') def createH5File(self): # Create header and data group and table names headerGroupName = 'header' headerTableName = 'header' dataGroupName = 'stack' dataTableName = 'stack' timeTableName = 'time' # Create lookup names for header information runColName = 'run' targetColName = 'targetName' obsFileColName = 'obsFileName' wvlCalFileColName = 'wvlCalFileName' flatCalFileColName = 'flatCalFileName' nRowColName = 'nRow' nColColName = 'nCol' RAColName = 'RA' DecColName = 'Dec' deadPixColName = 'deadPixFileName' hotPixColName = 'hotPixFileName' lowWvlColName = 'lowWvlCutoff' highWvlColName = 'highWvlCutoff' expTimeColName = 'exptime' lstColName = 'lst' integrationTimeColName = 'integrationTime' HA_offsetColName = 'HA_offset' # Create and h5 output file, create header and data groups fileh = tables.openFile(self.outputFilename, mode='w') headerGroup = fileh.createGroup("/", headerGroupName, 'Header') stackGroup = fileh.createGroup("/", dataGroupName, 'Image Stack') # Create row for header information headerTable = fileh.createTable(headerGroup, headerTableName, self.headerDescription, 'Header Info') header = headerTable.row # Fill in the header with possibly useful information. header[runColName] = self.run header[targetColName] = self.target header[obsFileColName] = self.obsFn header[nColColName] = self.numberCols header[nRowColName] = self.numberRows header[RAColName] = self.RA header[DecColName] = self.Dec header[expTimeColName] = self.exptime header[lstColName] = self.lst header[integrationTimeColName] = self.integrationTime header[HA_offsetColName] = self.hourAngleOffset if self.useDeadPixelMasking: header[deadPixColName] = self.deadPixelFilename if self.useHotPixelMasking: header[hotPixColName] = self.hotPixelFilename if self.useWavelengthCalibration: header[wvlCalFileColName] = self.ob.wvlCalFileName header[lowWvlColName] = self.lowerWavelengthCutoff header[highWvlColName] = self.upperWavelengthCutoff if self.useFlatCalibration: header[flatCalFileColName] = self.flatCalFilename header.append() # Create an h5 array for the midtime of each frame in the image cube. timeTable = fileh.createCArray(stackGroup, timeTableName, Float64Atom(), (1,len(self.times))) timeTable[:] = self.times # Create an h5 table for the image cube. stackTable = fileh.createCArray(stackGroup, dataTableName, Float64Atom(), (self.numberRows,self.numberCols, self.cube.shape[2])) stackTable[:] = self.cube # Flush the h5 output file fileh.flush() fileh.close() # Start process for creating image stacks def stackProcess(self): # Check for valid params file if self.paramFileExists: # Load settings choosen from gui self.loadSettings() if self.validSettings: # Loop through all files in input list for iFile in range(self.fileCount): # Create ObsFile instance self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text() self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs()) print 'Processing file ' + self.obsFn + '...' self.ob = ObsFile(self.obsFn) self.numberRows = self.ob.nRow self.numberCols = self.ob.nCol # Load time adjustment file if self.useTimeAdjustment: print 'Loading time adjustment file...' self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments()) # Load hot pixel mask if self.useHotPixelMasking: print 'Loading hot pixel mask...' self.loadHotMask() # Load wave cal solution if self.useWavelengthCalibration: if self.useBestWavelengthCalibration: print 'Loading best wavelength calibration...' self.ob.loadBestWvlCalFile() else: print 'Loading selected wavelength calibration...' self.ob.loadWvlCalFile(self.wvlCalFilename) # Load flatcal solution if self.useFlatCalibration: print 'Loading flat calibration...' self.ob.loadFlatCalFile(self.flatCalFilename) # Load dead pixel mask if self.useDeadPixelMasking: print 'Loading dead pixel mask...' self.loadDeadMask() # Set wavelength cutoffs if self.useWavelengthCalibration: print 'Setting wavelength cutoffs...' self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff) # Timing self.unix = self.ob.getFromHeader('unixtime') self.startJD = self.unix/86400.+2440587.5 self.exptime = self.ob.getFromHeader('exptime') self.lst = self.ob.getFromHeader('lst') self.times = [] self.frames = [] # Create Image Stack print 'Stacking images...' for iSec in np.arange(0,self.exptime,self.integrationTime): #add seconds offset to julian date, move jd to center of bin self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.) self.times.append(self.jd) print 'Creating frame for time ' + str(self.jd) self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt) self.frame = self.frameData['image'] if self.ui.verticalFlipBox.isChecked(): self.frame = np.flipud(self.frame) if self.useDeadPixelMasking: self.frame[self.deadMask == 0] = np.nan self.frames.append(self.frame) self.cube = np.dstack(self.frames) self.times = np.array(self.times) # Create output file self.createOutputName() print 'Saving image stack to ' + self.outputFilename self.createH5File() # Invalid params file else: print 'Invalid parameter file...' # Choose an image stack def chooseStack(self): self.defaultLoadStackDirectory = str(self.displayStackPath) self.stackName = '' self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("H5 (*.h5)")) if self.stackName == '': print 'No file chosen' else: loadStackApp = LoadImageStack.LoadImageStack(stackName = self.stackName) loadStackApp.show() loadStackApp.exec_()
def main(): #open the sky file for hr9087 run = 'PAL2012' date = '20121210' wvlCal = '20121211-052230' obsTimestamp = '20121211-051650' flatCalDate = '20121211' flatCalTstamp = '20121212-074700' obsFN = FileName(run=run,date=date,tstamp=obsTimestamp) obsFileName = obsFN.obs() timeMaskFileName = obsFN.timeMask() wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln() flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln() if not os.path.exists(timeMaskFileName): print 'Running hotpix for ',obsFileName hp.findHotPixels(obsFileName,timeMaskFileName) print "Flux file pixel mask saved to %s"%(timeMaskFileName) obs = ObsFile(obsFileName) obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments()) obs.loadWvlCalFile(wvlFileName) obs.loadFlatCalFile(flatFileName) obs.loadHotPixCalFile(timeMaskFileName) #obs.setWvlCutoffs(4000,8000) #get image before and after flat cal print 'getting images' beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True) rawCubeDict = obs.getSpectralCube(weighted=False) rawCube = np.array(rawCubeDict['cube'],dtype=np.double) effIntTime = rawCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) rawCube *= maxIntTime / effIntTime3d rawCube[np.isnan(rawCube)] = 0 rawCube[rawCube == np.inf] = 0 beforeImg = np.sum(rawCube,axis=-1) print 'finished first cube' flatCubeDict = obs.getSpectralCube(weighted=True) flatCube = np.array(flatCubeDict['cube'],dtype=np.double) effIntTime = flatCubeDict['effIntTime'] maxIntTime = np.max(effIntTime) #add third dimension for broadcasting effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) flatCube *= maxIntTime / effIntTime3d flatCube[np.isnan(flatCube)] = 0 flatCube[flatCube == np.inf] = 0 afterImg = np.sum(flatCube,axis=-1) plotArray(title='before flatcal',image=beforeImg) plotArray(title='after flatcal',image=afterImg) print 'before sdev',np.std(beforeImg[afterImg!=0]) print 'after sdev',np.std(afterImg[afterImg!=0]) np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) > 2: fileNum = str(sys.argv[2]) else: fileNum = "0" # science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile, "r") for line in f: params.append(line) f.close() datadir = params[0].split("=")[1].strip() flatdir = params[1].split("=")[1].strip() wvldir = params[2].split("=")[1].strip() obsfile = params[3].split("=")[1].strip() skyfile = params[4].split("=")[1].strip() flatfile = params[5].split("=")[1].strip() wvlfile = params[6].split("=")[1].strip() objectName = params[9].split("=")[1].strip() # wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121205" # objectName = "crabNight1" if len(params) > 10: xpix = int(params[10].split("=")[1].strip()) ypix = int(params[11].split("=")[1].strip()) apertureRadius = int(params[12].split("=")[1].strip()) startTime = int(params[13].split("=")[1].strip()) intTime = int(params[14].split("=")[1].strip()) obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) obs = ObsFile(obsFileName) # obs.loadWvlCalFile(wvlCalFileName) obs.loadBestWvlCalFile() obs.loadFlatCalFile(flatCalFileName) print "analyzing file %s" % (obsFileName) print "loaded data file and calibrations\n---------------------\n" nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") # wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow) # nWvlBins=len(wvlBinEdges)-1 # print np.shape(obsSpectra) # print nRow # print nCol # print nWvlBins # Apply Hot pixel masking before getting dead time correction # HotPixFile = getTimeMaskFileName(obsFileName) HotPixFile = FileName(obsFile=obs).timeMask() print "making hot pixel file ", HotPixFile if not os.path.exists(HotPixFile): # check if hot pix file already exists hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile) print "Flux file pixel mask saved to %s" % (HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s" % (HotPixFile) # GET RAW PIXEL COUNT IMAGE TO CALCULATE CORRECTION FACTORS print "Making raw cube to get dead time correction" cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=False) cube = np.array(cubeDict["cube"], dtype=np.double) wvlBinEdges = cubeDict["wvlBinEdges"] effIntTime = cubeDict["effIntTime"] print "median effective integration time = ", np.median(effIntTime) nWvlBins = len(wvlBinEdges) - 1 print "cube shape ", np.shape(cube) print "effIntTime shape ", np.shape(effIntTime) # add third dimension to effIntTime for broadcasting effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,)) # put cube into counts/s in each pixel cube /= effIntTime # CALCULATE DEADTIME CORRECTION # NEED TOTAL COUNTS PER SECOND FOR EACH PIXEL TO DO PROPERLY # ASSUMES SAME CORRECTION FACTOR APPLIED FOR EACH WAVELENGTH, MEANING NO WL DEPENDANCE ON DEAD TIME EFFECT DTCorr = np.zeros((np.shape(cube)[0], np.shape(cube)[1]), dtype=float) for f in range(0, np.shape(cube)[2]): print cube[:, :, f] print "-----------------------" DTCorr += cube[:, :, f] print DTCorr print "\n=====================\n" # Correct for 100 us dead time DTCorrNew = DTCorr / (1 - DTCorr * 100e-6) CorrFactors = DTCorrNew / DTCorr # This is what the frames need to be multiplied by to get their true values print "Dead time correction factors = " print CorrFactors # REMAKE CUBE WITH FLAT WEIGHTS AND APPLY DEAD TIME CORRECTION AS WELL print "Making Weighted cube" # load/generate hot pixel mask file # HotPixFile = getTimeMaskFileName(obsFileName) HotPixFile = FileName(obsFile=obs).timeMask() if not os.path.exists(HotPixFile): # check if hot pix file already exists hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile) print "Flux file pixel mask saved to %s" % (HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s" % (HotPixFile) cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=False) # cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=True) cube = np.array(cubeDict["cube"], dtype=np.double) wvlBinEdges = cubeDict["wvlBinEdges"] effIntTime = cubeDict["effIntTime"] print "median effective integration time = ", np.median(effIntTime) nWvlBins = len(wvlBinEdges) - 1 print "cube shape ", np.shape(cube) print "effIntTime shape ", np.shape(effIntTime) # add third dimension to effIntTime for broadcasting effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,)) # put cube into counts/s in each pixel cube /= effIntTime # add third dimension to CorrFactors for broadcasting CorrFactors = np.reshape(CorrFactors, np.shape(CorrFactors) + (1,)) # apply dead time correction factors cube *= CorrFactors # calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins), dtype=float) for n in xrange(nWvlBins): binsize = wvlBinEdges[n + 1] - wvlBinEdges[n] wvls[n] = wvlBinEdges[n] + (binsize / 2.0) print "wvls ", wvls # reshape cube for makeMovie movieCube = np.zeros((nWvlBins, np.shape(cube)[0], np.shape(cube)[1]), dtype=float) for i in xrange(nWvlBins): movieCube[i, :, :] = cube[:, :, i] # show individual frames as they are made to debug # plt.matshow(movieCube[i],vmin = 0, vmax = 100) # plt.show() print "movieCube shape ", np.shape(movieCube) print "wvls shape ", np.shape(wvls) # print cube # print "--------------------------" # print movieCube outdir = "/home/srmeeker/scratch/standards/" np.savez(outdir + "%s_raw_%s.npz" % (objectName, fileNum), stack=movieCube, wvls=wvls) utils.makeMovie( movieCube, frameTitles=wvls, cbar=True, outName=outdir + "%s_raw_%s.gif" % (objectName, fileNum), normMin=0, normMax=50, ) """
def main(): """ params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() fluxdir = params[2].split('=')[1].strip() wvldir = params[3].split('=')[1].strip() obsfile = params[4].split('=')[1].strip() skyfile = params[5].split('=')[1].strip() flatfile = params[6].split('=')[1].strip() fluxfile = params[7].split('=')[1].strip() wvlfile = params[8].split('=')[1].strip() objectName = params[9].split('=')[1].strip() fluxCalObject = params[10].split('=')[1].strip() obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) fluxCalFileName = os.path.join(fluxdir, fluxfile) """ if len(sys.argv) >2: fileNum = str(sys.argv[2]) else: fileNum = '0' #science object parameter file params = [] paramfile = sys.argv[1] f = open(paramfile,'r') for line in f: params.append(line) f.close() datadir = params[0].split('=')[1].strip() flatdir = params[1].split('=')[1].strip() wvldir = params[2].split('=')[1].strip() obsfile = params[3].split('=')[1].strip() skyfile = params[4].split('=')[1].strip() flatfile = params[5].split('=')[1].strip() wvlfile = params[6].split('=')[1].strip() objectName = params[9].split('=')[1].strip() wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121206" if len(params)>10: xpix = int(params[10].split('=')[1].strip()) ypix = int(params[11].split('=')[1].strip()) apertureRadius = int(params[12].split('=')[1].strip()) startTime = int(params[13].split('=')[1].strip()) intTime =int(params[14].split('=')[1].strip()) obsFileName = os.path.join(datadir, obsfile) skyFileName = os.path.join(datadir, skyfile) wvlCalFileName = os.path.join(wvldir, wvlfile) flatCalFileName = os.path.join(flatdir, flatfile) obs = ObsFile(obsFileName) obs.loadWvlCalFile(wvlCalFileName) obs.loadFlatCalFile(flatCalFileName) print "analyzing file %s"%(obsFileName) print "loaded data file and calibrations\n---------------------\n" nRow = obs.nRow nCol = obs.nCol obsTime = obs.getFromHeader("exptime") #wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow) #nWvlBins=len(wvlBinEdges)-1 #print np.shape(obsSpectra) #print nRow #print nCol #print nWvlBins #load/generate hot pixel mask file HotPixFile = getTimeMaskFileName(obsFileName) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFileName,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obs.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) ####### #EVERYTHING BEFORE HERE IS STANDARD FILE/CALFILE LOADING startWvl = 3000 #stopWvl = 7000 #for V-band stopWvl = 9000 #for R-band print "Making spectral cube" #for pg0220 first sec should be 80 since object is moving around before this #for pg0220A first sec should be 70, integration time is 140 #for landolt 9542 first sec should be 20, int time is -1 cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, wvlStart = startWvl, wvlStop = stopWvl, wvlBinEdges = [startWvl,stopWvl], weighted=False) cube= np.array(cubeDict['cube'], dtype=np.double) wvlBinEdges= cubeDict['wvlBinEdges'] effIntTime= cubeDict['effIntTime'] print "median effective integration time = ", np.median(effIntTime) nWvlBins=len(wvlBinEdges)-1 print "cube shape ", np.shape(cube) print "effIntTime shape ", np.shape(effIntTime) #add third dimension to effIntTime for broadcasting effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,)) cube /= effIntTime #put cube into counts/s #calculate midpoints of wvl bins for plotting wvls = np.empty((nWvlBins),dtype=float) for n in xrange(nWvlBins): binsize=wvlBinEdges[n+1]-wvlBinEdges[n] wvls[n] = (wvlBinEdges[n]+(binsize/2.0)) print "wvls ",wvls #reshape cube for makeMovie movieCube = np.zeros((nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float) for i in xrange(nWvlBins): movieCube[i,:,:] = cube[:,:,i] #show individual frames as they are made to debug #plt.matshow(movieCube[i],vmin = 0, vmax = 100) #plt.show() print "movieCube shape ", np.shape(movieCube) print "wvls shape ", np.shape(wvls) #print cube #print "--------------------------" #print movieCube print "adding frames with wvl below ", stopWvl finalframe = np.zeros((1,np.shape(movieCube)[1],np.shape(movieCube)[2])) for f in xrange(len(wvls[wvls<stopWvl])): print wvls[f] finalframe[0]+=movieCube[f] plt.matshow(movieCube[f],vmin=0,vmax = 40) plt.show() movieCube = finalframe np.savez('%s_%s.npz'%(objectName,fileNum),stack=movieCube,wvls=wvls) print "Saved frame to .npz file" plt.matshow(movieCube[0],vmin=0,vmax = 40) plt.show()
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False): ''' Test run of obsFile.writePhotonList. fileName can be used to specify the output file name. If not specified, default name/location is used. Now includes test for pixel remapping.... ''' #Details of example obs file to run test on. # run = 'PAL2012' # date = '20121207' # tstamp = '20121208-074649' # calTstamp='20121208-070505' # fluxTstamp='20121208-133002' run = 'PAL2013' date = '20131209' tstamp = '20131209-115553' centroidTstamp = tstamp calTstamp='20131209-132225' fluxTstamp='ones' flatTstamp='20131209' if doPixRemap==True: pixRemapFileName = FileName(run=run).pixRemap() else: pixRemapFileName = None #Load up the obs file obsFileName = FileName(run=run, date=date, tstamp=tstamp) obsFile = ObsFile(obsFileName.obs()) obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln()) obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln()) obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln()) HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask() if not os.path.exists(HotPixFile): #check if hot pix file already exists hp.findHotPixels(obsFileName.obs(),HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) obsFile.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) #hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8'' centroid_RA = '12:13:56.42' centroid_DEC = '32:56:30.8' CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList() #print "checking centroid file", CentFile #if not os.path.exists(CentFile): #check if cent pix file already exists cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10) # print "Flux file centroid cal saved to %s"%(CentFile) obsFile.loadCentroidListFile(CentFile) print "Centroid File loaded %s"%(CentFile) #Load up associated calibrations #obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments()) #obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask()) #obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList()) #And write out the results.... obsFile.writePhotonList(outputFileName,firstSec,integrationTime, pixRemapFileName=pixRemapFileName) #Read the results back in.... #photFile = photList.PhotFile(outputFilename) print "Wrote photon list file: ", outputFileName
ffn = flatCalFilenames[iSeq] sunsetDate = sunsetDates[iSeq] for i,ts in enumerate(timestampList): print 'loading',ts obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs() ob = ObsFile(obsFn) ob.loadTimeAdjustmentFile(FileName(run=run).timeAdjustments()) index1 = obsFn.find('_') hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:] if not os.path.exists(hotPixFn): hp.findHotPixels(obsFn,hotPixFn) print "Flux file pixel mask saved to %s"%(hotPixFn) ob.loadHotPixCalFile(hotPixFn,switchOnMask=True) ob.loadWvlCalFile(wfn) ob.loadFlatCalFile(ffn) ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff) bad_solution_mask=np.zeros((46,44)) bad_count=0; for y in range(46): for x in range(44): if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff): bad_solution_mask[y][x] = 1 unix = ob.getFromHeader('unixtime') startJD = unix/86400.+2440587.5 nSecInFile = ob.getFromHeader('exptime') #tic = time() deadMask = ob.getDeadPixels()
timestampLists = [[utcDate+'-'+str(ts) for ts in seq] for utcDate,seq in zip(utcDates,seqs)] wvlCalFilenames = [FileName(run=run,date=sunsetDate,tstamp=calTimestamp).calSoln() for sunsetDate,calTimestamp in zip(sunsetDates,calTimestamps)] flatCalFilenames = [FileName(run=run,date=sunsetDate,tstamp=calTimestamp).flatSoln() for sunsetDate,calTimestamp in zip(sunsetDates,calTimestamps)] #wvlCalFilenames[0] = '/Scratch/waveCalSolnFiles/20121210/calsol_20121211-074031.h5' #wvlCalFilenames[1] = '/home/danica/optimusP/testing/forMatt/calsol_20121211-044853.h5' flatCalFilenames[0] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5' flatCalFilenames[1] = '/Scratch/flatCalSolnFiles/20121207/flatsol_20121207.h5' fluxCalFileNames = ['/Scratch/fluxCalSolnFiles/20121206/fluxsol_20121207-124034.h5'] obsFn = FileName(run=run,date=sunsetDates[0],tstamp='20121209-120530').obs() ob = ObsFile(obsFn) print 'Loading wavelength calibration solution: ' + wvlCalFilenames[0] ob.loadWvlCalFile(wvlCalFilenames[0]) print 'Loading flat calibration solution: ' + flatCalFilenames[0] ob.loadFlatCalFile(flatCalFilenames[0]) ob.loadFluxCalFile(fluxCalFileNames[0]) #load/generate hot pixel mask file HotPixFile = getTimeMaskFileName(obsFn) if not os.path.exists(HotPixFile): hp.findHotPixels(obsFn,HotPixFile) print "Flux file pixel mask saved to %s"%(HotPixFile) ob.loadHotPixCalFile(HotPixFile) print "Hot pixel mask loaded %s"%(HotPixFile) frame = ob.getPixelCountImage(firstSec=0,integrationTime=300,weighted=True) #hotPixMask = hotPixels.checkInterval(image=frame, firstSec=0, intTime=300, weighted=True, display=False)['mask'] #summed_array,bin_edges=ob.getApertureSpectrum(pixelCol=14,pixelRow=8,radius=7) ob.plotApertureSpectrum(pixelCol=14,pixelRow=8,radius=7,weighted = True,fluxWeighted=True,lowCut=3000,highCut=9000)