def main():
#load the current PAL2012 beammap from an obs file from that run,
#a sky file for hr9087
run = 'PAL2012'
date = '20121210'
obsTimestamp = '20121211-051650'
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
obs = ObsFile(obsFileName)
#Load an existing definition of which pixels to remap where
pixMap = remapPixels.PixelMap(obsFN.pixRemap())
pixMapSourceList,pixMapDestList = pixMap.getRemappedPix()
#create a new h5 file and copy the existing beammap node hierarchy into it
newBeammapPath = 'beamimage_PAL2012_corrected.h5'
newBeammapFile = tables.openFile(newBeammapPath, mode='w')
newBeammapFile.copyNode(obs.file.root.beammap, newparent=newBeammapFile.root, newname='beammap', recursive=True)
newBeammap = newBeammapFile.root.beammap.beamimage
#for each array in the h5, move the pixels according to pixMap
newBeammap[:,:] = pixMap.remapArray(obs.beamImage)
newBeammapFile.root.beammap.resfreq[:,:] = pixMap.remapArray(newBeammapFile.root.beammap.resfreq.read())
newBeammapFile.root.beammap.atten[:,:] = pixMap.remapArray(newBeammapFile.root.beammap.atten.read())
newBeammapFile.flush()
newBeammapFile.close()
obs.file.close()
def testLoadBeammap():
'''
Test if a remapped beammap is actually remapped
'''
#open an obs file from PAL2012,the sky file for hr9087
#we'll use the embedded beammap file, which has some misplaced pixels
run = 'PAL2012'
date = '20121210'
obsTimestamp = '20121211-051650'
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
obs = ObsFile(obsFileName)
beammapFileName = obsFN.beammap()
#load the PixelMap for PAL2012 to know which pixels should be remapped
pixMap = remapPixels.PixelMap(obsFN.pixRemap())
pixMapSourceList,pixMapDestList = pixMap.getRemappedPix()
#load the corrected beammap into the obs
obs.loadBeammapFile(beammapFileName)
#check that each pixel that should be moved is moved
#by comparing the embedded beammap and the loaded corrected one
for source,dest in zip(pixMapSourceList,pixMapDestList):
assert obs.beamImage[dest] == obs.file.root.beammap.beamimage[source]
obs.file.close()
def makemovie1(self):
run = self.s['run']
sundownDate = self.s['sundownDate']
obsDate = self.s['obsDate']
stride = int(self.s['stride'])
seq5 = self.s['seq5'].split()
for seq in seq5:
inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb")
cosmicTimeList = pickle.load(inFile)
binContents = pickle.load(inFile)
cfn = "cosmicMask-%s.h5"%seq
intervals = ObsFile.readCosmicIntervalFromFile(cfn)
for interval in intervals:
print "interval=",interval
fn = FileName(run, sundownDate,obsDate+"-"+seq)
obsFile = ObsFile(fn.obs())
obsFile.loadTimeAdjustmentFile(fn.timeAdjustments())
i0=interval[0]
i1=interval[1]
intervalTime = i1-i0
dt = intervalTime/2
beginTime = max(0,i0-0.000200)
endTime = beginTime + 0.001
integrationTime = endTime-beginTime
nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
timeHgValues = np.zeros(nBins, dtype=np.int64)
ymax = sys.float_info.max/100.0
for iRow in range(obsFile.nRow):
for iCol in range(obsFile.nCol):
gtpl = obsFile.getTimedPacketList(iRow,iCol,
beginTime,integrationTime)
ts = (gtpl['timestamps'] - beginTime)*obsFile.ticksPerSec
ts64 = np.round(ts).astype(np.uint64)
tsBinner.tsBinner(ts64, timeHgValues)
plt.clf()
plt.plot(timeHgValues, label="data")
x0 = (i0-beginTime)*obsFile.ticksPerSec
x1 = (i1-beginTime)*obsFile.ticksPerSec
plt.fill_between((x0,x1),(0,0), (ymax,ymax), alpha=0.2, color='red')
plt.yscale("symlog",linthreshy=0.9)
plt.xlim(0,1000)
plt.ylim(-0.1,300)
tick0 = int(np.round(i0*obsFile.ticksPerSec))
plotfn = "cp-%05d-%s-%s-%s-%09d"%(timeHgValues.sum(),run,obsDate,seq,tick0)
plt.title(plotfn)
plt.legend()
plt.savefig(plotfn+".png")
plt.xlabel("nSigma=%d stride=%d threshold=%d"%(int(self.s['nSigma']),int(self.s['stride']),int(self.s['threshold'])))
print "plotfn=",plotfn
os.system("convert -delay 0 `ls -r cp*png` cp.gif")
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=True):
'''
Test run of obsFile.writePhotonList. fileName can be used
to specify the output file name. If not specified, default
name/location is used.
Now includes test for pixel remapping....
'''
#Details of example obs file to run test on.
# run = 'PAL2012'
# date = '20121207'
# tstamp = '20121208-074649'
# calTstamp='20121208-070505'
# fluxTstamp='20121208-133002'
run = 'PAL2012'
date = '20121208'
tstamp = '20121209-120530'
centroidTstamp = '20121209-120530'
calTstamp='20121209-131132'
fluxTstamp='20121209-020416'
flatTstamp='20121209-021036'
if doPixRemap==True:
pixRemapFileName = FileName(run=run).pixRemap()
else:
pixRemapFileName = None
#Load up the obs file
obsFileName = FileName(run=run, date=date, tstamp=tstamp)
obsFile = ObsFile(obsFileName.obs())
#Load up associated calibrations
obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())
obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
#And write out the results....
obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
pixRemapFileName=pixRemapFileName)
sigList = []
run = 'PAL2012'
sundownDate = '20121211'
obsDate = '20121212'
for seq in seq5:
inFile = open("cosmicTimeList-%s.pkl"%(seq),"rb")
cosmicTimeList = pickle.load(inFile)
binContents = pickle.load(inFile)
cfn = "cosmicMax-%s.h5"%seq
intervals = ObsFile.readCosmicIntervalFromFile(cfn)
for interval in intervals:
print "interval=",interval
fn = FileName(run, sundownDate,obsDate+"-"+seq)
obsFile = ObsFile(fn.obs())
obsFile.loadTimeAdjustmentFile(fn.timeAdjustments())
i0=interval[0]
i1=interval[1]
intervalTime = i1-i0
dt = intervalTime/2
beginTime = max(0,i0-0.000200)
endTime = beginTime + 0.001
integrationTime = endTime-beginTime
nBins = int(np.round(obsFile.ticksPerSec*(endTime-beginTime)+1))
timeHgValues = np.zeros(nBins, dtype=np.int64)
ymax = sys.float_info.max/100.0
for iRow in range(obsFile.nRow):
for iCol in range(obsFile.nCol):
gtpl = obsFile.getTimedPacketList(iRow,iCol,
beginTime,integrationTime)
def __init__(self,paramFile,plots=False,verbose=False):
"""
Opens flux file, prepares standard spectrum, and calculates flux factors for the file.
Method is provided in param file. If 'relative' is selected, an obs file with standard star defocused over
the entire array is expected, with accompanying sky file to do sky subtraction.
If any other method is provided, 'absolute' will be done by default, wherein a point source is assumed
to be present. The obs file is then broken into spectral frames with photometry (psf or aper) performed
on each frame to generate the ARCONS observed spectrum.
"""
self.verbose=verbose
self.plots = plots
self.params = readDict()
self.params.read_from_file(paramFile)
run = self.params['run']
sunsetDate = self.params['fluxSunsetLocalDate']
self.fluxTstamp = self.params['fluxTimestamp']
skyTstamp = self.params['skyTimestamp']
wvlSunsetDate = self.params['wvlCalSunsetLocalDate']
wvlTimestamp = self.params['wvlCalTimestamp']
flatCalFileName = self.params['flatCalFileName']
needTimeAdjust = self.params['needTimeAdjust']
self.deadtime = float(self.params['deadtime']) #from firmware pulse detection
self.timeSpacingCut = self.params['timeSpacingCut']
bLoadBeammap = self.params.get('bLoadBeammap',False)
self.method = self.params['method']
self.objectName = self.params['object']
self.r = float(self.params['energyResolution'])
self.photometry = self.params['photometry']
self.centroidRow = self.params['centroidRow']
self.centroidCol = self.params['centroidCol']
self.aperture = self.params['apertureRad']
self.annulusInner = self.params['annulusInner']
self.annulusOuter = self.params['annulusOuter']
self.collectingArea = self.params['collectingArea']
self.startTime = self.params['startTime']
self.intTime = self.params['integrationTime']
fluxFN = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp)
self.fluxFileName = fluxFN.obs()
self.fluxFile = ObsFile(self.fluxFileName)
if self.plots:
self.plotSavePath = os.environ['MKID_PROC_PATH']+os.sep+'fluxCalSolnFiles'+os.sep+run+os.sep+sunsetDate+os.sep+'plots'+os.sep
if not os.path.exists(self.plotSavePath): os.mkdir(self.plotSavePath)
if self.verbose: print "Created directory %s"%self.plotSavePath
obsFNs = [fluxFN]
self.obsList = [self.fluxFile]
if self.startTime in ['',None]: self.startTime=0
if self.intTime in ['',None]: self.intTime=-1
if self.method=="relative":
try:
print "performing Relative Flux Calibration"
skyFN = FileName(run=run,date=sunsetDate,tstamp=skyTstamp)
self.skyFileName = skyFN.obs()
self.skyFile = ObsFile(self.skyFileName)
obsFNs.append(skyFN)
self.obsList.append(self.skyFile)
except:
print "For relative flux calibration a sky file must be provided in param file"
self.__del__()
else:
self.method='absolute'
print "performing Absolute Flux Calibration"
if self.photometry not in ['aperture','PSF']: self.photometry='PSF' #default to PSF fitting if no valid photometry selected
timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
timeAdjustFileName = FileName(run=run).timeAdjustments()
#make filename for output fluxCalSoln file
self.fluxCalFileName = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp).fluxSoln()
print "Creating flux cal: %s"%self.fluxCalFileName
if wvlSunsetDate != '':
wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
if flatCalFileName =='':
flatCalFileName=FileName(obsFile=self.fluxFile).flatSoln()
#load cal files for flux file and, if necessary, sky file
for iObs,obs in enumerate(self.obsList):
if bLoadBeammap:
print 'loading beammap',os.environ['MKID_BEAMMAP_PATH']
obs.loadBeammapFile(os.environ['MKID_BEAMMAP_PATH'])
if wvlSunsetDate != '':
obs.loadWvlCalFile(wvlCalFileName)
else:
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatCalFileName)
obs.setWvlCutoffs(-1,-1)
if needTimeAdjust:
obs.loadTimeAdjustmentFile(timeAdjustFileName)
timeMaskFileName = timeMaskFileNames[iObs]
print timeMaskFileName
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obs
hp.findHotPixels(obsFile=obs,outputFileName=timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
print "Flux cal/sky file pixel mask saved to %s"%(timeMaskFileName)
obs.loadHotPixCalFile(timeMaskFileName)
if self.verbose: print "Loaded hot pixel file %s"%timeMaskFileName
#get flat cal binning information since flux cal will need to match it
self.wvlBinEdges = self.fluxFile.flatCalFile.root.flatcal.wavelengthBins.read()
self.nWvlBins = self.fluxFile.flatWeights.shape[2]
self.binWidths = np.empty((self.nWvlBins),dtype=float)
self.binCenters = np.empty((self.nWvlBins),dtype=float)
for i in xrange(self.nWvlBins):
self.binWidths[i] = self.wvlBinEdges[i+1]-self.wvlBinEdges[i]
self.binCenters[i] = (self.wvlBinEdges[i]+(self.binWidths[i]/2.0))
if self.method=='relative':
print "Extracting ARCONS flux and sky spectra"
self.loadRelativeSpectrum()
print "Flux Spectrum loaded"
self.loadSkySpectrum()
print "Sky Spectrum loaded"
elif self.method=='absolute':
print "Extracting ARCONS point source spectrum"
self.loadAbsoluteSpectrum()
print "Loading standard spectrum"
try:
self.loadStdSpectrum(self.objectName)
except KeyError:
print "Invalid spectrum object name"
self.__del__()
sys.exit()
print "Generating sensitivity curve"
self.calculateFactors()
print "Sensitivity Curve calculated"
print "Writing fluxCal to file %s"%self.fluxCalFileName
self.writeFactors(self.fluxCalFileName)
if self.plots: self.makePlots()
print "Done"
from util.ObsFile import ObsFile
import os
import hotpix.hotPixels as hp
from util.FileName import FileName
from util.popup import plotArray
import astrometry.CentroidCalc as cc
from photonlist.photlist import writePhotonList
from headers.ArconsHeaders import PulsarPhotonList
if __name__=='__main__':
run = 'PAL2014'
date = '20140924'
tstamp = '20140925-112538'
obsFN = FileName(run=run,date=date,tstamp=tstamp)
obsPath = obsFN.obs()
flatPath = FileName(run=run,date=date).flatSoln()
hotPath = obsFN.timeMask()
centroidPath = obsFN.centroidList()
fluxPath = FileName(run='PAL2012',date='20121211',tstamp='absolute_021727').fluxSoln()
obs = ObsFile(obsPath)
if not os.path.exists(hotPath):
hp.findHotPixels(obsPath,hotPath)
obs.loadHotPixCalFile(hotPath,switchOnMask=False)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatPath)
obs.loadFluxCalFile(fluxPath)
obs.loadCentroidListFile(centroidPath)
obs.setWvlCutoffs(3000,11000)
def testWritePhotonList(outputFileName=None,firstSec=0,integrationTime=-1,doPixRemap=False):
'''
Test run of obsFile.writePhotonList. fileName can be used
to specify the output file name. If not specified, default
name/location is used.
Now includes test for pixel remapping....
'''
#Details of example obs file to run test on.
# run = 'PAL2012'
# date = '20121207'
# tstamp = '20121208-074649'
# calTstamp='20121208-070505'
# fluxTstamp='20121208-133002'
run = 'PAL2013'
date = '20131209'
tstamp = '20131209-115553'
centroidTstamp = tstamp
calTstamp='20131209-132225'
fluxTstamp='ones'
flatTstamp='20131209'
if doPixRemap==True:
pixRemapFileName = FileName(run=run).pixRemap()
else:
pixRemapFileName = None
#Load up the obs file
obsFileName = FileName(run=run, date=date, tstamp=tstamp)
obsFile = ObsFile(obsFileName.obs())
obsFile.loadWvlCalFile(FileName(run=run,date=date,tstamp=calTstamp).calSoln())
obsFile.loadFlatCalFile(FileName(run=run,date=date,tstamp=flatTstamp).flatSoln())
obsFile.loadFluxCalFile(FileName(run=run,date=date,tstamp=fluxTstamp).fluxSoln())
HotPixFile = FileName(run=run,date=date,tstamp=tstamp).timeMask()
if not os.path.exists(HotPixFile): #check if hot pix file already exists
hp.findHotPixels(obsFileName.obs(),HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obsFile.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
#hz21 location alpha(2000) = 12h 13m 56.42s , delta(2000) = +32d 56' 30.8''
centroid_RA = '12:13:56.42'
centroid_DEC = '32:56:30.8'
CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList()
#print "checking centroid file", CentFile
#if not os.path.exists(CentFile): #check if cent pix file already exists
cent.centroidCalc(obsFile, centroid_RA, centroid_DEC, outputFileName = CentFile, guessTime=10, integrationTime=10)
# print "Flux file centroid cal saved to %s"%(CentFile)
obsFile.loadCentroidListFile(CentFile)
print "Centroid File loaded %s"%(CentFile)
#Load up associated calibrations
#obsFile.loadTimeAdjustmentFile(FileName(run=run,date=date,tstamp=tstamp).timeAdjustments())
#obsFile.loadHotPixCalFile(FileName(run=run,date=date,tstamp=tstamp).timeMask())
#obsFile.loadCentroidListFile(FileName(run=run,date=date,tstamp=tstamp).centroidList())
#And write out the results....
obsFile.writePhotonList(outputFileName,firstSec,integrationTime,
pixRemapFileName=pixRemapFileName)
#Read the results back in....
#photFile = photList.PhotFile(outputFilename)
print "Wrote photon list file: ", outputFileName
def main():
#open the sky file for hr9087
run = 'PAL2012'
date = '20121210'
wvlCal = '20121211-052230'
obsTimestamp = '20121211-051650'
flatCalDate = '20121211'
flatCalTstamp = '20121212-074700'
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
timeMaskFileName = obsFN.timeMask()
wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obsFileName
hp.findHotPixels(obsFileName,timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs = ObsFile(obsFileName)
obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
obs.loadWvlCalFile(wvlFileName)
obs.loadFlatCalFile(flatFileName)
obs.loadHotPixCalFile(timeMaskFileName)
#obs.setWvlCutoffs(4000,8000)
#get image before and after flat cal
print 'getting images'
beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True)
rawCubeDict = obs.getSpectralCube(weighted=False)
rawCube = np.array(rawCubeDict['cube'],dtype=np.double)
effIntTime = rawCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
rawCube *= maxIntTime / effIntTime3d
rawCube[np.isnan(rawCube)] = 0
rawCube[rawCube == np.inf] = 0
beforeImg = np.sum(rawCube,axis=-1)
print 'finished first cube'
flatCubeDict = obs.getSpectralCube(weighted=True)
flatCube = np.array(flatCubeDict['cube'],dtype=np.double)
effIntTime = flatCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
flatCube *= maxIntTime / effIntTime3d
flatCube[np.isnan(flatCube)] = 0
flatCube[flatCube == np.inf] = 0
afterImg = np.sum(flatCube,axis=-1)
plotArray(title='before flatcal',image=beforeImg)
plotArray(title='after flatcal',image=afterImg)
print 'before sdev',np.std(beforeImg[afterImg!=0])
print 'after sdev',np.std(afterImg[afterImg!=0])
np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)
import hotpix.hotPixels as hp
from photonlist import photlist
import matplotlib.pylab as plt
run = 'PAL2013'
date = '20131209'
tstamps = ['20131209-115553','20131209-120100', '20131209-121634', '20131209-122152']
for tstamp in tstamps:
centroidTstamp = tstamp
calTstamp='20131209-132225'
fluxTstamp='ones'
flatTstamp='20131209'
obsFileName = FileName(run=run, date=date, tstamp=tstamp)
obsFile = ObsFile(obsFileName.obs())
frames = []
print '-----------------------'
print tstamp
for i in xrange(30):
print "making frame ", i
frame = obsFile.getFrame(10*i,10)
#plt.matshow(frame,vmin=10000,vmax=40000,cmap=plt.cm.gray)
#plt.show()
frames.append(frame)
makeMovie(frames,frameTitles=np.arange(30)*10,cbar=True,outName='%s.gif'%(tstamp), normMin=400, normMax=4000)
CentFile = FileName(run=run,date=date,tstamp=tstamp).centroidList()
obsFile.loadCentroidListFile(CentFile)
#obsTimestamp ='20140925-124603' #early twilight
#obsTimestamp ='20140925-124706' #early twilight
#obsTimestamp ='20140925-124809' #early twilight
#obsTimestamp ='20140925-124912' #early twilight
#obsTimestamp ='20140925-125015' #late twilight
#obsTimestamp ='20140925-125118' #late twilight
#obsTimestamp ='20140925-125221' #late twilight
#obsTimestamp ='20140925-125324' #late twilight
#obsTimestamp ='20140925-125427' #late twilight
#date = '20140925'
#obsTimestamp ='20140926-104528' #psr J0337
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
print obsFileName
timeMaskFileName = obsFN.timeMask()
if wvlCal != '':
wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
illumFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).illumSoln()
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obsFileName
hp.findHotPixels(inputFileName=obsFileName,outputFileName=timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs = ObsFile(obsFileName)
exptime = obs.getFromHeader('exptime')