def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess,hotPath,flatPath):
obs = ObsFile(obsPath)
print obsPath,obs.getFromHeader('exptime'),obs
if not os.path.exists(hotPath):
hp.findHotPixels(obsFile=obs,outputFileName=hotPath)
obs.loadHotPixCalFile(hotPath,switchOnMask=False)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatPath)
obs.setWvlCutoffs(3000,8000)
cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath)
print 'done centroid',centroidPath
del obs
def centroidObs(obsPath,centroidPath,centroidRa,centroidDec,haOffset,xGuess,yGuess):
obs = ObsFile(obsPath)
print obsPath,obs.getFromHeader('exptime'),obs
obs.loadAllCals()
# obs.loadBestWvlCalFile()
# obs.loadFlatCalFile(flatPath)
obs.setWvlCutoffs(3000,11000)
# if not os.path.exists(hotPath):
# hp.findHotPixels(obsFile=obs,outputFileName=hotPath,display=True,fwhm=2.,boxSize=5, nSigmaHot=4.0,)
# obs.loadHotPixCalFile(hotPath,switchOnMask=True)
cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath,usePsfFit=True,radiusOfSearch=8)
print 'done centroid',centroidPath
del obs
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) >2:
fileNum = str(sys.argv[2])
else:
fileNum = '0'
#science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
wvldir = params[2].split('=')[1].strip()
obsfile = params[3].split('=')[1].strip()
skyfile = params[4].split('=')[1].strip()
flatfile = params[5].split('=')[1].strip()
wvlfile = params[6].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121206"
if len(params)>10:
xpix = int(params[10].split('=')[1].strip())
ypix = int(params[11].split('=')[1].strip())
apertureRadius = int(params[12].split('=')[1].strip())
startTime = int(params[13].split('=')[1].strip())
intTime =int(params[14].split('=')[1].strip())
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
obs = ObsFile(obsFileName)
obs.loadWvlCalFile(wvlCalFileName)
obs.loadFlatCalFile(flatCalFileName)
print "analyzing file %s"%(obsFileName)
print "loaded data file and calibrations\n---------------------\n"
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
#wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
#nWvlBins=len(wvlBinEdges)-1
#print np.shape(obsSpectra)
#print nRow
#print nCol
#print nWvlBins
#load/generate hot pixel mask file
HotPixFile = getTimeMaskFileName(obsFileName)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFileName,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
#######
#EVERYTHING BEFORE HERE IS STANDARD FILE/CALFILE LOADING
startWvl = 3000
#stopWvl = 7000 #for V-band
stopWvl = 9000 #for R-band
print "Making spectral cube"
#for pg0220 first sec should be 80 since object is moving around before this
#for pg0220A first sec should be 70, integration time is 140
#for landolt 9542 first sec should be 20, int time is -1
cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, wvlStart = startWvl, wvlStop = stopWvl, wvlBinEdges = [startWvl,stopWvl], weighted=False)
cube= np.array(cubeDict['cube'], dtype=np.double)
wvlBinEdges= cubeDict['wvlBinEdges']
effIntTime= cubeDict['effIntTime']
print "median effective integration time = ", np.median(effIntTime)
nWvlBins=len(wvlBinEdges)-1
print "cube shape ", np.shape(cube)
print "effIntTime shape ", np.shape(effIntTime)
#add third dimension to effIntTime for broadcasting
effIntTime = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
cube /= effIntTime #put cube into counts/s
#calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins),dtype=float)
for n in xrange(nWvlBins):
binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
wvls[n] = (wvlBinEdges[n]+(binsize/2.0))
print "wvls ",wvls
#reshape cube for makeMovie
movieCube = np.zeros((nWvlBins,np.shape(cube)[0],np.shape(cube)[1]),dtype=float)
for i in xrange(nWvlBins):
movieCube[i,:,:] = cube[:,:,i]
#show individual frames as they are made to debug
#plt.matshow(movieCube[i],vmin = 0, vmax = 100)
#plt.show()
print "movieCube shape ", np.shape(movieCube)
print "wvls shape ", np.shape(wvls)
#print cube
#print "--------------------------"
#print movieCube
print "adding frames with wvl below ", stopWvl
finalframe = np.zeros((1,np.shape(movieCube)[1],np.shape(movieCube)[2]))
for f in xrange(len(wvls[wvls<stopWvl])):
print wvls[f]
finalframe[0]+=movieCube[f]
plt.matshow(movieCube[f],vmin=0,vmax = 40)
plt.show()
movieCube = finalframe
np.savez('%s_%s.npz'%(objectName,fileNum),stack=movieCube,wvls=wvls)
print "Saved frame to .npz file"
plt.matshow(movieCube[0],vmin=0,vmax = 40)
plt.show()
if not os.path.exists(hotPixFn):
hp.findHotPixels(obsFn, hotPixFn)
print "Flux file pixel mask saved to %s" % (hotPixFn)
ob.loadHotPixCalFile(hotPixFn, switchOnMask=True)
ob.loadWvlCalFile(wfn)
ob.loadFlatCalFile(ffn)
ob.setWvlCutoffs(wvlLowerCutoff, wvlUpperCutoff)
bad_solution_mask = np.zeros((46, 44))
bad_count = 0
for y in range(46):
for x in range(44):
if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff):
bad_solution_mask[y][x] = 1
unix = ob.getFromHeader("unixtime")
startJD = unix / 86400.0 + 2440587.5
nSecInFile = ob.getFromHeader("exptime")
deadMask = ob.getDeadPixels()
y_values, x_values = np.where(
np.logical_and(bad_solution_mask == 0, np.logical_and(apertureMask != 0, deadMask == 1))
)
y_sky, x_sky = np.where(np.logical_and(bad_solution_mask == 0, np.logical_and(skyMask != 0, deadMask == 1)))
# tic = time()
for sec in np.arange(0, nSecInFile, integrationTime):
jd = (
startJD + sec / (24.0 * 3600.0) + integrationTime / 2.0 / (24.0 * 3600.0)
) # add seconds offset to julian date, move jd to center of bin
print count, jd
count += 1
def main():
run = 'PAL2014'
year = '2014'
initialPath = '/Scratch'
packetMasterLogDir = '/LABTEST/PacketMasterLogs/'
initialPath = os.path.join(initialPath,run)
outPath = FileName(run=run,mkidDataDir='/Scratch/').timeAdjustments()
outFile = tables.openFile(outPath,'w')
timeAdjustGroup = outFile.createGroup('/','timeAdjust','Times to add to timestamps')
firmwareDelayTable = outFile.createTable(timeAdjustGroup,'firmwareDelay',firmwareDelayDescription,'Times to add to all timestamps taken with a firmware bof')
newFirmwareEntry = firmwareDelayTable.row
newFirmwareEntry['firmwareName']='chan_svf_2014_Aug_06_1839.bof'
newFirmwareEntry['firmwareDelay']=-41e-6 #s, subtract 41 us from timestamps
newFirmwareEntry.append()
firmwareDelayTable.flush()
firmwareDelayTable.close()
roachDelayTable = outFile.createTable(timeAdjustGroup,'roachDelays',roachDelaysDescription,'Times to add to each roach\'s timestamps')
for sunsetDatePath in sorted(glob.glob(os.path.join(initialPath,year+'*'))):
sunsetDate = os.path.basename(sunsetDatePath)
for fullObsPath in sorted(glob.glob(os.path.join(sunsetDatePath,'obs*.h5'))):
obsFileName = os.path.basename(fullObsPath)
obsTStamp = obsFileName.split('.')[0].split('_')[1]
print obsFileName
obsFN = FileName(run=run,date=sunsetDate,tstamp=obsTStamp,packetMasterLogDir=packetMasterLogDir)
pmLogFileName = obsFN.packetMasterLog()
try:
ob = ObsFile(fullObsPath)
except:
print 'can\'t open file'
continue
try:
if os.path.getsize(pmLogFileName) <= 0:
continue
except:
print 'can\'t open Packet Master Log ',pmLogFileName
continue
try:
f = open(pmLogFileName,'r')
except:
print 'can\'t open Packet Master Log ',pmLogFileName
continue
lastTstampLines = np.zeros(8)
firstTstampLines = np.zeros(8)
tstampLine = ''
for line in f:
if 'here' in line:
#skip over large log files with debug info
print 'skipping file with "here"'
continue
if line.split(' ')[0] == 'bundle':
tstampLine = line
break
if tstampLine == '':
print 'skipping file without "bundle"'
#didn't find lines with 'bundle' in them
continue
f.seek(0)
for line in f:
if line.split(' ')[0] == 'bundle':
try:
at = float(line.split('at')[1].split()[0])
except:
break
lastTstampLine = at
iRoach = int(line.split('roach')[1].split('took')[0].strip())
lastTstampLines[iRoach] = at
if firstTstampLines[iRoach] == 0:
firstTstampLines[iRoach] = at
packetReceivedUnixTimestamp = float((tstampLine.split('took')[1].split('total')[0].strip()))
firstPacketDelay = packetReceivedUnixTimestamp-int(ob.getFromHeader('unixtime'))
roachSecDelays =np.array(np.floor(lastTstampLines+firstPacketDelay-ob.getFromHeader('exptime')),dtype=np.int)
print 'roach delays',roachSecDelays
if np.all(roachSecDelays >= 0):
newEntry = roachDelayTable.row
newEntry['obsFileName'] = os.path.basename(fullObsPath)
newEntry['roachDelays'] = roachSecDelays
newEntry.append()
roachDelayTable.flush()
else:
print 'obs was aborted midway, delays cannot be calculated'
roachDelayTable.close()
outFile.close()
class Cosmic:
def __init__(self, fn, beginTime=0, endTime='exptime',
nBinsPerSec=10, flashMergeTime=1.0,
applyCosmicMask = False,
loggingLevel=logging.CRITICAL,
loggingHandler=logging.StreamHandler()):
"""
Opens fileName in MKID_RAW_PATH, sets roachList
endTime is exclusive
"""
self.logger = logging.getLogger("cosmic")
self.logger.setLevel(loggingLevel)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
loggingHandler.setFormatter(formatter)
self.logger.addHandler(loggingHandler)
self.logger.info("Cosmic: begin init for obsFile=%s"%fn.obs())
self.fn = fn
self.fileName = fn.obs();
self.file = ObsFile(self.fileName)
# apply Matt's time fix
timeAdjustments = self.fn.timeAdjustments()
if os.path.exists(timeAdjustments):
self.file.loadTimeAdjustmentFile(timeAdjustments)
# apply Julian's time masks
timeMaskFile = self.fn.timeMask();
if os.path.exists(timeMaskFile):
self.file.loadHotPixCalFile(timeMaskFile,switchOnMask=True)
# apply standard mask
if applyCosmicMask:
self.file.loadStandardCosmicMask()
self._setRoachList()
self._setAllSecs()
self.exptime = self.file.getFromHeader('exptime')
if endTime =='exptime':
self.endTime = float(self.exptime)
else:
self.endTime = float(endTime)
if ( (self.endTime > self.exptime) or (endTime < 0)):
raise RuntimeError("bad endTime: endTime=%s exptime=%s" % \
(str(endTime),str(self.exptime)))
self.beginTime = float(beginTime)
self.timeHgs = "none"
self.nBinsPerSec = nBinsPerSec
self.flashMergeTime = flashMergeTime
self.times = \
np.arange(self.beginTime, self.endTime, 1.0/self.nBinsPerSec)
# for measuring flashes, indexed by roach name
self.rMean = {} # mean from meanclip
self.rSigma = {} # sigma from meanclip
self.rNSurvived = {} # number of survivors from meanclip
self.rNormed = {} # (value-mean)/sigma
self.flashInterval = {}
self.logger.info("Cosmic: end init: beginTime=%s endTime=%s"%(str(self.beginTime),str(self.endTime)))
def __del__(self):
"""
Close any open files
"""
# print "now in Cosmic.__del__ for ",self.fileName
try:
del self.file
except:
pass
def _setRoachList(self):
self.roachList = []
for row in self.file.beamImage:
for name in row:
roachName = name.split("/")[1]
if roachName not in self.roachList:
self.roachList.append(roachName)
self.roachList.sort()
def _setAllSecs(self):
nRow = self.file.nRow
nCol = self.file.nCol
self.allSecs = \
dict( ((i,j),None) for i in range(nRow) for j in range(nCol))
for iRow in np.arange(nRow):
for iCol in np.arange(nCol):
self.allSecs[iRow,iCol] = \
self.file.file.getNode(self.file.beamImage[iRow][iCol])
def nPhoton(self, beginTime=0, endTime='expTime'):
"""
trivial example of counting the number of photons in a file
"""
nPhoton = 0
for iRow in range(self.file.nRow):
for iCol in range(self.file.nCol):
for iSec in range(self.beginTime, self.endTime):
sec = self.allSecs[iRow,iCol][iSec]
nPhoton += len(sec)
return nPhoton
def findFlashes(self, clipsig=3.0, maxiter=5,\
converge_num=0.05, verbose=0, flashsig=6):
"""
find flashes by looking at the time histograms. Calculate the
mean,sigma using meanclip and the parameters clipsig, maxiter, converge_num
A time bin has a flash if the normalized signal (measured-mean)/sigma
is larger than flashsig.
"""
# make the blank data structures
if self.timeHgs == "none":
self.makeTimeHgs()
self.flashInterval["all"] = []
# find the flashes in each roach
for roach in self.roachList:
self.rMean[roach],self.rSigma[roach],self.rNSurvived[roach] = \
meanclip.meanclip(\
np.array(self.timeHgs[roach]), clipsig, maxiter, converge_num,\
verbose)
self.rNormed[roach] = \
(self.timeHgs[roach]-self.rMean[roach])/self.rSigma[roach]
self.flashInterval[roach] = interval()
prev = 0
a = self.rNormed[roach]
for i in range(len(a)):
this = a[i] > flashsig
if (this != prev):
if (this):
iBegin = i
else:
self.flashInterval[roach] = \
self.flashInterval[roach] | interval[iBegin,i]
prev = this
if (prev):
self.flashInterval[roach] = \
self.flashInterval[roach] | interval[iBegin,i]
# union of all flashes
self.flashInterval["all"] = interval()
for roach in self.roachList:
self.flashInterval["all"] = \
self.flashInterval["all"] | self.flashInterval[roach]
# look for gaps smaller than self.flashMergeTime and plug them
dMax = self.nBinsPerSec*self.flashMergeTime
extrema = self.flashInterval["all"].extrema
for i in range(len(extrema)/2-1):
i0 = extrema[2*i+1][0]
i1 = extrema[2*(i+1)][0]
if (i1-i0) <= dMax:
t0 = self.beginTime + float(i0)/self.nBinsPerSec
t1 = self.beginTime + float(i1)/self.nBinsPerSec
self.flashInterval["all"] = \
self.flashInterval["all"] | interval[i0,i1]
# convert to ticks since the beginning of the data file
rlAll = list(self.roachList)
rlAll.append("all")
ticksPerSecond = int(1.0/self.file.tickDuration)
offset = self.beginTime*ticksPerSecond
scale = 1.0/(self.file.tickDuration*self.nBinsPerSec)
for roach in rlAll:
self.flashInterval[roach] = offset+scale*self.flashInterval[roach]
def writeFlashesToHdf5(self,overwrite=1):
"""
write intervals with flashes to the timeMask file
"""
# get the output file name, and make the directory if you need to
cmFileName = self.fn.cosmicMask()
(cmDir,name) = os.path.split(cmFileName)
if not os.path.exists(cmDir):
os.makedirs(cmDir)
# write parameters used to find flashes
h5f = tables.openFile(cmFileName, 'w')
fnode = filenode.newNode(h5f, where='/', name='timeMaskHdr')
fnode.attrs.beginTime = self.beginTime
fnode.attrs.endTime = self.endTime
fnode.attrs.nBinsPerSec = self.nBinsPerSec
fnode.attrs.flashMergeTime = self.flashMergeTime
fnode.close();
# write the times where flashes are located
tbl = h5f.createTable('/','timeMask',TimeMask.TimeMask,"Time Mask")
rlAll = list(self.roachList)
rlAll.append("all")
for roach in rlAll:
extrema = self.flashInterval[roach].extrema
for i in range(len(extrema)/2):
row = tbl.row
row['tBegin'] = int(extrema[2*i][0])
row['tEnd'] = int(extrema[2*i+1][0])
if (roach == "all"):
reason = "Merged Flash"
else:
reason = "Flash in %s" % roach
row['reason'] = TimeMask.timeMaskReason[reason]
row.append()
tbl.flush()
tbl.close()
h5f.close()
def makeTimeHgs(self):
"""
Fill in the timeHgs variable
This is a dictionary, indexed by the roach name, of the time histograms
"""
self.timeHgs = {}
for iSec in range(self.beginTime, self.endTime):
self.logger.info("in makeTimeHgs iSec=%4d / %4d" % (iSec,self.endTime))
hgsThisSec = {}
for iRow in range(self.file.nRow):
for iCol in range(self.file.nCol):
sec = self.allSecs[iRow,iCol][iSec]
if len(sec) > 0:
times = sec & self.file.timestampMask
hg,edges = \
np.histogram(times,bins=self.nBinsPerSec, \
range=(0,1.0/self.file.tickDuration))
roachName = \
self.file.beamImage[iRow][iCol].split("/")[1]
if not hgsThisSec.has_key(roachName):
hgsThisSec[roachName] = \
np.zeros(self.nBinsPerSec,dtype=np.int64)
hgsThisSec[roachName] += hg
for roachName in hgsThisSec.keys():
if not self.timeHgs.has_key(roachName):
self.timeHgs[roachName] = []
self.timeHgs[roachName] += list(hgsThisSec[roachName])
def plotTimeHgs(self):
"""
Plot the time HGS in plt structure, with legend
"""
plt.clf()
plt.figure(1)
keys = self.timeHgs.keys()
keys.sort()
plt.subplot(211)
for roachName in keys:
hg = self.timeHgs[roachName]
plt.plot(self.times, hg,label=roachName)
plt.legend()
dt = 1.0/self.nBinsPerSec
plt.ylabel("photons/%.2f sec" % dt)
plt.title("Cosmic timeHgs "+ self.fileName)
plt.subplot(212)
for roachName in keys:
plt.plot(self.times, \
self.rNormed[roachName],label=roachName)
plt.xlabel("time (sec)")
plt.ylim(-23,30)
dt = 1.0/self.nBinsPerSec
plt.ylabel("normalized photons/%.2f sec" % dt)
y = -5
x0 = self.beginTime + 0.1*(self.endTime-self.beginTime)
xmax = plt.xlim()[1]
rlAll = list(self.roachList)
rlAll.append("all")
for roach in rlAll:
print "plot for roach=",roach
plt.plot([x0,xmax],[y,y], linestyle=":", color="gray")
plt.text(x0, y, roach, fontsize=8, va="center")
extrema = self.flashInterval[roach].extrema
for i in range(len(extrema)/2):
t0 = (extrema[2*i][0] - 0.5)*self.file.tickDuration
t1 = (extrema[2*i+1][0] - 0.5)*self.file.tickDuration
plt.plot([t0,t1],[y,y],'r', linewidth=4)
y -= 2
def findCosmics(self, stride=10, threshold=100,
populationMax=2000, nSigma=5, writeCosmicMask=False,
ppsStride=10000):
"""
Find cosmics ray suspects. Histogram the number of photons
recorded at each timeStamp. When the number of photons in a
group of stride timeStamps is greater than threshold in second
iSec, add (iSec,timeStamp) to cosmicTimeLists. Also keep
track of the histogram of the number of photons per stride
timeStamps.
return a dictionary of 'populationHg', 'cosmicTimeLists',
'binContents', 'timeHgValues', 'interval', 'frameSum', and 'pps'
populationHg is a histogram of the number of photons in each time bin.
This is a poisson distribution with a long tail due to cosmic events
cosmicTimeLists is a numpy array of all the sequences that are
suspects for cosmic rays
binContents corresponds to cosmicTimeLists. For each time in
cosmicTimeLists, binContents is the number of photons detected
at that time.
timeHgValues is a histogram of the number of photons in each time
interval
frameSum is a two dimensional numpy array of the number of photons
detected by each pixel
interval is the interval of data to be masked out
pps is photons per second, calculated every ppsStride bins.
"""
self.logger.info("findCosmics: begin stride=%d threshold=%d populationMax=%d nSigma=%d writeCosmicMask=%s"%(stride,threshold,populationMax,nSigma,writeCosmicMask))
exptime = self.endTime-self.beginTime
nBins = int(np.round(self.file.ticksPerSec*exptime+1))
bins = np.arange(0, nBins, 1)
timeHgValues,frameSum = self.getTimeHgAndFrameSum(self.beginTime,self.endTime)
remainder = len(timeHgValues)%ppsStride
if remainder > 0:
temp = timeHgValues[:-remainder]
else:
temp = timeHgValues
ppsTime = (ppsStride*self.file.tickDuration)
pps = np.sum(temp.reshape(-1, ppsStride), axis=1)/ppsTime
self.logger.info("findCosmics: call populationFromTimeHgValues")
pfthgv = Cosmic.populationFromTimeHgValues\
(timeHgValues,populationMax,stride,threshold)
#now build up all of the intervals in seconds
self.logger.info("findCosmics: build up intervals: nCosmicTime=%d"%len(pfthgv['cosmicTimeList']))
i = interval()
iCount = 0
secondsPerTick = self.file.tickDuration
for cosmicTime in pfthgv['cosmicTimeList']:
#t0 = max(0,self.beginTime+(cosmicTime-50)/1.e6)
#t1 = min(self.endTime,self.beginTime+(cosmicTime+50)/1.e6)
#intTime = t1-t0
t0 = self.beginTime+cosmicTime*secondsPerTick
dt = stride*secondsPerTick
t1 = t0+dt
left = max(self.beginTime, t0-nSigma*dt)
right = min(self.endTime, t1+2*nSigma*dt)
i = i | interval[left,right]
self.logger.debug("findCosmics: iCount=%d t0=%f t1=%f left=%f right=%f"%(iCount,t0,t1,left,right))
iCount+=1
tMasked = Cosmic.countMaskedBins(i)
ppmMasked = 1000000*tMasked/(self.endTime-self.beginTime)
retval = {}
retval['timeHgValues'] = timeHgValues
retval['populationHg'] = pfthgv['populationHg']
retval['cosmicTimeList'] = pfthgv['cosmicTimeList']
retval['binContents'] = pfthgv['binContents']
retval['frameSum'] = frameSum
retval['interval'] = i
retval['ppmMasked'] = ppmMasked
retval['pps'] = pps
retval['ppsTime'] = ppsTime
if writeCosmicMask:
cfn = self.fn.cosmicMask()
self.logger.info("findCosmics: write masks to =%s"%cfn)
ObsFile.writeCosmicIntervalToFile(i, self.file.ticksPerSec,
cfn,self.beginTime, self.endTime,
stride, threshold, nSigma, populationMax)
self.logger.info("findCosmics: end with ppm masked=%d"%ppmMasked)
return retval
def getTimeHgAndFrameSum(self, beginTime, endTime):
integrationTime = endTime - beginTime
nBins = int(np.round(self.file.ticksPerSec*integrationTime+1))
timeHgValues = np.zeros(nBins, dtype=np.int64)
frameSum = np.zeros((self.file.nRow,self.file.nCol))
self.logger.info("get all time stamps for integrationTime=%f"%integrationTime)
for iRow in range(self.file.nRow):
#print "Cosmic.findCosmics: iRow=",iRow
for iCol in range(self.file.nCol):
# getTimedPacketList is slow. Use getPackets instead.
#gtpl = self.file.getTimedPacketList(iRow,iCol,beginTime,
# integrationTime)
gtpl = self.file.getPackets(iRow,iCol,
beginTime,integrationTime)
timestamps = gtpl['timestamps']
if timestamps.size > 0:
timestamps = \
(timestamps - beginTime)*self.file.ticksPerSec
# per Matt S. suggestion 2013-07-09
ts32 = np.round(timestamps).astype(np.uint32)
tsBinner.tsBinner32(ts32, timeHgValues)
frameSum[iRow,iCol] += ts32.size
return timeHgValues,frameSum
@staticmethod
def countMaskedBins(maskInterval):
retval = 0
for x in maskInterval:
retval += x[1]-x[0]
return retval
@staticmethod
def populationFromTimeHgValues(timeHgValues,populationMax,stride,threshold):
"""
Rebin the timgHgValues histogram by combining stride bins. If
stride > 1, then bin a second time after shifting by stride/2
Create populationHg, a histogram of the number of photons in
the large bins. Also, create (and then sort) a list
cosmicTimeList of the start of bins (in original time units)
of overpopulated bins that have more than threshold number of
photons.
return a dictionary containing populationHg and cosmicTimeList
"""
popRange = (-0.5,populationMax-0.5)
if stride==1:
populationHg = np.histogram(\
timeHgValues, populationMax, range=popRange)
cosmicTimeList = np.where(timeHgValues > threshold)[0]
binContents = np.extract(timeHgValues > threshold, timeHgValues)
else:
# rebin the timeHgValues before counting the populations
length = timeHgValues.size
remainder = length%stride
if remainder == 0:
end = length
else:
end = -remainder
timeHgValuesTrimmed = timeHgValues[0:end]
timeHgValuesRebinned0 = np.reshape(
timeHgValuesTrimmed, [length/stride, stride]).sum(axis=1)
populationHg0 = np.histogram(
timeHgValuesRebinned0, populationMax, range=popRange)
cosmicTimeList0 = stride*np.where(\
timeHgValuesRebinned0 > threshold)[0]
binContents0 = np.extract(timeHgValuesRebinned0 > threshold,
timeHgValuesRebinned0)
timeHgValuesRebinned1 = np.reshape(
timeHgValuesTrimmed[stride/2:-stride/2],
[(length-stride)/stride, stride]).sum(axis=1)
populationHg1 = np.histogram(\
timeHgValuesRebinned1, populationMax, range=popRange)
cosmicTimeList1 = (stride/2)+stride*np.where(\
timeHgValuesRebinned1 > threshold)[0]
binContents1 = np.extract(timeHgValuesRebinned1 > threshold,
timeHgValuesRebinned1)
populationHg = (populationHg0[0]+populationHg1[0],\
populationHg0[1])
cosmicTimeList = np.concatenate((cosmicTimeList0,cosmicTimeList1))
binContents = np.concatenate((binContents0, binContents1))
args = np.argsort(cosmicTimeList)
cosmicTimeList = cosmicTimeList[args]
binContents = binContents[args]
cosmicTimeList.sort()
retval = {}
retval['populationHg'] = populationHg
retval['cosmicTimeList'] = cosmicTimeList
retval['binContents'] = binContents
return retval
def makeMovies(self,beginTick, endTick, backgroundFrame, accumulate=False):
tick0 = np.uint64(beginTick)
tick1 = np.uint64(endTick)
for iRow in range(cosmic.file.nRow):
for iCol in range(cosmic.file.nCol):
gtpl = self.getTimedPacketList(iRow,iCol,sec0,1)
timestamps = gtpl['timestamps']
timestamps *= cosmic.file.ticksPerSec
ts32 = timestamps.astype(np.uint32)
for ts in ts32:
tindex = ts-t0
try:
listOfPixelsToMark[tindex].append((iRow,iCol))
except IndexError:
pass
for tick in range(t0,t1):
frames.append(frameSum)
title = makeTitle(tick,t0,t1)
titles.append(title)
mfn0 = "m-%s-%s-%s-%s-%010d-%010d-i.gif"%(run,sundownDate,obsDate,seq,t0,t1)
utils.makeMovie(frames, titles, outName=mfn0, delay=0.1, colormap=mpl.cm.gray,
listOfPixelsToMark=listOfPixelsToMark,
pixelMarkColor='red')
for i in range(len(listOfPixelsToMark)-1):
listOfPixelsToMark[i+1].extend(listOfPixelsToMark[i])
mfn1 = "m-%s-%s-%s-%s-%010d-%010d-a.gif"%(run,sundownDate,obsDate,seq,t0,t1)
utils.makeMovie(frames, titles, outName=mfn1, delay=0.1, colormap=mpl.cm.gray,
listOfPixelsToMark=listOfPixelsToMark,
pixelMarkColor='green')
def fitDecayTime(self,t0Sec,lengthSec=200,plotFileName='none'):
print "hello from fitDecayTime"
timedPacketList = self.file.getTimedPacketList(
iRow, iCol, sec0, lengthSec)
def fitExpon(self, t0, t1):
"""
Fit an exponential to all photons from time t0 to time t1
t0 and t1 are in ticks, 1e6 ticks per second
return a dictionary of: timeStamps,fitParams,chi2
"""
firstSec = int(t0/1e6) # in seconds
integrationTime = 1+int((t1-t0)/1e6) # in seconds
nBins = integrationTime*1e6 # number of microseconds; one bin per microsecond
timeHgValues = np.zeros(nBins, dtype=np.int64)
print "firstSec=",firstSec," integrationTime=",integrationTime
for iRow in range(self.file.nRow):
for iCol in range(self.file.nCol):
timedPacketList = self.file.getTimedPacketList(
iRow, iCol, firstSec=firstSec,
integrationTime=integrationTime)
timeStamps = timedPacketList['timestamps']
if (len(timeStamps) > 0):
# covert the time values to microseconds, and
# make it the type np.uint64
# round per Matt S. suggestion 2013-07-09
#ts64 = (timeStamps).astype(np.uint64)
ts32round = np.round(timeStamps).astype(np.uint32)
tsBinner.tsBinner(ts32round, timeHgValues)
temp = 1e6*(timeStamps-firstSec)
for i in range(len(timeStamps)):
ts32 = ((timeStamps-firstSec)*1e6).astype(np.uint32)
# add these timestamps to the histogram timeHgValues
remain0 = int(t0%1e6)
remain1 = int(t1%1e6)
timeHgValues = timeHgValues[remain0:remain1]
x = np.arange(len(timeHgValues))
y = timeHgValues
xArray = np.arange(0, dtype=np.int64)
yArray = np.arange(0, dtype=np.int64)
for i in range(len(x)):
if y[i] > 2:
xArray = np.append(xArray,i)
yArray = np.append(yArray,y[i])
ySigma = np.sqrt(yArray)
mean = (x*y).sum()/float(y.sum())
bExponGuess = 1/mean
aExponGuess = bExponGuess*timeHgValues.sum()
cExponGuess = 0
dExponGuess = 0
pExponGuess = [aExponGuess, bExponGuess, cExponGuess, dExponGuess]
bGaussGuess = mean
avgx2 = (x*x*y).sum()/float(y.sum())
cGaussGuess = np.sqrt(avgx2-bGaussGuess*bGaussGuess)
aGaussGuess = (timeHgValues.sum()/(cGaussGuess*np.sqrt(2*np.pi)))
pGaussGuess = [aGaussGuess, bGaussGuess, cGaussGuess]
xLimit = [bGaussGuess-4*cGaussGuess, bGaussGuess+4*cGaussGuess]
retval = {'timeHgValues':timeHgValues, 'pExponFit':pExponGuess,
'pGaussFit':pGaussGuess, 'xLimit':xLimit,
'cGaussGuess':cGaussGuess, 'timeStamps':timeStamps}
return retval
@staticmethod
def intervalTime(intervals):
"""
return the time (in seconds) masked by the intervals
"""
time = 0
for interval in intervals:
time += interval[1]-interval[0]
return time
@staticmethod
def funcExpon(x, a, b, c, d):
retval = a*np.exp(-b*(x-d)) + c
retval[x < d] = 0
return retval
@staticmethod
def funcGauss(x, a, b, c):
return a*np.exp(-(x-b)**2/(2.*c**2))
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) > 2:
fileNum = str(sys.argv[2])
else:
fileNum = "0"
# science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile, "r")
for line in f:
params.append(line)
f.close()
datadir = params[0].split("=")[1].strip()
flatdir = params[1].split("=")[1].strip()
wvldir = params[2].split("=")[1].strip()
obsfile = params[3].split("=")[1].strip()
skyfile = params[4].split("=")[1].strip()
flatfile = params[5].split("=")[1].strip()
wvlfile = params[6].split("=")[1].strip()
objectName = params[9].split("=")[1].strip()
# wvldir = "/Scratch/waveCalSolnFiles/oldbox_numbers/20121205"
# objectName = "crabNight1"
if len(params) > 10:
xpix = int(params[10].split("=")[1].strip())
ypix = int(params[11].split("=")[1].strip())
apertureRadius = int(params[12].split("=")[1].strip())
startTime = int(params[13].split("=")[1].strip())
intTime = int(params[14].split("=")[1].strip())
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
obs = ObsFile(obsFileName)
# obs.loadWvlCalFile(wvlCalFileName)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatCalFileName)
print "analyzing file %s" % (obsFileName)
print "loaded data file and calibrations\n---------------------\n"
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
# wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
# nWvlBins=len(wvlBinEdges)-1
# print np.shape(obsSpectra)
# print nRow
# print nCol
# print nWvlBins
# Apply Hot pixel masking before getting dead time correction
# HotPixFile = getTimeMaskFileName(obsFileName)
HotPixFile = FileName(obsFile=obs).timeMask()
print "making hot pixel file ", HotPixFile
if not os.path.exists(HotPixFile): # check if hot pix file already exists
hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
print "Flux file pixel mask saved to %s" % (HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s" % (HotPixFile)
# GET RAW PIXEL COUNT IMAGE TO CALCULATE CORRECTION FACTORS
print "Making raw cube to get dead time correction"
cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=False)
cube = np.array(cubeDict["cube"], dtype=np.double)
wvlBinEdges = cubeDict["wvlBinEdges"]
effIntTime = cubeDict["effIntTime"]
print "median effective integration time = ", np.median(effIntTime)
nWvlBins = len(wvlBinEdges) - 1
print "cube shape ", np.shape(cube)
print "effIntTime shape ", np.shape(effIntTime)
# add third dimension to effIntTime for broadcasting
effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
# put cube into counts/s in each pixel
cube /= effIntTime
# CALCULATE DEADTIME CORRECTION
# NEED TOTAL COUNTS PER SECOND FOR EACH PIXEL TO DO PROPERLY
# ASSUMES SAME CORRECTION FACTOR APPLIED FOR EACH WAVELENGTH, MEANING NO WL DEPENDANCE ON DEAD TIME EFFECT
DTCorr = np.zeros((np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
for f in range(0, np.shape(cube)[2]):
print cube[:, :, f]
print "-----------------------"
DTCorr += cube[:, :, f]
print DTCorr
print "\n=====================\n"
# Correct for 100 us dead time
DTCorrNew = DTCorr / (1 - DTCorr * 100e-6)
CorrFactors = DTCorrNew / DTCorr # This is what the frames need to be multiplied by to get their true values
print "Dead time correction factors = "
print CorrFactors
# REMAKE CUBE WITH FLAT WEIGHTS AND APPLY DEAD TIME CORRECTION AS WELL
print "Making Weighted cube"
# load/generate hot pixel mask file
# HotPixFile = getTimeMaskFileName(obsFileName)
HotPixFile = FileName(obsFile=obs).timeMask()
if not os.path.exists(HotPixFile): # check if hot pix file already exists
hp.findHotPixels(inputFileName=obsFileName, outputFileName=HotPixFile)
print "Flux file pixel mask saved to %s" % (HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s" % (HotPixFile)
cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=False)
# cubeDict = obs.getSpectralCube(firstSec=startTime, integrationTime=intTime, weighted=True, fluxWeighted=True)
cube = np.array(cubeDict["cube"], dtype=np.double)
wvlBinEdges = cubeDict["wvlBinEdges"]
effIntTime = cubeDict["effIntTime"]
print "median effective integration time = ", np.median(effIntTime)
nWvlBins = len(wvlBinEdges) - 1
print "cube shape ", np.shape(cube)
print "effIntTime shape ", np.shape(effIntTime)
# add third dimension to effIntTime for broadcasting
effIntTime = np.reshape(effIntTime, np.shape(effIntTime) + (1,))
# put cube into counts/s in each pixel
cube /= effIntTime
# add third dimension to CorrFactors for broadcasting
CorrFactors = np.reshape(CorrFactors, np.shape(CorrFactors) + (1,))
# apply dead time correction factors
cube *= CorrFactors
# calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins), dtype=float)
for n in xrange(nWvlBins):
binsize = wvlBinEdges[n + 1] - wvlBinEdges[n]
wvls[n] = wvlBinEdges[n] + (binsize / 2.0)
print "wvls ", wvls
# reshape cube for makeMovie
movieCube = np.zeros((nWvlBins, np.shape(cube)[0], np.shape(cube)[1]), dtype=float)
for i in xrange(nWvlBins):
movieCube[i, :, :] = cube[:, :, i]
# show individual frames as they are made to debug
# plt.matshow(movieCube[i],vmin = 0, vmax = 100)
# plt.show()
print "movieCube shape ", np.shape(movieCube)
print "wvls shape ", np.shape(wvls)
# print cube
# print "--------------------------"
# print movieCube
outdir = "/home/srmeeker/scratch/standards/"
np.savez(outdir + "%s_raw_%s.npz" % (objectName, fileNum), stack=movieCube, wvls=wvls)
utils.makeMovie(
movieCube,
frameTitles=wvls,
cbar=True,
outName=outdir + "%s_raw_%s.gif" % (objectName, fileNum),
normMin=0,
normMax=50,
)
"""
def main():
#filenames = ['obs_20121212-033825.h5',
fn = FileName(run='PAL2012',date='20121211',tstamp='20121212-055428').obs()
ob = ObsFile(fn)
frame = ob.getPixelCountImage(0,30,weighted=False)
hotPixMask = hotPixels.findHotPixels(image=frame,firstSec=0,intTime=30,weighted=False)['badflag']
frame[hotPixMask == 1] = 0
def plotFrame(fig,axes):
hMat=axes.matshow(frame,cmap=matplotlib.cm.gnuplot,origin='lower',vmax=np.mean(frame)+3*np.std(frame))
fig.colorbar(hMat)
PopUp(plotFunc=plotFrame)
#d = datetime.date(2012,10,15)#date of table value
#d2 = datetime.date(2012,12,11)#date of observation
#dt=(d2-d).total_seconds()#seconds between obs and table value
##crabtime ephemeris
#nu=29.6957720714 #1/us
#pdot=4.2013878e-13#us/s
#period=(1.0/nu)+pdot*dt#us
#goldstone ephemeris
F0=29.7414493465249770#Hz
deltaF0=0.0000000055983574
F1=-3.669005118205e-10#df0/dt Hz/s
F2=-3.085573120457e-20#d^2f0/dt^2 Hz/s^2
pEpoch = 54781.604891 #Modified Julian Date corresponding to F0
pEpoch = pEpoch+2400000.5#convert mjd to jd
pEpoch *= 24*3600 #in seconds
#obsDate = ob.getFromHeader('jd')
unixEpochJD = 2440587.5
unixSecsInDay = 86400.
headerUnixtime = ob.getFromHeader('unixtime')
obsDate = headerUnixtime/unixSecsInDay+unixEpochJD
startTime = obsDate*24*3600#in seconds
dt = startTime-pEpoch#seconds since pepoch
#freq = F0+F1*dt+F2/2*dt**2
freq = F0+F1*dt
period = 1.0/freq
print 'period=',period,'s'
#period=0.03367660643405371
#period=0.03367664238573182
#period=0.03367662440988317
iRow = 10
iCol = 14
integrationTime = 30
circCol,circRow = circ(iCol,iRow,radius=5)
firstSec = 0
dt = startTime-pEpoch + firstSec
freq = F0+F1*dt
period = 1.0/freq
print 'period=',period,'s'
nPhaseBins = 200
#np.set_printoptions(threshold=np.nan)
jdTimes = np.array([],dtype=np.float64)
times = np.array([])
for i in range(len(circCol)):
iRow = circRow[i]
iCol = circCol[i]
timestamps,peaks,baselines = ob.getTimedPacketList(iRow,iCol,firstSec,integrationTime)
timestamps = np.array(timestamps,dtype=np.float64)
jdTimestamps = obsDate+timestamps /(24.*3600.)
jdTimes = np.append(jdTimes,jdTimestamps)
times = np.append(times,timestamps)
jdTimes -= 2400000.5 #convert to modified jd
np.savetxt('crabOpticalSample-20121212-055428.txt',jdTimes)
periodDays = period/(24.*3600.)
phaseOffset = .2
phases = (jdTimes % periodDays)/periodDays+.2
phases = phases % 1.0
print len(phases)
histPhases,phaseBinEdges = np.histogram(phases,bins=nPhaseBins)
print jdTimes[0:10]
print times[0:10]
plt.plot(phaseBinEdges[0:-1],histPhases)
plt.show()
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) >3:
filenum = str('_'+sys.argv[3])
else:
filenum = '_0'
#science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
wvldir = params[2].split('=')[1].strip()
obsfile = params[3].split('=')[1].strip()
skyfile = params[4].split('=')[1].strip()
flatfile = params[5].split('=')[1].strip()
wvlfile = params[6].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
if len(params)>10:
xpix = params[10].split('=')[1].strip()
ypix = params[11].split('=')[1].strip()
apertureRadius = params[12].split('=')[1].strip()
#flux cal object parameter file
params2 = []
param2file = sys.argv[2]
f = open(param2file,'r')
for line in f:
params2.append(line)
f.close()
fluxdir = params2[7].split('=')[1].strip()
fluxfile = params2[8].split('=')[1].strip()
fluxCalObject = params2[9].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
print "obsfile = ",obsFileName
print "skyfile = ",skyFileName
print "wvlcal = ", wvlCalFileName
print "flatcal = ", flatCalFileName
print "fluxcal = ", fluxCalFileName
print "object = ", objectName
print "flux cal object = ", fluxCalObject
print "\n---------------------\n"
obs = ObsFile(obsFileName)
obs.loadWvlCalFile(wvlCalFileName)
obs.loadFlatCalFile(flatCalFileName)
obs.loadFluxCalFile(fluxCalFileName)
print "loaded data file and calibrations\n---------------------\n"
HotPixFile = getTimeMaskFileName(obsFileName)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFileName,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
wvlBinEdges,obsSpectra,obsIntTime = loadSpectra(obs,nCol,nRow)
nWvlBins=len(wvlBinEdges)-1
#divide spectrum by bin widths
binWidths = np.empty(nWvlBins)
for i in xrange(nWvlBins):
binWidths[i] = wvlBinEdges[i+1]-wvlBinEdges[i]
obsSpectra = obsSpectra/binWidths
#divide by effective integration time
for iRow in xrange(nRow):
for iCol in xrange(nCol):
obsSpectra[iRow][iCol] = obsSpectra[iRow][iCol] / obsIntTime[iRow][iCol]
#print np.shape(obsSpectra)
#print nRow
#print nCol
#print nWvlBins
medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
print "target spectrum loaded\n---------------------\n"
if skyfile != "None":
sky = ObsFile(skyFileName)
sky.loadWvlCalFile(wvlCalFileName)
sky.loadFlatCalFile(flatCalFileName)
sky.loadFluxCalFile(fluxCalFileName)
skyTime = sky.getFromHeader("exptime")
skyHotPixFile = getTimeMaskFileName(skyFileName)
if not os.path.exists(skyHotPixFile):
hp.findHotPixels(skyFileName,skyHotPixFile)
print "Flux file pixel mask saved to %s"%(skyHotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(skyHotPixFile)
wvlBinEdges,skySpectra,skyIntTime = loadSpectra(sky,nCol,nRow)
skySpectra = skySpectra/binWidths
#divide by effective integration time
for iRow in xrange(nRow):
for iCol in xrange(nCol):
skySpectra[iRow][iCol] = skySpectra[iRow][iCol] / skyIntTime[iRow][iCol]
skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
#skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
print "sky spectrum loaded\n---------------------\n"
else:
#if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
skySpectrum = medianObsSpectrum
print "sky spectrum estimated as median of target file spectrum\n---------------------\n"
#subtract sky spectrum from every pixel
allSkySpectrum = obsSpectra-skySpectrum
#set any negative values to 0 after sky subtraction
allSkySpectrum[allSkySpectrum<0]=0
#take median of remaining sky subtracted spectra to get median object spectrum
finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
#load std spectrum for comparison
realSpectra = loadStd(objectName,wvlBinEdges)
print "real std spectrum loaded for reference\n---------------------\n"
#create plots
plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
pp = PdfPages(fullFluxPlotFileName)
matplotlib.rcParams['font.size']=6
#calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins),dtype=float)
for n in xrange(nWvlBins):
binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
wvls[n] = (wvlBinEdges[n]+(binsize/2.0))
plt.figure()
ax1 = plt.subplot(231)
ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
plt.plot(wvls,medianObsSpectrum)
#plt.show()
ax2 = plt.subplot(232)
ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
plt.plot(wvls,skySpectrum)
#plt.show()
ax5 = plt.subplot(233)
ax5.set_title('Sensitivity Spectrum')
plt.plot(wvls,obs.fluxWeights)
ax3 = plt.subplot(234)
ax3.set_title('MKID data minus sky in counts')
plt.plot(wvls,finalSpectrum/max(finalSpectrum))
ax4 = plt.subplot(235)
ax4.set_title('Rebinned Std Spectrum of %s'%(objectName))
plt.plot(wvls,realSpectra)
#ax7 = plt.subplot(337)
#ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
#plt.plot(wvls,fluxFactors*subtractedSpectra)
pp.savefig()
pp.close()
#del obs
#del sky
print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)
txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
fullFluxTxtFileName = os.path.join(txtDir,txtFileName)
outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
outarr[:,0]=wvls
outarr[:,1]=medianObsSpectrum
#save sensitivity spectrum to file
np.savetxt(fullFluxTxtFileName, outarr)
print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
class DisplayStack(QMainWindow):
def __init__(self):
# 0) Start up gui
QWidget.__init__(self, parent=None)
self.ui = Ui_DisplayStack_gui()
self.ui.setupUi(self)
# Initialize Variables
self.initializeVariables()
# Initialize run and target, load param file
self.chooseRun()
'''
# Browse wavecal button
self.ui.wavelengthCalibrationSolutionButton.clicked.connect(self.chooseWavelengthFile)
# Browse flatcal button
self.ui.flatCalibrationSolutionButton.clicked.connect(self.chooseFlatFile)
'''
# Use wavelength calibration checkbox
self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal)
# Use flat calibration checkbox
self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal)
# Buttons for obs list creation
self.ui.addButton.clicked.connect(self.addObservation)
self.ui.removeButton.clicked.connect(self.removeObservation)
self.ui.clearButton.clicked.connect(self.createObsList)
# Start process button
self.ui.stackButton.clicked.connect(self.stackProcess)
# Load image stack button
self.ui.loadStackButton.clicked.connect(self.chooseStack)
# 1) Initialize Variables
def initializeVariables(self):
# Define path names
self.displayStackPath = '/Scratch/DisplayStack/'
self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/'
self.defaultFlatPath = '/Scratch/flatCalSolnFiles/'
# Arrays with run names and buttons
self.runNames = ['LICK2012','PAL2012','PAL2013']
self.runButtons = [self.ui.lick2012Button, self.ui.pal2012Button, self.ui.pal2013Button]
# Arrays with target names and buttons
self.lick2012TargetNames = ['LICK2012_Object']
self.lick2012TargetButtons = [self.ui.lick2012ObjectButton]
self.pal2012TargetNames = ['SDSS_J0651', 'SDSS_J0926']
self.pal2012TargetButtons = [self.ui.sdss0651Button,self.ui.sdss0926Button]
self.pal2013TargetNames = ['PAL2013_Object']
self.pal2013TargetButtons = [self.ui.pal2013ObjectButton]
self.targetNames = [self.lick2012TargetNames, self.pal2012TargetNames, self.pal2013TargetNames]
self.targetButtons = [self.lick2012TargetButtons, self.pal2012TargetButtons, self.pal2013TargetButtons]
# Activate run and target buttons
for iRun in range(len(self.runButtons)):
self.runButtons[iRun].clicked.connect(self.chooseRun)
for iTarget in range(len(self.targetButtons[iRun])):
self.targetButtons[iRun][iTarget].clicked.connect(self.chooseTarget)
self.ui.sunsetList.itemClicked.connect(self.createObsList)
def testFunction(self):
print 'test'
# 2) Choose Run
def chooseRun(self):
for iRun in range(len(self.runButtons)):
for iTarget in range(len(self.targetButtons[iRun])):
self.targetButtons[iRun][iTarget].setVisible(False)
if self.runButtons[iRun].isChecked():
self.runNumber = iRun
for iTarget in range(len(self.targetButtons[iRun])):
self.targetButtons[iRun][iTarget].setVisible(True)
self.run = self.runNames[self.runNumber]
self.targetNumber = 0
self.targetButtons[self.runNumber][self.targetNumber].setChecked(True)
self.target = self.targetNames[self.runNumber][self.targetNumber]
self.loadParamFile()
# 3) Choose Target
def chooseTarget(self):
for iTarget in range(len(self.targetButtons[self.runNumber])):
if self.targetButtons[self.runNumber][iTarget].isChecked():
self.targetNumber = iTarget
self.target = self.targetNames[self.runNumber][self.targetNumber]
self.loadParamFile()
# 4) Load Param File
def loadParamFile(self):
try:
self.paramData = readDict()
self.paramData.read_from_file(self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict')
self.obsTimes = np.array(self.paramData['obsTimes'])
self.utcDates = self.paramData['utcDates']
self.sunsetDates = self.paramData['sunsetDates']
self.calTimestamps = self.paramData['calTimestamps']
self.flatCalDates = self.paramData['flatCalDates']
print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.createSunsetList()
#self.createObsList()
self.createWavelengthList()
self.createFlatList()
self.paramFileExists = True
except IOError:
print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.ui.sunsetList.clear()
self.ui.obsList.clear()
self.ui.inputList.clear()
self.ui.wavelengthList.clear()
self.ui.flatList.clear()
self.paramFileExists = False
# 5) Choose Obs File
# Create list of available sunset dates
def createSunsetList(self):
self.ui.sunsetList.clear()
for iDate in range(len(self.sunsetDates)):
self.ui.sunsetList.addItem(self.sunsetDates[iDate])
# Create Initial Obs file list
def createObsList(self):
self.ui.obsList.clear()
self.ui.inputList.clear()
self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
for iObs in range(len(self.singleDayObservations)):
self.ui.obsList.addItem(self.singleDayObservations[iObs])
# Add observation to input list
def addObservation(self):
self.selectedObs = self.ui.obsList.currentItem()
self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs))
self.ui.inputList.addItem(self.selectedObs)
self.ui.inputList.sortItems()
# Remove observation from input list
def removeObservation(self):
self.removedObs = self.ui.inputList.currentItem()
self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs))
self.ui.obsList.addItem(self.removedObs)
self.ui.obsList.sortItems()
# 6) Load settings
def loadSettings(self):
self.validSettings = True
# Run and target information
self.run = self.runNames[self.runNumber]
self.target = self.targetNames[self.runNumber][self.targetNumber]
# General settings
self.integrationTime = int(self.ui.integrationTimeLine.text())
self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked()
self.useHotPixelMasking = self.ui.hotPixelBox.isChecked()
# Wavelength calibration settings
self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked()
self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text())
self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text())
# Flat calibration settings
self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked()
self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked()
self.fileCount = self.ui.inputList.count()
self.weighted = self.useFlatCalibration
self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration)
self.scaleByEffInt = self.useHotPixelMasking
if self.ui.sunsetList.currentItem() != None:
if self.fileCount == 0:
print 'Please select files to process...'
self.validSettings = False
if self.useWavelengthCalibration:
if self.ui.wavelengthList.currentItem() == None:
print 'Please select wavelength calibration...'
self.validSettings = False
else:
self.selectedWvlCal = self.ui.wavelengthList.currentItem().text()
self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln())
if self.useFlatCalibration:
if self.ui.flatList.currentItem() == None:
print 'Please select flat calibration...'
self.validSettings = False
else:
self.flatCalNight = self.ui.flatList.currentItem().text()
self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln())
else:
print 'Please select sunset night...'
self.validSettings = False
# 7) Load hot pixel mask
def loadHotMask(self):
self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5')
if not os.path.exists(self.hotPixelFilename):
hp.findHotPixels(self.obsFn,self.hotPixelFilename)
print "Hot pixel mask saved to %s"%(self.hotPixelFilename)
self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True)
# 8) Create wavelength cal file list
def createWavelengthList(self):
self.ui.wavelengthList.clear()
for iCal in range(len(self.calTimestamps)):
self.ui.wavelengthList.addItem(self.calTimestamps[iCal])
# Enable/disable wavecal options
def useWaveCal(self):
if self.ui.wavelengthCalibrationBox.isChecked():
self.ui.lowerWavelengthCutoffLine.setEnabled(True)
self.ui.lowerWavelengthCutoffLabel.setEnabled(True)
self.ui.upperWavelengthCutoffLine.setEnabled(True)
self.ui.upperWavelengthCutoffLabel.setEnabled(True)
self.ui.wavelengthList.setEnabled(True)
self.ui.flatCalibrationBox.setEnabled(True)
self.ui.flatCalibrationBox.setChecked(True)
self.ui.deadPixelBox.setEnabled(True)
self.ui.deadPixelBox.setChecked(True)
self.ui.flatList.setEnabled(True)
else:
self.ui.lowerWavelengthCutoffLine.setEnabled(False)
self.ui.lowerWavelengthCutoffLabel.setEnabled(False)
self.ui.upperWavelengthCutoffLine.setEnabled(False)
self.ui.upperWavelengthCutoffLabel.setEnabled(False)
self.ui.wavelengthList.setEnabled(False)
self.ui.flatCalibrationBox.setEnabled(False)
self.ui.flatCalibrationBox.setChecked(False)
self.ui.deadPixelBox.setChecked(False)
self.ui.deadPixelBox.setEnabled(False)
self.ui.flatList.setEnabled(False)
# 9) Create flat cal file list
def createFlatList(self):
self.ui.flatList.clear()
for iCal in range(len(self.flatCalDates)):
self.ui.flatList.addItem(self.flatCalDates[iCal])
# Enable/disable flatcal options
def useFlatCal(self):
if self.ui.flatCalibrationBox.isChecked():
self.ui.deadPixelBox.setEnabled(True)
self.ui.deadPixelBox.setChecked(True)
self.ui.flatList.setEnabled(True)
else:
self.ui.deadPixelBox.setChecked(False)
self.ui.deadPixelBox.setEnabled(False)
self.ui.flatList.setEnabled(False)
# 10) Load dead pixel mask
def loadDeadMask(self):
self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz')
if not os.path.exists(self.deadPixelFilename):
self.deadMask = self.ob.getDeadPixels()
np.savez(self.deadPixelFilename,deadMask = self.deadMask)
print "Dead pixel mask saved to %s"%(self.deadPixelFilename)
else:
self.deadFile = np.load(self.deadPixelFilename)
self.deadMask = self.deadFile['deadMask']
# 12) Create output name
def createOutputName(self):
self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's')
if self.useWavelengthCalibration and self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.npz')
elif self.useWavelengthCalibration and not self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.npz')
elif not self.waveLengthCalibration and self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_hp.npz')
else:
self.outputFilename = str(self.rawName + '.npz')
# Start process for creating image stacks
def stackProcess(self):
# Check for valid params file
if self.paramFileExists:
# 6) Load settings choosen from gui
self.loadSettings()
if self.validSettings:
# Loop through all files in input list
for iFile in range(self.fileCount):
# Create ObsFile instance
self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text()
self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs())
print 'Processing file ' + self.obsFn + '...'
self.ob = ObsFile(self.obsFn)
# Load time adjustment file
if self.useTimeAdjustment:
print 'Loading time adjustment file...'
self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments())
# 7) Load hot pixel mask
if self.useHotPixelMasking:
print 'Loading hot pixel mask...'
self.loadHotMask()
# 8) Load wave cal solution
if self.useWavelengthCalibration:
print 'Loading wavelength calibration...'
self.ob.loadWvlCalFile(self.wvlCalFilename)
# 9) Load flatcal solution
if self.useFlatCalibration:
print 'Loading flat calibration...'
self.ob.loadFlatCalFile(self.flatCalFilename)
# 10) Load dead pixel mask
if self.useDeadPixelMasking:
print 'Loading dead pixel mask...'
self.loadDeadMask()
# 11) Set wavelength cutoffs
if self.useWavelengthCalibration:
print 'Setting wavelength cutoffs...'
self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff)
# Timing
self.unix = self.ob.getFromHeader('unixtime')
self.startJD = self.unix/86400.+2440587.5
self.exptime = self.ob.getFromHeader('exptime')
self.times = []
self.frames = []
# 11) Create Image Stack
print 'Stacking images...'
for iSec in np.arange(0,self.exptime,self.integrationTime):
#add seconds offset to julian date, move jd to center of bin
self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.)
'Creating frame for time ' + self.jd
self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt)
self.frame = self.frameData['image']
if self.useDeadPixelMasking:
self.frame[self.deadMask == 0] = np.nan
self.frames.append(self.frame)
self.cube = np.dstack(self.frames)
self.times = np.array(self.times)
# 12) Create output file
self.createOutputName()
print 'Saving image stack to ' + self.outputFilename
np.savez(self.outputFilename, stack=self.cube, jd=self.times)
# Invalid params file
else:
print 'Invalid parameter file...'
# Choose an image stack
def chooseStack(self):
self.defaultLoadStackDirectory = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks')
self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("NPZ (*.npz)"))
if self.stackName == '':
print 'No file chosen'
else:
self.loadStack()
# Load that image stack for viewing
def loadStack(self):
print 'Loading cal files'
ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
ob.loadWvlCalFile(wfn)
ob.loadFlatCalFile(ffn)
ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)
print 'Finding bad wvl pixels'
bad_solution_mask=np.zeros((46,44))
bad_count=0;
for y in range(46):
for x in range(44):
if (ob.wvlRangeTable[y][x][1] < 11000):
bad_solution_mask[y][x] = 1
bad_count = bad_count+1
startJD = ob.getFromHeader('jd')
nSecInFile = ob.getFromHeader('exptime')
#tic = time()
print 'Finding dead pixels'
deadMask = ob.getDeadPixels()
#print 'Dead mask load time = ', time()-tic
print 'Getting images'
for sec in range(0,nSecInFile,integrationTime):
jd = startJD + sec/(24.*3600.)#add seconds offset to julian date
print count,jd
count+=1
times.append(jd)
titles.append('%.6f'%jd)
frameData = ob.getPixelCountImage(firstSec=sec,integrationTime=integrationTime,weighted=True)
frame = frameData['image']
showFrame = np.array(frame)
def main():
"""
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
fluxdir = params[2].split('=')[1].strip()
wvldir = params[3].split('=')[1].strip()
obsfile = params[4].split('=')[1].strip()
skyfile = params[5].split('=')[1].strip()
flatfile = params[6].split('=')[1].strip()
fluxfile = params[7].split('=')[1].strip()
wvlfile = params[8].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
fluxCalObject = params[10].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
"""
if len(sys.argv) >3:
filenum = str('_'+sys.argv[3])
else:
filenum = '_0'
#science object parameter file
params = []
paramfile = sys.argv[1]
f = open(paramfile,'r')
for line in f:
params.append(line)
f.close()
datadir = params[0].split('=')[1].strip()
flatdir = params[1].split('=')[1].strip()
wvldir = params[2].split('=')[1].strip()
obsfile = params[3].split('=')[1].strip()
skyfile = params[4].split('=')[1].strip()
flatfile = params[5].split('=')[1].strip()
wvlfile = params[6].split('=')[1].strip()
objectName = params[9].split('=')[1].strip()
if len(params)>10:
xpix = int(params[10].split('=')[1].strip())
ypix = int(params[11].split('=')[1].strip())
apertureRadius = int(params[12].split('=')[1].strip())
#flux cal object parameter file
params2 = []
param2file = sys.argv[2]
f = open(param2file,'r')
for line in f:
params2.append(line)
f.close()
fluxdir = params2[7].split('=')[1].strip()
fluxfile = params2[8].split('=')[1].strip()
fluxCalObject = params2[9].split('=')[1].strip()
obsFileName = os.path.join(datadir, obsfile)
skyFileName = os.path.join(datadir, skyfile)
wvlCalFileName = os.path.join(wvldir, wvlfile)
flatCalFileName = os.path.join(flatdir, flatfile)
fluxCalFileName = os.path.join(fluxdir, fluxfile)
print "obsfile = ",obsFileName
print "skyfile = ",skyFileName
print "wvlcal = ", wvlCalFileName
print "flatcal = ", flatCalFileName
print "fluxcal = ", fluxCalFileName
print "object = ", objectName
print "flux cal object = ", fluxCalObject
print "\n---------------------\n"
obs = ObsFile(obsFileName)
obs.loadWvlCalFile(wvlCalFileName)
obs.loadFlatCalFile(flatCalFileName)
obs.loadFluxCalFile(fluxCalFileName)
print "loaded data file and calibrations\n---------------------\n"
nRow = obs.nRow
nCol = obs.nCol
obsTime = obs.getFromHeader("exptime")
#wvlBinEdges,obsSpectra = loadSpectra(obs,nCol,nRow)
#nWvlBins=len(wvlBinEdges)-1
#print np.shape(obsSpectra)
#print nRow
#print nCol
#print nWvlBins
"""
medianObsSpectrum = calculateMedian(obsSpectra,nCol,nRow,nWvlBins)
print "target spectrum loaded\n---------------------\n"
if skyfile != "None":
sky = ObsFile(skyFileName)
sky.loadWvlCalFile(wvlCalFileName)
sky.loadFlatCalFile(flatCalFileName)
sky.loadFluxCalFile(fluxCalFileName)
skyTime = sky.getFromHeader("exptime")
wvlBinEdges,skySpectra = loadSpectra(sky,nCol,nRow)
skySpectrum = calculateMedian(skySpectra, nCol, nRow, nWvlBins)
skySpectrum = skySpectrum*float(obsTime)/float(skyTime) #scale sky spectrum to target observation time
print "sky spectrum loaded\n---------------------\n"
else:
#if no sky file given, estimate sky spectrum as median spectrum of obs file, assuming object is tiny
skySpectrum = calculateMedian(obsSpectra, nCol, nRow, nWvlBins)
print "sky spectrum estimated as median of target file spectrum\n---------------------\n"
#subtract sky spectrum from every pixel
allSkySpectrum = obsSpectra-skySpectrum
#set any negative values to 0 after sky subtraction
allSkySpectrum[allSkySpectrum<0]=0
#take median of remaining sky subtracted spectra to get median object spectrum
finalSpectrum = calculateMedian(allSkySpectrum,nCol,nRow,nWvlBins)
"""
#load/generate hot pixel mask file
HotPixFile = getTimeMaskFileName(obsFileName)
if not os.path.exists(HotPixFile):
hp.findHotPixels(obsFileName,HotPixFile)
print "Flux file pixel mask saved to %s"%(HotPixFile)
obs.loadHotPixCalFile(HotPixFile)
print "Hot pixel mask loaded %s"%(HotPixFile)
print "Making spectrum with Aperture Spectrum in ObsFile"
#use Aperture Spectrum from obsfile
medianObsSpectrum, wvlBinEdges = obs.getApertureSpectrum(pixelCol=ypix,pixelRow=xpix,radius1=apertureRadius, radius2 = apertureRadius*2.0,weighted=True, fluxWeighted=True, lowCut=3000, highCut=7000)
nWvlBins=len(wvlBinEdges)-1
#load std spectrum for comparison
try:
realSpectra = loadStd(objectName,wvlBinEdges)
print "real std spectrum loaded for reference\n---------------------\n"
stdTitle = "Rebinned Std Spectrum of %s"%(objectName)
except KeyError:
print "Key Error loading MKIDStd"
realSpectra = np.ones(nWvlBins)
stdTitle = "No MKIDStd spectrum available for %s"%(objectName)
#create plots
plotDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/plots"
plotFileName = "%s_from_%s%s.pdf"%(objectName,fluxCalObject,filenum)
fullFluxPlotFileName = os.path.join(plotDir,plotFileName)
pp = PdfPages(fullFluxPlotFileName)
matplotlib.rcParams['font.size']=6
#calculate midpoints of wvl bins for plotting
wvls = np.empty((nWvlBins),dtype=float)
for n in xrange(nWvlBins):
binsize=wvlBinEdges[n+1]-wvlBinEdges[n]
wvls[n] = (wvlBinEdges[n]+(binsize/2.0))
plt.figure()
ax1 = plt.subplot(221)
ax1.set_title('ARCONS median flat/flux cal\'d obs in counts')
ax1.set_xlim((4000,11000))
ax1.set_ylim((min(medianObsSpectrum[(wvls>4000) & (wvls<11000)]),max(medianObsSpectrum[(wvls>4000) & (wvls<8000)])))
plt.plot(wvls,medianObsSpectrum)
#plt.show()
#ax2 = plt.subplot(232)
#ax2.set_title('ARCONS median flat/flux cal\'d sky in counts')
#plt.plot(wvls,skySpectrum)
#plt.show()
ax5 = plt.subplot(223)
ax5.set_title('Sensitivity Spectrum')
ax5.set_xlim((3000,13000))
ax5.set_ylim((0,5))
plt.plot(wvls,obs.fluxWeights)
#ax3 = plt.subplot(234)
#ax3.set_title('MKID data minus sky in counts')
#plt.plot(wvls,finalSpectrum/max(finalSpectrum))
ax4 = plt.subplot(222)
ax4.set_title(stdTitle)
plt.plot(wvls,realSpectra)
#ax7 = plt.subplot(337)
#ax7.set_title('Flux Cal\'d ARCONS Spectrum of Std')
#plt.plot(wvls,fluxFactors*subtractedSpectra)
pp.savefig()
pp.close()
#del obs
#del sky
print "output plots to %s\n---------------------\n"%(fullFluxPlotFileName)
txtDir = "/home/srmeeker/ARCONS-pipeline/fluxcal/test/txt"
txtFileName = "%s_from_%s%s.txt"%(objectName,fluxCalObject,filenum)
fullFluxTxtFileName = os.path.join(txtDir,txtFileName)
outarr = np.empty((len(medianObsSpectrum),2),dtype=float)
outarr[:,0]=wvls
outarr[:,1]=medianObsSpectrum
#save sensitivity spectrum to file
np.savetxt(fullFluxTxtFileName, outarr)
print "output txt file to %s\n---------------------\n"%(fullFluxPlotFileName)
class DisplayStack(QMainWindow):
def __init__(self):
# Start up gui
QWidget.__init__(self, parent=None)
self.ui = Ui_DisplayStack_gui()
self.ui.setupUi(self)
# Initialize Variables
self.initializeVariables()
# Lists and buttons used for specifying run and target information.
# Click item in runList to select a run. Run number corresponds to array index. Load up corresponding target list.
self.ui.runList.itemClicked.connect(self.selectRun)
# Click item in targetList to select a target.
self.ui.targetList.itemClicked.connect(self.selectTarget)
# Click button in
self.ui.targetButton.clicked.connect(self.loadTarget)
self.ui.sunsetList.itemClicked.connect(self.createObsList)
# Use wavelength calibration checkbox
self.ui.wavelengthCalibrationBox.clicked.connect(self.useWaveCal)
# Use flat calibration checkbox
self.ui.flatCalibrationBox.clicked.connect(self.useFlatCal)
# Buttons for obs list creation
self.ui.addButton.clicked.connect(self.addObservation)
self.ui.removeButton.clicked.connect(self.removeObservation)
self.ui.clearButton.clicked.connect(self.createObsList)
# Start process button
self.ui.stackButton.clicked.connect(self.stackProcess)
# Load image stack button
self.ui.loadStackButton.clicked.connect(self.chooseStack)
# Initialize Variables
def initializeVariables(self):
# Define path names
self.displayStackPath = '/Scratch/DisplayStack/'
self.defaultWavelengthPath = '/Scratch/waveCalSolnFiles/PAL2012/master_cals/'
self.defaultFlatPath = '/Scratch/flatCalSolnFiles/'
# Load and display list of run names from /Scratch/DisplayStack/runList.dict
self.loadRunData()
# Load list of target names from /Scratch/DisplayStack/runName/runName.dict, for all runs
self.loadTargetData()
# Function to load list of run names. Runs on initialization.
def loadRunData(self):
# Load run data from /Scratch/DisplayStack/runList.dict
self.runData = readDict()
self.runData.read_from_file(self.displayStackPath + '/runList.dict')
self.runNames = np.array(self.runData['runs'])
# Populate runList table with run names.
for iRun in range(len(self.runNames)):
self.ui.runList.addItem(self.runNames[iRun])
# Function to load a table of target names.
def loadTargetData(self):
self.targetNames = []
# Cycle through runs and extract target name information from various dictionaries.
for iRun in range(len(self.runNames)):
self.targetData = readDict()
self.targetData.read_from_file(self.displayStackPath + self.runNames[iRun] + '/' + self.runNames[iRun] + '.dict')
self.iTargets = np.array(self.targetData['targets'])
self.targetNames.append(self.iTargets)
# Function to select a run and populate the target list for that particular run.
def selectRun(self):
# Clear list of target information for previously selected run.
self.ui.targetList.clear()
# Define a run number by the index of the selected run.
self.runNumber = self.ui.runList.row(self.ui.runList.currentItem())
# Populate targetList table with target names for selected run.
for iTarget in range(len(self.targetNames[self.runNumber])):
self.ui.targetList.addItem(self.targetNames[self.runNumber][iTarget])
def selectTarget(self):
self.targetNumber = self.ui.targetList.row(self.ui.targetList.currentItem())
def loadTarget(self):
self.run = self.runNames[self.runNumber]
self.target = self.targetNames[self.runNumber][self.targetNumber]
try:
self.paramName = self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.paramData = readDict()
self.paramData.read_from_file(self.paramName)
self.obsTimes = np.array(self.paramData['obsTimes'])
self.utcDates = self.paramData['utcDates']
self.sunsetDates = self.paramData['sunsetDates']
self.calTimestamps = self.paramData['calTimestamps']
self.flatCalDates = self.paramData['flatCalDates']
self.RA = self.paramData['RA']
self.Dec = self.paramData['Dec']
self.hourAngleOffset = self.paramData['HA_offset']
print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.createSunsetList()
#self.createObsList()
self.createWavelengthList()
self.createFlatList()
self.paramFileExists = True
except IOError:
print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.ui.sunsetList.clear()
self.ui.obsList.clear()
self.ui.inputList.clear()
self.ui.wavelengthList.clear()
self.ui.flatList.clear()
self.paramFileExists = False
# Choose Obs File
# Create list of available sunset dates
def createSunsetList(self):
self.ui.sunsetList.clear()
for iDate in range(len(self.sunsetDates)):
self.ui.sunsetList.addItem(self.sunsetDates[iDate])
# Create Initial Obs file list
def createObsList(self):
self.ui.obsList.clear()
self.ui.inputList.clear()
self.currentSunsetDate = self.sunsetDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
self.currentUTCDate = self.utcDates[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
self.singleDayObservations = self.obsTimes[self.ui.sunsetList.row(self.ui.sunsetList.currentItem())]
for iObs in range(len(self.singleDayObservations)):
self.ui.obsList.addItem(self.singleDayObservations[iObs])
# Add observation to input list
def addObservation(self):
self.selectedObs = self.ui.obsList.currentItem()
self.ui.obsList.takeItem(self.ui.obsList.row(self.selectedObs))
self.ui.inputList.addItem(self.selectedObs)
self.ui.inputList.sortItems()
# Remove observation from input list
def removeObservation(self):
self.removedObs = self.ui.inputList.currentItem()
self.ui.inputList.takeItem(self.ui.inputList.row(self.removedObs))
self.ui.obsList.addItem(self.removedObs)
self.ui.obsList.sortItems()
# Load settings
def loadSettings(self):
self.validSettings = True
# Run and target information
self.run = self.runNames[self.runNumber]
self.target = self.targetNames[self.runNumber][self.targetNumber]
# General settings
self.integrationTime = int(self.ui.integrationTimeLine.text())
self.useTimeAdjustment = self.ui.timeAdjustmentBox.isChecked()
self.useHotPixelMasking = self.ui.hotPixelBox.isChecked()
# Wavelength calibration settings
self.useWavelengthCalibration = self.ui.wavelengthCalibrationBox.isChecked()
self.useBestWavelengthCalibration = self.ui.bestWavelengthCalibrationBox.isChecked()
self.lowerWavelengthCutoff = float(self.ui.lowerWavelengthCutoffLine.text())
self.upperWavelengthCutoff = float(self.ui.upperWavelengthCutoffLine.text())
# Flat calibration settings
self.useFlatCalibration = self.ui.flatCalibrationBox.isChecked()
self.useDeadPixelMasking = self.ui.deadPixelBox.isChecked()
self.fileCount = self.ui.inputList.count()
self.weighted = self.useFlatCalibration
self.useRawCounts = not (self.useWavelengthCalibration and self.useFlatCalibration)
self.scaleByEffInt = self.useHotPixelMasking
if self.ui.sunsetList.currentItem() != None:
if self.fileCount == 0:
print 'Please select files to process...'
self.validSettings = False
if self.useWavelengthCalibration:
if self.useBestWavelengthCalibration:
print 'Using best wavelength calibration...'
elif self.ui.wavelengthList.currentItem() == None:
print 'Please select wavelength calibration...'
self.validSettings = False
else:
self.selectedWvlCal = self.ui.wavelengthList.currentItem().text()
print "Chose wvl cal: ", str(self.defaultWavelengthPath+self.selectedWvlCal)
self.wvlCalFilename = str(self.defaultWavelengthPath+self.selectedWvlCal)
#self.wvlCalFilename = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.selectedWvlCal).calSoln())
if self.useFlatCalibration:
if self.ui.flatList.currentItem() == None:
print 'Please select flat calibration...'
self.validSettings = False
else:
self.flatCalNight = self.ui.flatList.currentItem().text()
self.flatCalFilename = str(FileName(run=self.run,date=self.flatCalNight).flatSoln())
else:
print 'Please select sunset night...'
self.validSettings = False
# Load hot pixel mask
def loadHotMask(self):
#self.hotPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/HotPixelMasks/hotPixelMask_' + self.obsTS + '.h5')
self.hotPixelFilename = str(FileName(obsFile = self.ob).timeMask())
if not os.path.exists(self.hotPixelFilename):
hp.findHotPixels(obsFile=self.ob,outputFileName=self.hotPixelFilename)
print "Hot pixel mask saved to %s"%(self.hotPixelFilename)
self.ob.loadHotPixCalFile(self.hotPixelFilename,switchOnMask=True)
# Create wavelength cal file list
def createWavelengthList(self):
self.ui.wavelengthList.clear()
for iCal in range(len(self.calTimestamps)):
self.ui.wavelengthList.addItem(self.calTimestamps[iCal])
# Enable/disable wavecal options
def useWaveCal(self):
if self.ui.wavelengthCalibrationBox.isChecked():
self.ui.lowerWavelengthCutoffLine.setEnabled(True)
self.ui.lowerWavelengthCutoffLabel.setEnabled(True)
self.ui.upperWavelengthCutoffLine.setEnabled(True)
self.ui.upperWavelengthCutoffLabel.setEnabled(True)
self.ui.wavelengthList.setEnabled(True)
self.ui.flatCalibrationBox.setEnabled(True)
self.ui.flatCalibrationBox.setChecked(True)
self.ui.deadPixelBox.setEnabled(True)
self.ui.deadPixelBox.setChecked(True)
self.ui.flatList.setEnabled(True)
self.ui.bestWavelengthCalibrationBox.setEnabled(True)
self.ui.bestWavelengthCalibrationBox.setChecked(True)
else:
self.ui.lowerWavelengthCutoffLine.setEnabled(False)
self.ui.lowerWavelengthCutoffLabel.setEnabled(False)
self.ui.upperWavelengthCutoffLine.setEnabled(False)
self.ui.upperWavelengthCutoffLabel.setEnabled(False)
self.ui.wavelengthList.setEnabled(False)
self.ui.flatCalibrationBox.setEnabled(False)
self.ui.flatCalibrationBox.setChecked(False)
self.ui.deadPixelBox.setChecked(False)
self.ui.deadPixelBox.setEnabled(False)
self.ui.flatList.setEnabled(False)
self.ui.bestWavelengthCalibrationBox.setEnabled(False)
self.ui.bestWavelengthCalibrationBox.setChecked(False)
# Create flat cal file list
def createFlatList(self):
self.ui.flatList.clear()
for iCal in range(len(self.flatCalDates)):
self.ui.flatList.addItem(self.flatCalDates[iCal])
# Enable/disable flatcal options
def useFlatCal(self):
if self.ui.flatCalibrationBox.isChecked():
self.ui.deadPixelBox.setEnabled(True)
self.ui.deadPixelBox.setChecked(True)
self.ui.flatList.setEnabled(True)
else:
self.ui.deadPixelBox.setChecked(False)
self.ui.deadPixelBox.setEnabled(False)
self.ui.flatList.setEnabled(False)
# Load dead pixel mask
def loadDeadMask(self):
self.deadPixelFilename = str(self.displayStackPath + self.run + '/' + self.target + '/DeadPixelMasks/deadPixelMask_' + self.obsTS + '.npz')
if not os.path.exists(self.deadPixelFilename):
self.deadMask = self.ob.getDeadPixels()
np.savez(self.deadPixelFilename,deadMask = self.deadMask)
print "Dead pixel mask saved to %s"%(self.deadPixelFilename)
else:
self.deadFile = np.load(self.deadPixelFilename)
self.deadMask = self.deadFile['deadMask']
# Describe the structure of the header row
class headerDescription(tables.IsDescription):
targetName = tables.StringCol(100, dflt='')
run = tables.StringCol(100, dflt='')
obsFileName = tables.StringCol(100, dflt='')
wvlCalFileName = tables.StringCol(100, dflt=np.nan)
flatCalFileName = tables.StringCol(100, dflt='')
deadPixFileName = tables.StringCol(100, dflt='')
hotPixFileName = tables.StringCol(100, dflt='')
nCol = tables.UInt32Col(dflt=-1)
nRow = tables.UInt32Col(dflt=-1)
lowWvlCutoff = tables.Float64Col(dflt=np.nan)
highWvlCutoff = tables.Float64Col(dflt=np.nan)
exptime = tables.Float64Col(dflt=np.nan)
lst = tables.StringCol(100, dflt='')
integrationTime = tables.Float64Col(dflt=np.nan)
RA = tables.StringCol(100, dflt='')
Dec = tables.StringCol(100, dflt='')
HA_offset = tables.Float64Col(dflt=0.0)
# Create output name
def createOutputName(self):
self.rawName = str(self.displayStackPath + self.run + '/' + self.target + '/ImageStacks/' + 'ImageStack_' + self.obsTS + '_' + str(self.integrationTime) + 's')
if self.useWavelengthCalibration and self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '_hp.h5')
elif self.useWavelengthCalibration and not self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_' + str(int(self.lowerWavelengthCutoff)) + '-' + str(int(self.upperWavelengthCutoff)) + '.h5')
elif not self.useWavelengthCalibration and self.useHotPixelMasking:
self.outputFilename = str(self.rawName + '_hp.h5')
else:
self.outputFilename = str(self.rawName + '.h5')
def createH5File(self):
# Create header and data group and table names
headerGroupName = 'header'
headerTableName = 'header'
dataGroupName = 'stack'
dataTableName = 'stack'
timeTableName = 'time'
# Create lookup names for header information
runColName = 'run'
targetColName = 'targetName'
obsFileColName = 'obsFileName'
wvlCalFileColName = 'wvlCalFileName'
flatCalFileColName = 'flatCalFileName'
nRowColName = 'nRow'
nColColName = 'nCol'
RAColName = 'RA'
DecColName = 'Dec'
deadPixColName = 'deadPixFileName'
hotPixColName = 'hotPixFileName'
lowWvlColName = 'lowWvlCutoff'
highWvlColName = 'highWvlCutoff'
expTimeColName = 'exptime'
lstColName = 'lst'
integrationTimeColName = 'integrationTime'
HA_offsetColName = 'HA_offset'
# Create and h5 output file, create header and data groups
fileh = tables.openFile(self.outputFilename, mode='w')
headerGroup = fileh.createGroup("/", headerGroupName, 'Header')
stackGroup = fileh.createGroup("/", dataGroupName, 'Image Stack')
# Create row for header information
headerTable = fileh.createTable(headerGroup, headerTableName, self.headerDescription,
'Header Info')
header = headerTable.row
# Fill in the header with possibly useful information.
header[runColName] = self.run
header[targetColName] = self.target
header[obsFileColName] = self.obsFn
header[nColColName] = self.numberCols
header[nRowColName] = self.numberRows
header[RAColName] = self.RA
header[DecColName] = self.Dec
header[expTimeColName] = self.exptime
header[lstColName] = self.lst
header[integrationTimeColName] = self.integrationTime
header[HA_offsetColName] = self.hourAngleOffset
if self.useDeadPixelMasking:
header[deadPixColName] = self.deadPixelFilename
if self.useHotPixelMasking:
header[hotPixColName] = self.hotPixelFilename
if self.useWavelengthCalibration:
header[wvlCalFileColName] = self.ob.wvlCalFileName
header[lowWvlColName] = self.lowerWavelengthCutoff
header[highWvlColName] = self.upperWavelengthCutoff
if self.useFlatCalibration:
header[flatCalFileColName] = self.flatCalFilename
header.append()
# Create an h5 array for the midtime of each frame in the image cube.
timeTable = fileh.createCArray(stackGroup, timeTableName, Float64Atom(), (1,len(self.times)))
timeTable[:] = self.times
# Create an h5 table for the image cube.
stackTable = fileh.createCArray(stackGroup, dataTableName, Float64Atom(), (self.numberRows,self.numberCols, self.cube.shape[2]))
stackTable[:] = self.cube
# Flush the h5 output file
fileh.flush()
fileh.close()
# Start process for creating image stacks
def stackProcess(self):
# Check for valid params file
if self.paramFileExists:
# Load settings choosen from gui
self.loadSettings()
if self.validSettings:
# Loop through all files in input list
for iFile in range(self.fileCount):
# Create ObsFile instance
self.obsTS = str(self.currentUTCDate) + '-' + self.ui.inputList.item(iFile).text()
self.obsFn = str(FileName(run=self.run,date=self.currentSunsetDate,tstamp=self.obsTS).obs())
print 'Processing file ' + self.obsFn + '...'
self.ob = ObsFile(self.obsFn)
self.numberRows = self.ob.nRow
self.numberCols = self.ob.nCol
# Load time adjustment file
if self.useTimeAdjustment:
print 'Loading time adjustment file...'
self.ob.loadTimeAdjustmentFile(FileName(run=self.run).timeAdjustments())
# Load hot pixel mask
if self.useHotPixelMasking:
print 'Loading hot pixel mask...'
self.loadHotMask()
# Load wave cal solution
if self.useWavelengthCalibration:
if self.useBestWavelengthCalibration:
print 'Loading best wavelength calibration...'
self.ob.loadBestWvlCalFile()
else:
print 'Loading selected wavelength calibration...'
self.ob.loadWvlCalFile(self.wvlCalFilename)
# Load flatcal solution
if self.useFlatCalibration:
print 'Loading flat calibration...'
self.ob.loadFlatCalFile(self.flatCalFilename)
# Load dead pixel mask
if self.useDeadPixelMasking:
print 'Loading dead pixel mask...'
self.loadDeadMask()
# Set wavelength cutoffs
if self.useWavelengthCalibration:
print 'Setting wavelength cutoffs...'
self.ob.setWvlCutoffs(self.lowerWavelengthCutoff,self.upperWavelengthCutoff)
# Timing
self.unix = self.ob.getFromHeader('unixtime')
self.startJD = self.unix/86400.+2440587.5
self.exptime = self.ob.getFromHeader('exptime')
self.lst = self.ob.getFromHeader('lst')
self.times = []
self.frames = []
# Create Image Stack
print 'Stacking images...'
for iSec in np.arange(0,self.exptime,self.integrationTime):
#add seconds offset to julian date, move jd to center of bin
self.jd = self.startJD + iSec/(24.*3600.) + self.integrationTime/2./(24.*3600.)
self.times.append(self.jd)
print 'Creating frame for time ' + str(self.jd)
self.frameData = self.ob.getPixelCountImage(firstSec=iSec,integrationTime=self.integrationTime,weighted=self.weighted,getRawCount=self.useRawCounts,scaleByEffInt=self.scaleByEffInt)
self.frame = self.frameData['image']
if self.ui.verticalFlipBox.isChecked():
self.frame = np.flipud(self.frame)
if self.useDeadPixelMasking:
self.frame[self.deadMask == 0] = np.nan
self.frames.append(self.frame)
self.cube = np.dstack(self.frames)
self.times = np.array(self.times)
# Create output file
self.createOutputName()
print 'Saving image stack to ' + self.outputFilename
self.createH5File()
# Invalid params file
else:
print 'Invalid parameter file...'
# Choose an image stack
def chooseStack(self):
self.defaultLoadStackDirectory = str(self.displayStackPath)
self.stackName = ''
self.stackName = QFileDialog.getOpenFileName(parent=None, directory=self.defaultLoadStackDirectory, caption=str("Choose Image Stack"), filter=str("H5 (*.h5)"))
if self.stackName == '':
print 'No file chosen'
else:
loadStackApp = LoadImageStack.LoadImageStack(stackName = self.stackName)
loadStackApp.show()
loadStackApp.exec_()
tic = time()
print 'Retrieving photon timestamps...'
# Define the area of an observation containing the object.
appMask = aperture(15,8,8)
yValues,xValues = np.where(appMask==0)
# Get a list of timestamps with photons from each pixel in the aperture.
timestamps=[]
for j in range(len(xValues)):
x=ob.getPixelWvlList(iRow=yValues[j],iCol=xValues[j])
timestamps = np.append(timestamps,x['timestamps'])
print 'Total retrieval time: ' + str(time()-tic) + 's'
# Use timestamp data to perform periodograms.
timestep = 5.0*10**-4
averagingTime = 2.0
exptime = ob.getFromHeader('exptime')
totalAverageBins = int(exptime/averagingTime)
timestepsPerBin = int(averagingTime/timestep)
# Specify frequencies for which the periodogram algorithm should find transform components.
freqs = np.linspace(1,1000,num=10**4)
angularFreqs=2*np.pi*freqs
binnedCounts, binEdges = np.histogram(timestamps,range=[0,exptime],bins = exptime/timestep)
times = binEdges[0:timestepsPerBin]
periodogramStack=np.zeros((totalAverageBins,len(freqs)))
tic = time()
print 'Calculating individual Fourier Transforms...'
for bin in range(int(totalAverageBins)):
counts = binnedCounts[bin*timestepsPerBin:(bin+1)*timestepsPerBin]
scaledCounts = (counts-counts.mean())/counts.std()
out = '/home/mstrader/data/nltt/'+'timestream'+runlabel+'.txt'
outh5 = '/home/mstrader/data/nltt/'+'list'+runlabel+'.h5'
timestampList = [utcDate+'-'+ts for ts in timestampListPost]
files = []
pl = []
times = []
for i,ts in enumerate(timestampList):
print 'loading',ts
fn = FileName(run=run,date=sunsetDate,tstamp=ts).photonList()
obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
files.append(tables.openFile(fn,mode='r'))
ob = ObsFile(obsFn)
ut = ob.getFromHeader('unixtime')
pl.append(files[i].root.photons.photons.read())
times.append(np.array(pl[i]['ArrivalTime'],dtype=np.float64) + ut)
all=np.concatenate(pl)
allTimes = np.concatenate(times)
nlttPSFByPixels = []
nlttTimesByPixels = []
img = np.zeros((46,44))
rawImg = np.zeros((46,44))
for iPixel in range(len(circCol)):
x = circCol[iPixel]
y = circRow[iPixel]
timesInPixel = allTimes[np.logical_and(all['Xpix']==x,all['Ypix']==y)]
if not os.path.exists(hotPixFn):
hp.findHotPixels(obsFn,hotPixFn)
print "Flux file pixel mask saved to %s"%(hotPixFn)
ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
ob.loadWvlCalFile(wfn)
ob.loadFlatCalFile(ffn)
ob.setWvlCutoffs(wvlLowerCutoff,wvlUpperCutoff)
bad_solution_mask=np.zeros((46,44))
bad_count=0;
for y in range(46):
for x in range(44):
if (ob.wvlRangeTable[y][x][1] < wvlUpperCutoff) or (ob.wvlRangeTable[y][x][0] > wvlLowerCutoff):
bad_solution_mask[y][x] = 1
unix = ob.getFromHeader('unixtime')
startJD = unix/86400.+2440587.5
nSecInFile = ob.getFromHeader('exptime')
#tic = time()
deadMask = ob.getDeadPixels()
#print 'Dead mask load time = ', time()-tic
for sec in np.arange(0,nSecInFile,integrationTime):
jd = startJD + sec/(24.*3600.) + integrationTime/2./(24.*3600.)#add seconds offset to julian date, move jd to center of bin
print count,jd
count+=1
times.append(jd)
titles.append('%.6f'%jd)
frameData = ob.getPixelCountImage(firstSec=sec,integrationTime=integrationTime,weighted=True,scaleByEffInt=True)
frame = frameData['image']
showFrame = np.array(frame)
y_values,x_values = np.where(app_mask==0)
unixOffset=0.0
timestamps =[]
tic = time()
for iSeq in range(len(seqs)):
timestampList = timestampLists[iSeq]
wfn = wvlCalFilenames[iSeq]
ffn = flatCalFilenames[iSeq]
sunsetDate = sunsetDates[iSeq]
for i,ts in enumerate(timestampList):
print 'Loading',ts
obsFn = FileName(run=run,date=sunsetDate,tstamp=ts).obs()
ob = ObsFile(obsFn)
ob.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
exptime = ob.getFromHeader('exptime')
unixtime= ob.getFromHeader('unixtime')
if i == 0:
unixOffset = unixtime
exptimes[i]=exptime
index1 = obsFn.find('_')
hotPixFn = '/Scratch/timeMasks/timeMask' + obsFn[index1:]
if not os.path.exists(hotPixFn):
hp.findHotPixels(obsFn,hotPixFn)
print "Flux file pixel mask saved to %s"%(hotPixFn)
ob.loadHotPixCalFile(hotPixFn,switchOnMask=True)
ob.loadWvlCalFile(wfn)
ob.loadFlatCalFile(ffn)
ob.setWvlCutoffs(3000,8000)
for j in range(len(x_values)):
x=ob.getPixelWvlList(iRow=y_values[j],iCol=x_values[j])