def unremovePixel(timeMaskPath,pixelRow,pixelCol,reasons=['manual hot pixel','manual cold pixel'],nodePath='/'):
timeMaskFile = tables.openFile(timeMaskPath,mode='a')
hotPixelsOb = readHotPixels(timeMaskFile)
expTime = hotPixelsOb.expTime
reasonEnum = hotPixelsOb.reasonEnum
try:
reasonValues = [reasonEnum[eachReason] for eachReason in reasons]
except IndexError:
outStr = 'reason given for removing pixel,{}, is not in the timeMaskReason enum'.format(reason)
raise IndexError(outStr)
tableName = constructDataTableName(x=pixelCol, y=pixelRow)
eventListTable = timeMaskFile.getNode(nodePath + dataGroupName, name=tableName)
print eventListTable.read()
#find the rows where the reason flag matches one of those in the reasons list
rowIndicesWithGivenReasons = np.where(np.in1d(eventListTable.cols.reason,reasonValues))[0]
print rowIndicesWithGivenReasons
#as we start iterating through and deleting rows, the row numbers will change
#so we need to correct the row indices accordingly
#subtract from each row the number of rows deleted before it
rowIndicesWithGivenReasons.sort()
correctedRowIndices = rowIndicesWithGivenReasons - np.arange(len(rowIndicesWithGivenReasons))
#delete rows
for eachRowIdx in correctedRowIndices:
eventListTable.removeRows(eachRowIdx)
timeMaskFile.flush()
print eventListTable.read()
timeMaskFile.close()
def __init__(self,timeMaskPath,pixelRow,pixelCol,parent=None,nodePath='/'):
super(EditTimeMaskDialog,self).__init__(parent=parent)
self.parent=parent
self.timeMaskFile = tables.openFile(timeMaskPath,mode='a')
self.hotPixelsOb = readHotPixels(self.timeMaskFile)
self.expTime = self.hotPixelsOb.expTime
self.reasonEnum = self.hotPixelsOb.reasonEnum
self.reasonList = self.hotPixelsOb.reasons[pixelRow,pixelCol]
self.intervalList = self.hotPixelsOb.intervals[pixelRow,pixelCol]
self.nEntries = len(self.reasonList)
tableName = constructDataTableName(x=pixelCol, y=pixelRow)
self.eventListTable = self.timeMaskFile.getNode(nodePath + dataGroupName, name=tableName)
self.initUI()
self.exec_()
def removePixel(timeMaskPath,pixelRow,pixelCol,reason='manual hot pixel',nodePath='/',timeInterval=None):
timeMaskFile = tables.openFile(timeMaskPath,mode='a')
hotPixelsOb = readHotPixels(timeMaskFile)
expTime = hotPixelsOb.expTime
reasonEnum = hotPixelsOb.reasonEnum
if not timeInterval is None:
if len(timeInterval) == 2:
tBegin = timeInterval[0]*hotPixelsOb.ticksPerSec
tEnd = timeInterval[1]*hotPixelsOb.ticksPerSec
else:
raise TypeError('timeInterval needs to be a tuple (tBegin,tEnd), instead given:{}'.format(timeInterval))
else:
tBegin = 0.
tEnd = expTime*hotPixelsOb.ticksPerSec
try:
reasonValue = reasonEnum[reason]
except IndexError:
outStr = 'reason given for removing pixel,{}, is not in the enum embedded in the existing hot pixel file'.format(reason)
raise TypeError(outStr)
tableName = constructDataTableName(x=pixelCol, y=pixelRow)
eventListTable = timeMaskFile.getNode(nodePath + dataGroupName, name=tableName)
newRow = eventListTable.row
newRow['tBegin'] = tBegin
newRow['tEnd'] = tEnd
newRow['reason'] = reasonValue
newRow.append()
timeMaskFile.flush()
timeMaskFile.close()
def quantifyBadTime(inputFileName, startTime=0, endTime=-1,
defaultTimeMaskFileName='./testTimeMask.h5',
timeStep=1, fwhm=3.0, boxSize=5, nSigmaHot=3.0,
nSigmaCold=2.5,maxIter=5,binWidth=3,
dispToPickle=False, showHist=False, bkgdPercentile=50,
weighted=False,fluxWeighted=False, useRawCounts=True):
'''
Function to calculate various metrics for the degree of bad pixel behaviour in a raw
raw obs file. Calculates the mean total hot/cold/dead time per good pixel (i.e., per
pixel which is not permanently dead, hot, or cold).
Makes a couple of heat maps showing time spent bad in each way for each pixel,
as well as a histogram of times spent bad for the *temporarily* bad pixels.
JvE Nov 20 2013.
The parameters for the finding algorithm may need to be tuned a little,
but the defaults should basically work, in principle. nSigmaHot and nSigmaCold
are good places to start if things need playing around with.
e.g. call, in principle:
from hotpix import quantifyHotTime as qht
qht.quantifyHotTime('/Users/vaneyken/Data/UCSB/ARCONS/turkDataCopy/ScienceData/PAL2012/20121208/obs_20121209-120530.h5')
- should be all it needs....
INPUTS:
inputFileName - either a raw obs. filename or a time mask file. If the former,
runs a hot pixel search; otherwise uses time mask file instead.
startTime, endTime - start and end times within the obs file to
calculate the hot pixels for. (endTime =-1 means to end of file).
defaultTimeMaskFileName - use this filename to output new time mask to
(if inputFileName is an obs file)
binWidth - width of time bins for plotting the bad-time histogram (seconds)
dispToPickle - if not False, saves the data for the histogram plot to a pickle file.
If a string, then uses that as the name for the pickle file. Otherwise
saves to a default name. Saves a dictionary with four entries, the first
three of which are each a flat array of total times spent bad for every
pixel (in sec) ('hotTime','coldTime','deadTime'). The fourth, 'duration',
is the duration of the input time-mask in seconds.
showHist - if True, plot up a histogram of everything. Currently fails though
if there are no bad-flagged intervals in any of the type categories
(or their intersections, hot+cold, cold+dead).
Parameters passed on to findHotPixels routine if called (see also documentation
for that function):
timeStep #Check for hot pixels every timeStep seconds
fwhm #Expected full width half max of PSF in pixels. Any pixel
#with flux much tighter than this will be flagged as bad.
#Larger value => more sensitive hot pixel flagging.
boxSize #Compare flux in each pixel with median flux in a
#surrounding box of this size on a side.
nSigmaHot #Require flux in a pixel to be > nSigmaHot std. deviations
#above the max expected for a Gaussian PSF in order for it
#to be flagged as hot. Larger value => less sensitive flagging.
nSigmaCold #Require flux to be < nSigmaCold std. deviations below the median
#in a surrounding box in order to be flagged as cold (where std.
#deviation is estimated as the square root of the median flux).
maxIter #Max num. of iterations for the bad pixel detection algorithm.
bkdgPercentile, weighted, fluxWeighted - See findHotPixels() in hotpix.hotpixels.py
OUTPUTS:
A bunch of statistics on the different kinds of bad pixel behaviour, and a
'heat' plot showing how much time each pixel was bad in the array, for each
type of behaviour. In theory it can plot a histogram of everything too, but
currently it breaks if there are no bad intervals within any of the type
categories (hot only, hot and cold, cold only, cold and dead, dead only...)
'''
defaultPklFileName = 'badPixTimeHist.pickle'
#Check whether the input file is a time mask or a regular obs file.
absInputFileName = os.path.abspath(inputFileName) #To avoid weird issues with the way findHotPixels expands paths....
hdffile = tables.openFile(absInputFileName)
inputIsTimeMaskFile = '/timeMasks' in hdffile
hdffile.close()
#Decide whether to generate a new time mask file or not
if inputIsTimeMaskFile:
print 'Input file is a time mask file'
timeMaskFileName = absInputFileName
else:
print 'Assuming input file is an obs. file'
timeMaskFileName = os.path.abspath(defaultTimeMaskFileName)
if os.path.exists(timeMaskFileName):
response=''
while response != 'u' and response !='r':
response = raw_input(timeMaskFileName+' already exists - "u" to use this (default) or "r" to regenerate? ')
response = response.strip().lower()
if response == '': response = 'u'
else:
response = 'r' #If the file *didn't already exist, pretend the user entered 'r' despite not having been asked.
if response == 'r':
#Make/regenerate the hot pixel file.
print 'Making new hot pixel time mask file '+timeMaskFileName
hp.findHotPixels(inputFileName=absInputFileName,
outputFileName=timeMaskFileName,
startTime=startTime,
endTime=endTime,
timeStep=timeStep, fwhm=fwhm,
boxSize=boxSize, nSigmaHot=nSigmaHot, nSigmaCold=nSigmaCold,
display=True, dispToPickle=dispToPickle,
maxIter=maxIter, bkgdPercentile=bkgdPercentile,
weighted=weighted,fluxWeighted=fluxWeighted,useRawCounts=useRawCounts)
#Read in the time mask file and calculate hot, cold, and 'other' bad time per pixel.
timeMask = hp.readHotPixels(timeMaskFileName)
hotTime = np.zeros((timeMask.nRow,timeMask.nCol))
coldTime = np.zeros((timeMask.nRow,timeMask.nCol))
deadTime = np.zeros((timeMask.nRow,timeMask.nCol))
otherTime = np.zeros((timeMask.nRow,timeMask.nCol))
hotIntervals = np.array([])
coldIntervals = np.array([])
deadIntervals = np.array([])
otherIntervals = np.array([])
reasonStringMap = timeMask.reasonEnum
for iRow in range(timeMask.nRow):
for iCol in range(timeMask.nCol):
for eachInterval,eachReasonCode in zip(timeMask.intervals[iRow,iCol], timeMask.reasons[iRow,iCol]):
eachReasonString = reasonStringMap(eachReasonCode) #Convert integer code to human readable string
intSize = utils.intervalSize(eachInterval)
if eachReasonString == 'hot pixel':
hotTime[iRow,iCol] += intSize
hotIntervals = np.append(hotIntervals, intSize)
elif eachReasonString == 'cold pixel':
coldTime[iRow,iCol] += intSize
coldIntervals = np.append(coldIntervals, intSize)
elif eachReasonString == 'dead pixel':
deadTime[iRow,iCol] += intSize
deadIntervals = np.append(deadIntervals, intSize)
else:
otherTime[iRow,iCol] += intSize
otherIntervals = np.append(otherIntervals, intSize)
if np.size(hotIntervals) == 0: hotIntervals = np.array([-1])
if np.size(coldIntervals) == 0: coldIntervals = np.array([-1])
if np.size(deadIntervals) == 0: deadIntervals = np.array([-1])
if np.size(otherIntervals) == 0: otherIntervals = np.array([-1])
totBadTime = hotTime+coldTime+deadTime+otherTime
maskDuration = timeMask.endTime-timeMask.startTime
#Figure out which pixels are hot, cold, permanently hot, temporarily cold, etc. etc.
nPix = timeMask.nRow * timeMask.nCol
hotPix = hotTime > 0.1
coldPix = coldTime > 0.1
deadPix = deadTime > 0.1
otherPix = otherTime > 0.1
multiBehaviourPix = ( (np.array(hotPix,dtype=int)+np.array(coldPix,dtype=int)
+np.array(deadPix,dtype=int)+np.array(otherPix,dtype=int)) > 1)
#assert np.all(deadTime[deadPix] == maskDuration) #All dead pixels should be permanently dead....
tol = timeStep/10. #Tolerance for the comparison operators below.
permHotPix = hotTime >= maskDuration-tol
permColdPix = coldTime >= maskDuration-tol
permDeadPix = deadTime >= maskDuration-tol
permOtherPix = otherTime >= maskDuration-tol
permGoodPix = (hotTime+coldTime+deadTime+otherTime < tol)
permBadPix = permHotPix | permColdPix | permDeadPix | permOtherPix
tempHotPix = (hotTime < maskDuration-tol) & (hotTime > tol)
tempColdPix = (coldTime < maskDuration-tol) & (coldTime > tol)
tempDeadPix = (deadTime < maskDuration-tol) & (deadTime > tol)
tempOtherPix = (otherTime < maskDuration-tol) & (otherTime > tol)
tempGoodPix = tempHotPix | tempColdPix | tempDeadPix | tempOtherPix #Bitwise or should work okay with boolean arrays
tempBadPix = tempGoodPix #Just to be explicit about it....
nGoodPix = np.sum(permGoodPix | tempGoodPix) #A 'good pixel' is either temporarily or permanently good
#assert np.sum(tempDeadPix) == 0 #Shouldn't be any temporarily dead pixels. APPARENTLY THERE ARE....
assert np.all((tempHotPix & permHotPix)==False) #A pixel can't be permanently AND temporarily hot
assert np.all((tempColdPix & permColdPix)==False) #... etc.
assert np.all((tempOtherPix & permOtherPix)==False)
assert np.all((tempDeadPix & permDeadPix)==False)
#Print out the results
print '----------------------------------------------'
print
print '# pixels total: ', nPix
print 'Mask duration (sec): ', maskDuration
print
print '% hot pixels: ', float(np.sum(permHotPix+tempHotPix))/nPix * 100.
print '% cold pixels: ', float(np.sum(permColdPix+tempColdPix))/nPix * 100.
print '% dead pixels: ', float(np.sum(permDeadPix+tempDeadPix))/nPix * 100.
print '% other pixels: ', float(np.sum(permOtherPix+tempOtherPix))/nPix * 100.
print
print '% permanently hot pixels: ', float(np.sum(permHotPix))/nPix * 100.
print '% permanently cold pixels: ', float(np.sum(permColdPix))/nPix * 100.
print '% permanently dead pixels: ', float(np.sum(permDeadPix))/nPix * 100.
print '% permanently "other" bad pixels: ', float(np.sum(permOtherPix))/nPix * 100.
print
print '% temporarily hot pixels: ', float(np.sum(tempHotPix))/nPix * 100.
print '% temporarily cold pixels: ', float(np.sum(tempColdPix))/nPix * 100.
print '% temporarily dead pixels: ', float(np.sum(tempDeadPix))/nPix * 100.
print '% temporarily "other" bad pixels: ', float(np.sum(tempOtherPix))/nPix * 100.
print
print '% pixels showing multiple bad behaviours: ', float(np.sum(multiBehaviourPix))/nPix * 100.
print
print '% permanently bad pixels: ', float(np.sum(permBadPix))/nPix * 100.
print '% temporarily bad pixels: ', float(np.sum(tempGoodPix))/nPix * 100. #Temp. good == temp. bad!
print '% permanently good pixels: ', float(np.sum(permGoodPix))/nPix * 100.
print
print 'Mean temp. hot pixel time per good pixel: ', np.sum(hotTime[tempHotPix])/nGoodPix
print 'Mean temp. hot pixel time per temp. hot pixel: ', np.sum(hotTime[tempHotPix])/np.sum(tempHotPix)
print
print 'Mean temp. cold pixel time per good pixel: ', np.sum(coldTime[tempColdPix])/nGoodPix
print 'Mean temp. cold pixel time per temp. cold pixel: ', np.sum(coldTime[tempColdPix])/np.sum(tempColdPix)
print
print 'Mean temp. "other" bad pixel time per good pixel: ', np.sum(otherTime[tempOtherPix])/nGoodPix
print 'Mean temp. "other" bad pixel time per temp. "other" pixel: ', np.sum(otherTime[tempOtherPix])/np.sum(tempOtherPix)
print
print '(All times in seconds)'
print
print 'Done.'
print
if np.sum(tempOtherPix) > 0 or np.sum(permOtherPix) > 0:
print '--------------------------------------------------------'
print 'WARNING: Pixels flagged for "other" reasons detected - '
print 'Histogram plot will not account for these!!'
print '--------------------------------------------------------'
#Display contour plots of the array of total bad times for each pixel for each type of behaviour
utils.plotArray(hotTime, plotTitle='Hot time per pixel (sec)', fignum=None, cbar=True)
utils.plotArray(coldTime, plotTitle='Cold time per pixel (sec)', fignum=None, cbar=True)
utils.plotArray(otherTime, plotTitle='Other bad time per pixel (sec)', fignum=None, cbar=True)
utils.plotArray(deadTime, plotTitle='Dead time per pixel (sec)', fignum=None, cbar=True)
#Make histogram of time spent 'bad' for the temporarily bad pixels.
#Ignore pixels flagged as bad for 'other' reasons (other than hot/cold/dead),
#of which there should be none at the moment.
assert np.all(otherPix == False)
#Find total bad time for pixels which go only one of hot, cold, or dead
onlyHotBadTime = totBadTime[hotPix & ~coldPix & ~deadPix]
onlyColdBadTime = totBadTime[~hotPix & coldPix & ~deadPix]
onlyDeadBadTime = totBadTime[~hotPix & ~coldPix & deadPix]
#Find total bad time for pixels which alternate between more than one bad state
hotAndColdBadTime = totBadTime[hotPix & coldPix & ~deadPix]
hotAndDeadBadTime = totBadTime[hotPix & ~coldPix & deadPix]
coldAndDeadBadTime = totBadTime[~hotPix & coldPix & deadPix]
hotAndColdAndDeadBadTime = totBadTime[hotPix & coldPix & deadPix]
allGoodBadTime = totBadTime[~hotPix & ~coldPix & ~deadPix]
assert np.sum(allGoodBadTime) == 0
if dispToPickle is not False:
#Save to pickle file to feed into a separate plotting script, primarily for
#the pipeline paper.
if type(dispToPickle) is str:
pklFileName = dispToPickle
else:
pklFileName = defaultPklFileName
pDict = {'hotTime':hotTime,
'coldTime':coldTime,
'deadTime':deadTime,
'onlyHotBadTime':onlyHotBadTime,
'onlyColdBadTime':onlyColdBadTime,
'onlyDeadBadTime':onlyDeadBadTime,
'hotAndColdBadTime':hotAndColdBadTime,
'hotAndDeadBadTime':hotAndDeadBadTime,
'coldAndDeadBadTime':coldAndDeadBadTime,
'hotAndColdAndDeadBadTime':hotAndColdAndDeadBadTime,
'maskDuration':maskDuration}
#pDict = {"hotTime":hotTime.ravel(),"coldTime":coldTime.ravel(),"deadTime":deadTime.ravel(),
# "duration":maskDuration}
print 'Saving to file: ',pklFileName
output = open(pklFileName, 'wb')
pickle.dump(pDict,output)
output.close()
assert np.size(hotTime)==nPix and np.size(coldTime)==nPix and np.size(deadTime)==nPix
assert (len(onlyHotBadTime)+len(onlyColdBadTime)+len(onlyDeadBadTime)+len(hotAndColdBadTime)
+len(coldAndDeadBadTime)+len(hotAndDeadBadTime)+len(hotAndColdAndDeadBadTime)
+len(allGoodBadTime))==nPix
if showHist is True:
#Be sure it's okay to leave hot+dead pixels out, and hot+cold+dead pixels.
#assert len(hotAndDeadBadTime)==0 and len(hotAndColdAndDeadBadTime)==0
mpl.figure()
norm = 1. #1./np.size(hotTime)*100.
dataList = [onlyHotBadTime,hotAndColdBadTime,onlyColdBadTime,coldAndDeadBadTime,onlyDeadBadTime]
dataList2 = [x if np.size(x)>0 else np.array([-1.0]) for x in dataList] #-1 is a dummy value for empty arrays, so that pyplot.hist doesn't barf.
weights = [np.ones_like(x)*norm if np.size(x)>0 else np.array([0]) for x in dataList]
mpl.hist(dataList2, range=None, #[-0.1,maskDuration+0.001], #Eliminate data at 0sec and maskDuration sec. (always good or permanently bad)
weights=weights,
label=['Hot only','Hot/cold','Cold only','Cold/dead','Dead only'], #,'Hot/dead','Hot/cold/dead'],
color=['red','pink','lightgray','lightblue','blue'], #,'green','black'],
bins=maskDuration/binWidth,histtype='stepfilled',stacked=True,log=False)
mpl.title('Duration of Bad Pixel Behaviour - '+os.path.basename(inputFileName))
mpl.xlabel('Total time "bad" (sec)')
mpl.ylabel('Percantage of pixels')
mpl.legend()
mpl.figure()
mpl.hist(hotIntervals, bins=maskDuration/binWidth)
print 'Median hot interval: ', np.median(hotIntervals)
mpl.xlabel('Duration of hot intervals')
mpl.ylabel('Number of intervals')
mpl.figure()
mpl.hist(coldIntervals, bins=maskDuration/binWidth)
print 'Median cold interval: ', np.median(coldIntervals)
mpl.xlabel('Duration of cold intervals')
mpl.ylabel('Number of intervals')
mpl.figure()
mpl.hist(deadIntervals, bins=maskDuration/binWidth)
print 'Median dead interval: ', np.median(deadIntervals)
mpl.xlabel('Duration of dead intervals')
mpl.ylabel('Number of intervals')
mpl.figure()
mpl.hist(otherIntervals, bins=maskDuration/binWidth)
print 'Median "other" interval: ', np.median(otherIntervals)
mpl.xlabel('Duration of "other" intervals')
mpl.ylabel('Number of intervals')
print 'Mask duration (s): ',maskDuration
print 'Number of pixels: ',nPix
print 'Fraction at 0s (hot,cold,dead): ', np.array([np.sum(hotTime<tol),np.sum(coldTime<tol),
np.sum(deadTime<tol)]) / float(nPix)
print 'Fraction at '+str(maskDuration)+'s (hot,cold,dead): ', np.array([np.sum(hotTime>maskDuration-tol),
np.sum(coldTime>maskDuration-tol),
np.sum(deadTime>maskDuration-tol)])/float(nPix)
def hotPixelsTest2(startTime=12.3, endTime=23.1, getRawCount=True):
'''
Runs a check of the bad pixel time masking. Checks:
- Number of photons removed from returned image is consistent
with time masks.
- That timestamps of all photons in the time-masked image are outside
the time intervals defined for each pixel in the hot pixel file.
Outputs a test hot pixel file in the current directory, and runs checks
on it.
INPUTS:
startTime - time from beginning of obs file at which to start the check.
endTime - time from beginning to obs file at which to end (both in seconds).
getRawCounts - if True, use raw, non-wavelength calibrated photon counts
with no wavelength cutoffs applied.
'''
#dir = '/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/hotPixTest2/'
run = 'PAL2012'
date = '20121208'
obsFileName = FileName(run=run,date=date,tstamp='20121209-044636').obs() #'obs_20121209-044636.h5'
wvlCalFileName = FileName(run=run, date=date, tstamp='20121209-060704').calSoln() #'calsol_20121209-060704.h5'
flatCalFileName = FileName(run=run, date='20121210').flatSoln() #'flatsol_20121210.h5'
hotPixFileName = os.path.abspath('test-hotPix_20121209-044636.h5')
paramFile = os.path.join(os.path.dirname(__file__),'../../params/hotPixels.dict')
startTime = float(startTime) #Force these values to floats to make sure
endTime = float(endTime) #that getPixelCountImage calls getPixelSpectrum
#and applies wavelength cutoffs consistently.
if not os.path.exists(hotPixFileName):
print 'Creating hot pixel file....'
hp.findHotPixels(paramFile=paramFile, inputFileName=obsFileName,
outputFileName=hotPixFileName, timeStep=1,
startTime=0, endTime=-1,
fwhm=3.0, boxSize=5, nSigmaHot=2.5,
nSigmaCold=2.5, display=True)
print 'Done creating hot pixel file.'
print
intTime = endTime - startTime
obsFile = of.ObsFile(obsFileName)
obsFile.loadWvlCalFile(wvlCalFileName)
obsFile.loadFlatCalFile(flatCalFileName)
print 'Loading hot pixel file into obsFile...'
obsFile.loadHotPixCalFile(hotPixFileName)
obsFile.setWvlCutoffs()
print 'Getting image with masking...'
imhp = obsFile.getPixelCountImage(startTime, intTime, weighted=False,
getRawCount=getRawCount)['image']
print 'Getting image without masking...'
obsFile.switchOffHotPixTimeMask()
im = obsFile.getPixelCountImage(startTime, intTime, weighted=False,
getRawCount=getRawCount)['image']
diffim = im - imhp #Should end up containing the total number of photons masked from each pixel.
print 'Displaying images...'
mpl.ion()
mpl.matshow(imhp)
mpl.title('Hot-pixel masked')
mpl.colorbar()
mpl.matshow(im)
mpl.title('Unmasked')
mpl.colorbar()
print 'Loading local version of hot pixel file for direct inspection...'
hotPix = hp.readHotPixels(hotPixFileName)
if True:
print 'Checking for consistency in number of photons removed....'
for iRow in range(np.shape(diffim)[0]):
for iCol in range(np.shape(diffim)[1]):
nMaskedPhotons = 0
for eachInter in hotPix['intervals'][iRow, iCol]:
#Make sure there is only one component per interval
assert len(eachInter) == 1
#If the interval overlaps with the time range in question then
#add the number of photons in the interval to our running tally.
if eachInter[0][0] < endTime and eachInter[0][1] > startTime:
firstSec = max(eachInter[0][0], startTime)
lastSec = min(eachInter[0][1], endTime)
nMaskedPhotons += obsFile.getPixelCount(iRow, iCol, firstSec=firstSec,
integrationTime=lastSec - firstSec,
weighted=False, fluxWeighted=False,
getRawCount=getRawCount)['counts']
assert nMaskedPhotons == diffim[iRow, iCol]
print 'Okay.'
print 'Checking timestamps of remaining photons for consistency with exposure masks'
obsFile.switchOnHotPixTimeMask() #Switch on hot pixel masking
for iRow in range(np.shape(diffim)[0]):
for iCol in range(np.shape(diffim)[1]):
timeStamps = obsFile.getTimedPacketList(iRow, iCol, firstSec=startTime, integrationTime=intTime)['timestamps']
#timeStamps = timeStamps[np.logical_and(timeStamps<=endTime, timeStamps>=startTime)]
badInterval = interval.union(hotPix['intervals'][iRow, iCol])
#The following check would be nice, but doesn't work because getTimedPacketList doesn't do a wavelength cut like getPixelCount does.
#assert len(timeStamps) == im[iRow,iCol] #Double check that the number of photons returned matches the number in the masked image
#
for eachTimestamp in timeStamps:
assert eachTimestamp not in badInterval #Check that none of those photons' timestamps are in the masked time range
print 'Okay.'
print
print 'Done. All looks good.'
def hotPixelsTest(testFileName=FileName(run='PAL2012',date='20121208',tstamp='20121209-044636').obs()):
'''
Runs some basic checks for consistency between intermediate output
masks and the final 'bad time lists'.
To run
- from Python:
hotPixelsTest('someObsFile.h5')
- from command line:
python hotPixelsTest.py someObsFile.h5
(No parameter file need be supplied).
'''
workingDir = '/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/hotPixTest2/'
outputFile = workingDir + 'testoutput.h5'
paramFile = os.path.join(os.path.dirname(__file__),'../../params/hotPixels.dict') #/Users/vaneyken/UCSB/ARCONS/pipeline/github/ARCONS-pipeline/params/hotPixels.dict'
testStartTime = 2 #In seconds
testEndTime = 4 #In seconds
timeStep = 2 #In seconds (deliberately equal to start time - end time)
fwhm = 3.0
boxSize = 5
nSigmaHot = 2.5
nSigmaCold = 2.0
hp.findHotPixels(paramFile=paramFile, inputFileName=testFileName,
outputFileName=outputFile, timeStep=timeStep,
startTime=testStartTime, endTime=testEndTime,
fwhm=fwhm, boxSize=boxSize, nSigmaHot=nSigmaHot,
nSigmaCold=nSigmaCold, display=True)
intermediateOutput = hp.checkInterval(inputFileName=testFileName, display=True,
firstSec=testStartTime,
intTime=testEndTime - testStartTime,
fwhm=fwhm, boxSize=boxSize,
nSigmaHot=nSigmaHot, nSigmaCold=nSigmaCold)
hpOutput = hp.readHotPixels(outputFile)
intMask = intermediateOutput['mask'] > 0 #Make a Boolean mask - any code > 0 is bad for some reason.
intervals = hpOutput['intervals']
reasons = hpOutput['reasons']
#Find the number of entries for each pixel in both the 'intervals' and the
#'reasons' arrays.
nIntervals = np.reshape([len(x) for x in intervals.flat], np.shape(intervals))
nReasons = np.reshape([len(x) for x in reasons.flat], np.shape(reasons))
#Union the interval lists for each pixel to give an array of single (multi-component) interval objects:
uIntervals = np.reshape(np.array([interval.union(x) for x in intervals.flat],
dtype='object'), np.shape(intervals))
#Create a boolean mask that should be True for all bad (hot/cold/dead/other) pixels within the test time range
finalMask = np.reshape([(interval(testStartTime, testEndTime) in x)
for x in uIntervals.flat], np.shape(uIntervals))
assert np.all(np.equal(intMask, finalMask))
assert np.all(np.equal(nIntervals, nReasons))
print
print "All seems okay. The two plots shown should look identical."
def parseHotPixTimeMask(self):
'''Parse in the hot pixel time mask HDF data.'''
print 'Parsing pixel time-mask data'
self.hotPixTimeMask = hp.readHotPixels(self.file)
print 'Done parsing'
