def showArrayStdVsIntTime(self):
intTimes = [1,2,3,5,10,15,30]
sdevVsIntTime = []
madVsIntTime = []
medVsIntTime = []
for intTime in intTimes:
image = np.zeros((self.nRow,self.nCol))
for iOb,ob in enumerate(self.skyObList):
x = ob.getPixelCountImage(firstSec=0,integrationTime=intTime)
image+=x['image']
hotPixMask = hotPixels.checkInterval(image=image)['mask']
image[hotPixMask!=0]=0
countList = image[image!=0]
sdevVsIntTime.append(np.std(countList))
madVsIntTime.append(np.median(np.abs(countList-np.median(countList))))
medVsIntTime.append(np.median(countList))
PopUp(parent=self).plotArray(image,title=r'%d std=%f mad=%f med=%f'%(intTime,sdevVsIntTime[-1],madVsIntTime[-1],medVsIntTime[-1]))
medVsIntTime = np.array(medVsIntTime)
sqrtNVsIntTime = np.sqrt(medVsIntTime)
pop = PopUp(parent=self,title='showArrayStdVsIntTime')
pop.axes.set_xlabel('integration time (s)')
pop.axes.set_ylabel('$\sigma$')
pop.axes.plot(intTimes,sqrtNVsIntTime,'k--',label=r'$\sqrt(med(N))$')
pop.axes.plot(intTimes,sdevVsIntTime,'k')
pop.axes.plot(intTimes,madVsIntTime,'r')
pop.draw()
def createStack(self,stackLabel):
paramsLabel = stackLabel
weighted = True
if stackLabel == 'raw':
paramsLabel = 'obs'
weighted = False
getRawCounts = True
x = self.stackObsFileLists[paramsLabel][0].getSpectralCube(weighted=weighted)
spectra = x['cube']
wvlBinEdges = x['wvlBinEdges']
totalIntTime = 0
for ob in self.stackObsFileLists[paramsLabel][1:]:
print ob.fileName
x = ob.getSpectralCube(weighted=weighted)
cube = x['cube']
wvlBinEdges = x['wvlBinEdges']
spectra += cube
totalIntTime += ob.getFromHeader('exptime')
spectra = np.array(spectra,dtype=np.float64)
frame = np.sum(spectra,axis=2)
hotPixMask = hotPixels.checkInterval(image=frame)['mask']
frame[hotPixMask != 0] = np.nan
frame[frame == 0] = np.nan
stackFileName = self.params[stackLabel+'StackFileName']
np.savez(stackFileName,spectra=spectra,frame=frame,wvlBinEdges=wvlBinEdges,totalIntTime=totalIntTime)
def showTwilightArrayReducedChisqImage(self):
chisqImage = np.zeros((self.nRow,self.nCol))
nDeltaFromZero = np.zeros((self.nRow,self.nCol,self.nWvlBins))
for iRow in range(self.nRow):
for iCol in range(self.nCol):
x = self.getChisq(iRow,iCol)
chisqImage[iRow,iCol] = x['reducedChisq']
nDeltaFromZero[iRow,iCol,:] = x['nDeltaFromZero']
chisqImage[np.isnan(chisqImage)]=0
chisqImage[chisqImage == np.inf]=0
nDeltaFromZero = np.ma.array(nDeltaFromZero,mask=np.logical_and(np.isnan(nDeltaFromZero),nDeltaFromZero==np.inf))
hotPixMask = hotPixels.checkInterval(image=chisqImage)['mask']
chisqImage[hotPixMask != 0] = 0
#self.popUpArray(image=chisqImage,title='Flat Cal $\chi^{2}_{red}$',normNSigma=1.)
PopUp(parent=self,title='showTwilightArrayReducedChisqImage').plotArray(image=chisqImage,title='Flat Cal $\chi^{2}_{red}$',normNSigma=1.)
def makeRGBimage(result, fitsBaseName='Arp147-',
workingDir='/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/arp147/Arp147WorkingDir',
interpolate=False, medianSmooth=False, maskHotPixels=False):
'''
Take a 'result' (as returned by makeArp147Image()) and make an RGB image out
of it. Also create fits files of the three components.
INPUTS:
result: as returned by makeArp147Image()
fitsBaseName: fits files are output as fistBaseName+[color]+'.fits'
'''
rgb = result['final'] #Convenient shorthand
expTime = 120 #In seconds - an approx. estimate for the total exposure time
#of the image. Just for the hot pixel routine to estimate
#approx the right statistics...
#Write to fits files
for color in rgb.keys():
fileName = fitsBaseName + color + '.fits'
component = np.copy(rgb[color])
if maskHotPixels:
#Mask remaining hot pixels, whatever the heck reason they're there....
x = hotPixels.checkInterval(image=component*expTime, fwhm=4.5,
nSigma=3.0, maxIter=10, boxSize=5)
mask = x['mask']
print 'Niter: ',x['niter']
#assert 1==0
component[mask==1] = np.nan #Ignore cold pixels for now...
if interpolate:
component[component == 0] = np.nan
component = utils.replaceNaN(component, iterate=False)
if medianSmooth:
component = utils.median_filterNaN(component, size=3)
hdu = pyfits.PrimaryHDU(component)
hdu.writeto(os.path.join(workingDir, fileName))
roughShiftsY.append(dYs[i])
centroidsX.append(refPointX-dXs[i])
centroidsY.append(refPointY-dYs[i])
if i==0 and f==0:
#plotArray(processedIm,title='Dither Pos %i - dark'%i,origin='upper')#,vmin=0)
print np.shape(processedIm)
print np.shape(dark)
print sum(processedIm)
print np.where(processedIm==np.nan)
if (doHPM == 'one' and hpDict==None) or (doHPM=='all'):
if not os.path.exists(hpPklFile):
print "Creating time mask: %s"%(hpPklFile)
hpDict = hp.checkInterval(image=processedIm,fwhm=2.5, boxSize=5, nSigmaHot=3.0, dispToPickle=hpPklFile)
hpMask = hpDict['mask']
#open pkl file to grab masks in format that h5 file expects
pklDict = pickle.load(open(hpPklFile,"rb"))
hpm=pklDict["hotMask"]
cpm=pklDict["coldMask"]
dpm=pklDict["deadMask"]
#plotArray(hpMask,title='12 = hot, 19 = OK', origin='upper',vmin=0, vmax=13)
else:
print "Loading time mask: %s"%(hpPklFile)
pklDict = pickle.load(open(hpPklFile,"rb"))
hpMask = np.empty((numRows, numCols),dtype=int)
hpMask.fill(tm.timeMaskReason['none'])
def hotPixelsTest(testFileName=FileName(run='PAL2012',date='20121208',tstamp='20121209-044636').obs()):
'''
Runs some basic checks for consistency between intermediate output
masks and the final 'bad time lists'.
To run
- from Python:
hotPixelsTest('someObsFile.h5')
- from command line:
python hotPixelsTest.py someObsFile.h5
(No parameter file need be supplied).
'''
workingDir = '/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/hotPixTest2/'
outputFile = workingDir + 'testoutput.h5'
paramFile = os.path.join(os.path.dirname(__file__),'../../params/hotPixels.dict') #/Users/vaneyken/UCSB/ARCONS/pipeline/github/ARCONS-pipeline/params/hotPixels.dict'
testStartTime = 2 #In seconds
testEndTime = 4 #In seconds
timeStep = 2 #In seconds (deliberately equal to start time - end time)
fwhm = 3.0
boxSize = 5
nSigmaHot = 2.5
nSigmaCold = 2.0
hp.findHotPixels(paramFile=paramFile, inputFileName=testFileName,
outputFileName=outputFile, timeStep=timeStep,
startTime=testStartTime, endTime=testEndTime,
fwhm=fwhm, boxSize=boxSize, nSigmaHot=nSigmaHot,
nSigmaCold=nSigmaCold, display=True)
intermediateOutput = hp.checkInterval(inputFileName=testFileName, display=True,
firstSec=testStartTime,
intTime=testEndTime - testStartTime,
fwhm=fwhm, boxSize=boxSize,
nSigmaHot=nSigmaHot, nSigmaCold=nSigmaCold)
hpOutput = hp.readHotPixels(outputFile)
intMask = intermediateOutput['mask'] > 0 #Make a Boolean mask - any code > 0 is bad for some reason.
intervals = hpOutput['intervals']
reasons = hpOutput['reasons']
#Find the number of entries for each pixel in both the 'intervals' and the
#'reasons' arrays.
nIntervals = np.reshape([len(x) for x in intervals.flat], np.shape(intervals))
nReasons = np.reshape([len(x) for x in reasons.flat], np.shape(reasons))
#Union the interval lists for each pixel to give an array of single (multi-component) interval objects:
uIntervals = np.reshape(np.array([interval.union(x) for x in intervals.flat],
dtype='object'), np.shape(intervals))
#Create a boolean mask that should be True for all bad (hot/cold/dead/other) pixels within the test time range
finalMask = np.reshape([(interval(testStartTime, testEndTime) in x)
for x in uIntervals.flat], np.shape(uIntervals))
assert np.all(np.equal(intMask, finalMask))
assert np.all(np.equal(nIntervals, nReasons))
print
print "All seems okay. The two plots shown should look identical."
def makeArp147Image(obsFileList='inputFiles.list',
hotPixFileList='hotPixFiles.list',
offsetLog='raster_20121209-052620-060135-edited.log',
wvlCalFileName='calsol_20121209-060704.h5',
flatCalFileName='flatsol_20121210.h5',
workingDir='/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/arp147/Arp147WorkingDir',
showMasks=False,
outputFile='Arp147Image.pkl',
imageStack=None):
'''
Most parameters are hopefully somewhat self explanatory (though probably not. Tough).
If imageStack is supplied, it will skip reading in all the images and go straight to the
combining. imageStack is a dictionary of arrays as saved in the pickle file, under
keyname 'stack'.
'''
#matplotlib.use('gtkagg')
mpl.ion()
plateScale = 0.336281230787 #0.402 #in arcsec per pix.
imAngularWidth = 60 #Virtual image width in arcsec
imAngularHeight = 60 #Ditto height
theta = 0.257 #Angle between north and pixel column direction (rad)
#(assume constant...!)
slewTime = 2 #Block out this much time after each slew start (sec).
minExpFrac = 0.25 #Any pixel that has an effective integration time less than
#this fraction of the total integration time will be totally masked
#out instead of having its flux scaled up (to prevent too much noise
#being scaled along with it).
isFirstPlot = True #Flag to keep a record of whether the first progress plot has been made yet
wavebands = [{'color':'red', 'min':6000, 'max':8000}, #Define names and ranges (Ang)
{'color':'green', 'min':5000, 'max':6000}, #for the wavebands.
{'color':'blue', 'min':4000, 'max':5000}, #NOTE - REDDEST BAND SHOULD GO FIRST IF MASKS ARE TO BE MADE.
]
#wavebands = [{'color':'clear', 'min':4000, 'max':8000}] #Define names and ranges (Ang)
##Find the index of the reddest band in 'wavebands'
#reddestBand = np.argmax([(x['min']+x['max'])/2 for x in wavebands])
raOffset = np.array([]) #To take a list of RA offsets
decOffset = np.array([]) #Ditto for dec.
tSlewStart = np.array([]) #For list of slew start times for each offset
obsStart = np.array([]) #To take list of start times for each obs. file
obsEnd = np.array([]) #Ditto end times.
#Unit vectors in RA and dec directions on virtual grid (in pixels, assume const...)
raUnitVec = -np.array([np.cos(theta), np.sin(theta)]) / plateScale
decUnitVec = -np.array([-np.sin(theta), np.cos(theta)]) / plateScale
if imageStack is None:
#Read in file name list
f = open(os.path.join(workingDir, obsFileList), 'r')
try:
obsFileNames = np.array([os.path.join(workingDir, line.strip()) for line in f.readlines()])
finally:
f.close()
if hotPixFileList is None:
makeMasks = True #Used later on for deciding whether to make new masks or not
hotPixFileNames = np.repeat(None, len(ObsFileNames)) #Empty list if no hot pixel cal file list is provided
else:
makeMasks = False
#Read in the hot pixel time-mask filename list
f = open(os.path.join(workingDir, hotPixFileList), 'r')
try:
hotPixFileNames = np.array([os.path.join(workingDir, line.strip()) for line in f.readlines()])
finally:
f.close()
#Read in offset log
try:
f = open(os.path.join(workingDir, offsetLog), 'r')
for eachLine in f:
splitLine = eachLine.split()
if len(splitLine) >= 3: #If <3 entries in line, just skip it. (Takes care of empty lines).
raOffset = np.append(raOffset, float(splitLine[1]))
decOffset = np.append(decOffset, float(splitLine[2]))
tSlewStart = np.append(tSlewStart, dt.datetime.strptime(splitLine[0],
'%Y%m%d_%H%M%S'))
finally:
f.close()
#Start and end times for each offset location on the sky
tLocStart = tSlewStart + dt.timedelta(seconds=slewTime) #Start time of integration for each pointing
tLocEnd = np.append(tSlewStart[1:], np.NaN) #Last entry has no duration specified...
#Chop off the last pointing entry from consideration since there is no way of
#knowing its duration
tLocStart = tLocStart[0:-1]
tLocEnd = tLocEnd[0:-1]
raOffset = raOffset[0:-1]
decOffset = decOffset[0:-1]
nLoc = len(tLocStart) #Number of pointings (locations) on sky
#Get start and end times for each obs file
for eachFileName in obsFileNames:
obsFile = ObsFile.ObsFile(eachFileName)
tStart = dt.datetime.strptime(obsFile.getFromHeader('utc'),
'%a, %d %b %Y %H:%M:%S')
obsLength = dt.timedelta(seconds=float(obsFile.getFromHeader('exptime')))
tEnd = tStart + obsLength
obsStart = np.append(obsStart, tStart)
obsEnd = np.append(obsEnd, tEnd)
print eachFileName, tStart, tEnd
nDetRow = obsFile.nRow #Number of rows and columns on the detector
nDetCol = obsFile.nCol
print 'nDetRow, nDetCol:', nDetRow, nDetCol
#Virtual grid width and height in pixels
nVGridRow = np.ceil(imAngularHeight / plateScale)
nVGridCol = np.ceil(imAngularWidth / plateScale)
#Location of top left corner of a detector image if it were centered on the virtual grid.
xCornerCen = np.floor(nVGridCol / 2 - nDetCol / 2)
yCornerCen = np.floor(nVGridRow / 2 - nDetRow / 2)
#Initialise 'images' dictionary to look like:
#{'red':A, 'green':B, 'blue':C} (or whatever)
#Where A, B, C (,...) are 3D arrays (npointings x representing an image for each pointing.
images = {}
for eachWaveband in wavebands:
images[eachWaveband['color']] = np.zeros((nLoc, nVGridRow, nVGridCol))
images[eachWaveband['color']].fill(np.NaN)
#Loop through each raster scan location except the last (since the last has no specified duration)
print 'Looping through pointings'
for (iLoc, eachLocStart, eachLocEnd, eachRaOffset, eachDecOffset) in \
zip(range(len(tLocStart)), tLocStart, tLocEnd, raOffset, decOffset):
print 'Offset (RA, dec, in arcsec): ', eachRaOffset, eachDecOffset
print 'Time: ', eachLocStart, ' to ', eachLocEnd
#Find which obs files have a start or end time within the time-span of the current location
atThisLoc = ~((obsStart >= eachLocEnd) | (obsEnd <= eachLocStart))
obsNamesAtLoc = obsFileNames[atThisLoc] #The file names for obs files which overlap this pointing time
hotPixNamesAtLoc = hotPixFileNames[atThisLoc] #Corresponding hot pixel cal file names.
obsStartAtLoc = obsStart[atThisLoc] #The corresponding start times for those files
obsEndAtLoc = obsEnd[atThisLoc]
#Make an image in each waveband for this location
for eachWaveband in wavebands:
print eachWaveband['color']
#Stack all the images from all obs files who's times overlap this pointing
im = np.zeros((nDetRow, nDetCol)) #Initialise an empty image
effIntTimes = np.zeros((nDetRow, nDetCol)) #Empty array to take total effective integration times for each pixel in this pointing
totIntTime = 0 #To take summed integration time (sec) for this pointing.
for eachFileName, eachObsStart, eachObsEnd, eachHotPixFileName in zip(obsNamesAtLoc, obsStartAtLoc, obsEndAtLoc, hotPixNamesAtLoc):
print eachFileName, eachHotPixFileName
obsFile = ObsFile.ObsFile(eachFileName)
obsFile.loadFlatCalFile(os.path.join(workingDir, flatCalFileName))
obsFile.loadWvlCalFile(os.path.join(workingDir, wvlCalFileName))
obsFile.setWvlCutoffs(eachWaveband['min'], eachWaveband['max'])
if makeMasks is False:
print 'Loading hot pix cal file...'
obsFile.loadHotPixCalFile(eachHotPixFileName)
print 'Done'
#Find integration start/end times in seconds relative to start of obs. file
#Note that negative datetime objects give unexpected results - so don't go there!
if eachLocStart < eachObsStart:
firstSec = 0
else:
firstSec = (eachLocStart - eachObsStart).seconds #No. of seconds from start time of obs file.
if eachLocEnd > eachObsEnd:
lastSec = (eachObsEnd - eachObsStart).seconds #Will integrate to the end of the obs. file.
else:
lastSec = (eachLocEnd - eachObsStart).seconds
integrationTime = lastSec - firstSec
print 'Getting image...'
x = obsFile.getPixelCountImage(firstSec=firstSec, integrationTime=integrationTime,
weighted=True)
thisIm,thisEffIntTimes = x['image'],x['effIntTimes']
print 'Done...'
im += thisIm
effIntTimes += thisEffIntTimes
totIntTime += integrationTime #In seconds
#Clean up
del(obsFile)
#Now have a stacked image for this pointing.
#Scale each pixel by effective integration time
im *= totIntTime/effIntTimes
#Completely mask out pixels which have too little integration time (otherwise they will be really noisy)
im[effIntTimes/totIntTime < minExpFrac] = np.NaN
#Scale the image for the integration time (make all in counts per second):
print 'Total integration time: '+str(totIntTime)
im /= totIntTime
#Do bad pixel masking if not already done
if makeMasks is True:
if eachWaveband['color'] == wavebands[0]['color']: #If we're on the reddest band
hpMask = hotPixels.checkInterval(image=im, fwhm=4.0)['mask'] #make a pixel mask.
hpMask = (hpMask > 0) #Turn into a boolean array.
deadMask = (im == 0) #Dead pixels
combinedMask = hpMask | deadMask #Always uses the hot pixel mask from the reddest (first) band.
if showMasks:
imCopy = np.array(im) #Make a copy, because otherwise changes to im apparently can show up in matshow even AFTER the matshow call!!
mpl.matshow(imCopy, vmax=np.percentile(imCopy[~np.isnan(imCopy)], 99.5))
mpl.colorbar()
x = np.arange(np.shape(im)[1])
y = np.arange(np.shape(im)[0])
xx, yy = np.meshgrid(x, y)
mpl.scatter(xx[hpMask], yy[hpMask], c='y')
mpl.title('#' + str(iLoc) + ', ' + str(eachLocStart) + ' to ' + str(eachLocEnd)
+ 's, ' + str(eachWaveband['color']))
im[combinedMask] = np.nan #Set any bad pixels to NaN.
#Determine where image should be located on the virtual grid
gridOffset = eachRaOffset * raUnitVec + eachDecOffset * decUnitVec
xCorner = np.round(xCornerCen + gridOffset[0])
yCorner = np.round(yCornerCen + gridOffset[1])
print 'X,Y corners:', xCorner, yCorner
#And put it in the image stack, in the right location. All values in
#virtual grid outside the current image will remain NaN for this pointing.
images[eachWaveband['color']][iLoc, yCorner:yCorner + nDetRow, xCorner:xCorner + nDetCol] = im
#Display the current image for information
imToShow = np.copy(im)
imToShow[np.isnan(imToShow)] = 0
if isFirstPlot:
mpl.matshow(imToShow,vmax=np.percentile(imToShow,99.5)) #Make in a new plot window
isFirstPlot=False
else:
mpl.matshow(imToShow,vmax=np.percentile(imToShow,99.5), fignum=False) #Don't open a new plot window if it's not the first time.
#mpl.colorbar()
mpl.title(eachWaveband['color'])
mpl.draw()
else:
images = imageStack
#Now have a stack of images in 'images'
finalImage = {} #Will end up as dictionary with one image for each color.
print 'Combining images'
for eachWaveband in wavebands:
print eachWaveband['color']
#Make numpy masked array of the stack where NaNs and zeros are all masked
im = images[eachWaveband['color']]
im[im<=0] = np.nan
masked = np.ma.masked_invalid(images[eachWaveband['color']])
#Median combine the images (leaving out masked pixels)
finalImage[eachWaveband['color']] = np.ma.median(masked, axis=0).data
#assert 1==0
print 'Done....'
#assert 1==0
result = {'final':finalImage, 'stack':images}
if type(outputFile) is str and len(outputFile) > 0:
try:
f = open(os.path.join(workingDir, outputFile), 'wb')
pickle.dump(result, f)
except OSError:
print 'Could not write results to output file.'
finally:
f.close()
return result