def showArrayStdVsIntTime(self): intTimes = [1,2,3,5,10,15,30] sdevVsIntTime = [] madVsIntTime = [] medVsIntTime = [] for intTime in intTimes: image = np.zeros((self.nRow,self.nCol)) for iOb,ob in enumerate(self.skyObList): x = ob.getPixelCountImage(firstSec=0,integrationTime=intTime) image+=x['image'] hotPixMask = hotPixels.checkInterval(image=image)['mask'] image[hotPixMask!=0]=0 countList = image[image!=0] sdevVsIntTime.append(np.std(countList)) madVsIntTime.append(np.median(np.abs(countList-np.median(countList)))) medVsIntTime.append(np.median(countList)) PopUp(parent=self).plotArray(image,title=r'%d std=%f mad=%f med=%f'%(intTime,sdevVsIntTime[-1],madVsIntTime[-1],medVsIntTime[-1])) medVsIntTime = np.array(medVsIntTime) sqrtNVsIntTime = np.sqrt(medVsIntTime) pop = PopUp(parent=self,title='showArrayStdVsIntTime') pop.axes.set_xlabel('integration time (s)') pop.axes.set_ylabel('$\sigma$') pop.axes.plot(intTimes,sqrtNVsIntTime,'k--',label=r'$\sqrt(med(N))$') pop.axes.plot(intTimes,sdevVsIntTime,'k') pop.axes.plot(intTimes,madVsIntTime,'r') pop.draw()
def createStack(self,stackLabel): paramsLabel = stackLabel weighted = True if stackLabel == 'raw': paramsLabel = 'obs' weighted = False getRawCounts = True x = self.stackObsFileLists[paramsLabel][0].getSpectralCube(weighted=weighted) spectra = x['cube'] wvlBinEdges = x['wvlBinEdges'] totalIntTime = 0 for ob in self.stackObsFileLists[paramsLabel][1:]: print ob.fileName x = ob.getSpectralCube(weighted=weighted) cube = x['cube'] wvlBinEdges = x['wvlBinEdges'] spectra += cube totalIntTime += ob.getFromHeader('exptime') spectra = np.array(spectra,dtype=np.float64) frame = np.sum(spectra,axis=2) hotPixMask = hotPixels.checkInterval(image=frame)['mask'] frame[hotPixMask != 0] = np.nan frame[frame == 0] = np.nan stackFileName = self.params[stackLabel+'StackFileName'] np.savez(stackFileName,spectra=spectra,frame=frame,wvlBinEdges=wvlBinEdges,totalIntTime=totalIntTime)
def showTwilightArrayReducedChisqImage(self): chisqImage = np.zeros((self.nRow,self.nCol)) nDeltaFromZero = np.zeros((self.nRow,self.nCol,self.nWvlBins)) for iRow in range(self.nRow): for iCol in range(self.nCol): x = self.getChisq(iRow,iCol) chisqImage[iRow,iCol] = x['reducedChisq'] nDeltaFromZero[iRow,iCol,:] = x['nDeltaFromZero'] chisqImage[np.isnan(chisqImage)]=0 chisqImage[chisqImage == np.inf]=0 nDeltaFromZero = np.ma.array(nDeltaFromZero,mask=np.logical_and(np.isnan(nDeltaFromZero),nDeltaFromZero==np.inf)) hotPixMask = hotPixels.checkInterval(image=chisqImage)['mask'] chisqImage[hotPixMask != 0] = 0 #self.popUpArray(image=chisqImage,title='Flat Cal $\chi^{2}_{red}$',normNSigma=1.) PopUp(parent=self,title='showTwilightArrayReducedChisqImage').plotArray(image=chisqImage,title='Flat Cal $\chi^{2}_{red}$',normNSigma=1.)
def makeRGBimage(result, fitsBaseName='Arp147-', workingDir='/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/arp147/Arp147WorkingDir', interpolate=False, medianSmooth=False, maskHotPixels=False): ''' Take a 'result' (as returned by makeArp147Image()) and make an RGB image out of it. Also create fits files of the three components. INPUTS: result: as returned by makeArp147Image() fitsBaseName: fits files are output as fistBaseName+[color]+'.fits' ''' rgb = result['final'] #Convenient shorthand expTime = 120 #In seconds - an approx. estimate for the total exposure time #of the image. Just for the hot pixel routine to estimate #approx the right statistics... #Write to fits files for color in rgb.keys(): fileName = fitsBaseName + color + '.fits' component = np.copy(rgb[color]) if maskHotPixels: #Mask remaining hot pixels, whatever the heck reason they're there.... x = hotPixels.checkInterval(image=component*expTime, fwhm=4.5, nSigma=3.0, maxIter=10, boxSize=5) mask = x['mask'] print 'Niter: ',x['niter'] #assert 1==0 component[mask==1] = np.nan #Ignore cold pixels for now... if interpolate: component[component == 0] = np.nan component = utils.replaceNaN(component, iterate=False) if medianSmooth: component = utils.median_filterNaN(component, size=3) hdu = pyfits.PrimaryHDU(component) hdu.writeto(os.path.join(workingDir, fileName))
roughShiftsY.append(dYs[i]) centroidsX.append(refPointX-dXs[i]) centroidsY.append(refPointY-dYs[i]) if i==0 and f==0: #plotArray(processedIm,title='Dither Pos %i - dark'%i,origin='upper')#,vmin=0) print np.shape(processedIm) print np.shape(dark) print sum(processedIm) print np.where(processedIm==np.nan) if (doHPM == 'one' and hpDict==None) or (doHPM=='all'): if not os.path.exists(hpPklFile): print "Creating time mask: %s"%(hpPklFile) hpDict = hp.checkInterval(image=processedIm,fwhm=2.5, boxSize=5, nSigmaHot=3.0, dispToPickle=hpPklFile) hpMask = hpDict['mask'] #open pkl file to grab masks in format that h5 file expects pklDict = pickle.load(open(hpPklFile,"rb")) hpm=pklDict["hotMask"] cpm=pklDict["coldMask"] dpm=pklDict["deadMask"] #plotArray(hpMask,title='12 = hot, 19 = OK', origin='upper',vmin=0, vmax=13) else: print "Loading time mask: %s"%(hpPklFile) pklDict = pickle.load(open(hpPklFile,"rb")) hpMask = np.empty((numRows, numCols),dtype=int) hpMask.fill(tm.timeMaskReason['none'])
def hotPixelsTest(testFileName=FileName(run='PAL2012',date='20121208',tstamp='20121209-044636').obs()): ''' Runs some basic checks for consistency between intermediate output masks and the final 'bad time lists'. To run - from Python: hotPixelsTest('someObsFile.h5') - from command line: python hotPixelsTest.py someObsFile.h5 (No parameter file need be supplied). ''' workingDir = '/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/hotPixTest2/' outputFile = workingDir + 'testoutput.h5' paramFile = os.path.join(os.path.dirname(__file__),'../../params/hotPixels.dict') #/Users/vaneyken/UCSB/ARCONS/pipeline/github/ARCONS-pipeline/params/hotPixels.dict' testStartTime = 2 #In seconds testEndTime = 4 #In seconds timeStep = 2 #In seconds (deliberately equal to start time - end time) fwhm = 3.0 boxSize = 5 nSigmaHot = 2.5 nSigmaCold = 2.0 hp.findHotPixels(paramFile=paramFile, inputFileName=testFileName, outputFileName=outputFile, timeStep=timeStep, startTime=testStartTime, endTime=testEndTime, fwhm=fwhm, boxSize=boxSize, nSigmaHot=nSigmaHot, nSigmaCold=nSigmaCold, display=True) intermediateOutput = hp.checkInterval(inputFileName=testFileName, display=True, firstSec=testStartTime, intTime=testEndTime - testStartTime, fwhm=fwhm, boxSize=boxSize, nSigmaHot=nSigmaHot, nSigmaCold=nSigmaCold) hpOutput = hp.readHotPixels(outputFile) intMask = intermediateOutput['mask'] > 0 #Make a Boolean mask - any code > 0 is bad for some reason. intervals = hpOutput['intervals'] reasons = hpOutput['reasons'] #Find the number of entries for each pixel in both the 'intervals' and the #'reasons' arrays. nIntervals = np.reshape([len(x) for x in intervals.flat], np.shape(intervals)) nReasons = np.reshape([len(x) for x in reasons.flat], np.shape(reasons)) #Union the interval lists for each pixel to give an array of single (multi-component) interval objects: uIntervals = np.reshape(np.array([interval.union(x) for x in intervals.flat], dtype='object'), np.shape(intervals)) #Create a boolean mask that should be True for all bad (hot/cold/dead/other) pixels within the test time range finalMask = np.reshape([(interval(testStartTime, testEndTime) in x) for x in uIntervals.flat], np.shape(uIntervals)) assert np.all(np.equal(intMask, finalMask)) assert np.all(np.equal(nIntervals, nReasons)) print print "All seems okay. The two plots shown should look identical."
def makeArp147Image(obsFileList='inputFiles.list', hotPixFileList='hotPixFiles.list', offsetLog='raster_20121209-052620-060135-edited.log', wvlCalFileName='calsol_20121209-060704.h5', flatCalFileName='flatsol_20121210.h5', workingDir='/Users/vaneyken/Data/UCSB/ARCONS/Palomar2012/arp147/Arp147WorkingDir', showMasks=False, outputFile='Arp147Image.pkl', imageStack=None): ''' Most parameters are hopefully somewhat self explanatory (though probably not. Tough). If imageStack is supplied, it will skip reading in all the images and go straight to the combining. imageStack is a dictionary of arrays as saved in the pickle file, under keyname 'stack'. ''' #matplotlib.use('gtkagg') mpl.ion() plateScale = 0.336281230787 #0.402 #in arcsec per pix. imAngularWidth = 60 #Virtual image width in arcsec imAngularHeight = 60 #Ditto height theta = 0.257 #Angle between north and pixel column direction (rad) #(assume constant...!) slewTime = 2 #Block out this much time after each slew start (sec). minExpFrac = 0.25 #Any pixel that has an effective integration time less than #this fraction of the total integration time will be totally masked #out instead of having its flux scaled up (to prevent too much noise #being scaled along with it). isFirstPlot = True #Flag to keep a record of whether the first progress plot has been made yet wavebands = [{'color':'red', 'min':6000, 'max':8000}, #Define names and ranges (Ang) {'color':'green', 'min':5000, 'max':6000}, #for the wavebands. {'color':'blue', 'min':4000, 'max':5000}, #NOTE - REDDEST BAND SHOULD GO FIRST IF MASKS ARE TO BE MADE. ] #wavebands = [{'color':'clear', 'min':4000, 'max':8000}] #Define names and ranges (Ang) ##Find the index of the reddest band in 'wavebands' #reddestBand = np.argmax([(x['min']+x['max'])/2 for x in wavebands]) raOffset = np.array([]) #To take a list of RA offsets decOffset = np.array([]) #Ditto for dec. tSlewStart = np.array([]) #For list of slew start times for each offset obsStart = np.array([]) #To take list of start times for each obs. file obsEnd = np.array([]) #Ditto end times. #Unit vectors in RA and dec directions on virtual grid (in pixels, assume const...) raUnitVec = -np.array([np.cos(theta), np.sin(theta)]) / plateScale decUnitVec = -np.array([-np.sin(theta), np.cos(theta)]) / plateScale if imageStack is None: #Read in file name list f = open(os.path.join(workingDir, obsFileList), 'r') try: obsFileNames = np.array([os.path.join(workingDir, line.strip()) for line in f.readlines()]) finally: f.close() if hotPixFileList is None: makeMasks = True #Used later on for deciding whether to make new masks or not hotPixFileNames = np.repeat(None, len(ObsFileNames)) #Empty list if no hot pixel cal file list is provided else: makeMasks = False #Read in the hot pixel time-mask filename list f = open(os.path.join(workingDir, hotPixFileList), 'r') try: hotPixFileNames = np.array([os.path.join(workingDir, line.strip()) for line in f.readlines()]) finally: f.close() #Read in offset log try: f = open(os.path.join(workingDir, offsetLog), 'r') for eachLine in f: splitLine = eachLine.split() if len(splitLine) >= 3: #If <3 entries in line, just skip it. (Takes care of empty lines). raOffset = np.append(raOffset, float(splitLine[1])) decOffset = np.append(decOffset, float(splitLine[2])) tSlewStart = np.append(tSlewStart, dt.datetime.strptime(splitLine[0], '%Y%m%d_%H%M%S')) finally: f.close() #Start and end times for each offset location on the sky tLocStart = tSlewStart + dt.timedelta(seconds=slewTime) #Start time of integration for each pointing tLocEnd = np.append(tSlewStart[1:], np.NaN) #Last entry has no duration specified... #Chop off the last pointing entry from consideration since there is no way of #knowing its duration tLocStart = tLocStart[0:-1] tLocEnd = tLocEnd[0:-1] raOffset = raOffset[0:-1] decOffset = decOffset[0:-1] nLoc = len(tLocStart) #Number of pointings (locations) on sky #Get start and end times for each obs file for eachFileName in obsFileNames: obsFile = ObsFile.ObsFile(eachFileName) tStart = dt.datetime.strptime(obsFile.getFromHeader('utc'), '%a, %d %b %Y %H:%M:%S') obsLength = dt.timedelta(seconds=float(obsFile.getFromHeader('exptime'))) tEnd = tStart + obsLength obsStart = np.append(obsStart, tStart) obsEnd = np.append(obsEnd, tEnd) print eachFileName, tStart, tEnd nDetRow = obsFile.nRow #Number of rows and columns on the detector nDetCol = obsFile.nCol print 'nDetRow, nDetCol:', nDetRow, nDetCol #Virtual grid width and height in pixels nVGridRow = np.ceil(imAngularHeight / plateScale) nVGridCol = np.ceil(imAngularWidth / plateScale) #Location of top left corner of a detector image if it were centered on the virtual grid. xCornerCen = np.floor(nVGridCol / 2 - nDetCol / 2) yCornerCen = np.floor(nVGridRow / 2 - nDetRow / 2) #Initialise 'images' dictionary to look like: #{'red':A, 'green':B, 'blue':C} (or whatever) #Where A, B, C (,...) are 3D arrays (npointings x representing an image for each pointing. images = {} for eachWaveband in wavebands: images[eachWaveband['color']] = np.zeros((nLoc, nVGridRow, nVGridCol)) images[eachWaveband['color']].fill(np.NaN) #Loop through each raster scan location except the last (since the last has no specified duration) print 'Looping through pointings' for (iLoc, eachLocStart, eachLocEnd, eachRaOffset, eachDecOffset) in \ zip(range(len(tLocStart)), tLocStart, tLocEnd, raOffset, decOffset): print 'Offset (RA, dec, in arcsec): ', eachRaOffset, eachDecOffset print 'Time: ', eachLocStart, ' to ', eachLocEnd #Find which obs files have a start or end time within the time-span of the current location atThisLoc = ~((obsStart >= eachLocEnd) | (obsEnd <= eachLocStart)) obsNamesAtLoc = obsFileNames[atThisLoc] #The file names for obs files which overlap this pointing time hotPixNamesAtLoc = hotPixFileNames[atThisLoc] #Corresponding hot pixel cal file names. obsStartAtLoc = obsStart[atThisLoc] #The corresponding start times for those files obsEndAtLoc = obsEnd[atThisLoc] #Make an image in each waveband for this location for eachWaveband in wavebands: print eachWaveband['color'] #Stack all the images from all obs files who's times overlap this pointing im = np.zeros((nDetRow, nDetCol)) #Initialise an empty image effIntTimes = np.zeros((nDetRow, nDetCol)) #Empty array to take total effective integration times for each pixel in this pointing totIntTime = 0 #To take summed integration time (sec) for this pointing. for eachFileName, eachObsStart, eachObsEnd, eachHotPixFileName in zip(obsNamesAtLoc, obsStartAtLoc, obsEndAtLoc, hotPixNamesAtLoc): print eachFileName, eachHotPixFileName obsFile = ObsFile.ObsFile(eachFileName) obsFile.loadFlatCalFile(os.path.join(workingDir, flatCalFileName)) obsFile.loadWvlCalFile(os.path.join(workingDir, wvlCalFileName)) obsFile.setWvlCutoffs(eachWaveband['min'], eachWaveband['max']) if makeMasks is False: print 'Loading hot pix cal file...' obsFile.loadHotPixCalFile(eachHotPixFileName) print 'Done' #Find integration start/end times in seconds relative to start of obs. file #Note that negative datetime objects give unexpected results - so don't go there! if eachLocStart < eachObsStart: firstSec = 0 else: firstSec = (eachLocStart - eachObsStart).seconds #No. of seconds from start time of obs file. if eachLocEnd > eachObsEnd: lastSec = (eachObsEnd - eachObsStart).seconds #Will integrate to the end of the obs. file. else: lastSec = (eachLocEnd - eachObsStart).seconds integrationTime = lastSec - firstSec print 'Getting image...' x = obsFile.getPixelCountImage(firstSec=firstSec, integrationTime=integrationTime, weighted=True) thisIm,thisEffIntTimes = x['image'],x['effIntTimes'] print 'Done...' im += thisIm effIntTimes += thisEffIntTimes totIntTime += integrationTime #In seconds #Clean up del(obsFile) #Now have a stacked image for this pointing. #Scale each pixel by effective integration time im *= totIntTime/effIntTimes #Completely mask out pixels which have too little integration time (otherwise they will be really noisy) im[effIntTimes/totIntTime < minExpFrac] = np.NaN #Scale the image for the integration time (make all in counts per second): print 'Total integration time: '+str(totIntTime) im /= totIntTime #Do bad pixel masking if not already done if makeMasks is True: if eachWaveband['color'] == wavebands[0]['color']: #If we're on the reddest band hpMask = hotPixels.checkInterval(image=im, fwhm=4.0)['mask'] #make a pixel mask. hpMask = (hpMask > 0) #Turn into a boolean array. deadMask = (im == 0) #Dead pixels combinedMask = hpMask | deadMask #Always uses the hot pixel mask from the reddest (first) band. if showMasks: imCopy = np.array(im) #Make a copy, because otherwise changes to im apparently can show up in matshow even AFTER the matshow call!! mpl.matshow(imCopy, vmax=np.percentile(imCopy[~np.isnan(imCopy)], 99.5)) mpl.colorbar() x = np.arange(np.shape(im)[1]) y = np.arange(np.shape(im)[0]) xx, yy = np.meshgrid(x, y) mpl.scatter(xx[hpMask], yy[hpMask], c='y') mpl.title('#' + str(iLoc) + ', ' + str(eachLocStart) + ' to ' + str(eachLocEnd) + 's, ' + str(eachWaveband['color'])) im[combinedMask] = np.nan #Set any bad pixels to NaN. #Determine where image should be located on the virtual grid gridOffset = eachRaOffset * raUnitVec + eachDecOffset * decUnitVec xCorner = np.round(xCornerCen + gridOffset[0]) yCorner = np.round(yCornerCen + gridOffset[1]) print 'X,Y corners:', xCorner, yCorner #And put it in the image stack, in the right location. All values in #virtual grid outside the current image will remain NaN for this pointing. images[eachWaveband['color']][iLoc, yCorner:yCorner + nDetRow, xCorner:xCorner + nDetCol] = im #Display the current image for information imToShow = np.copy(im) imToShow[np.isnan(imToShow)] = 0 if isFirstPlot: mpl.matshow(imToShow,vmax=np.percentile(imToShow,99.5)) #Make in a new plot window isFirstPlot=False else: mpl.matshow(imToShow,vmax=np.percentile(imToShow,99.5), fignum=False) #Don't open a new plot window if it's not the first time. #mpl.colorbar() mpl.title(eachWaveband['color']) mpl.draw() else: images = imageStack #Now have a stack of images in 'images' finalImage = {} #Will end up as dictionary with one image for each color. print 'Combining images' for eachWaveband in wavebands: print eachWaveband['color'] #Make numpy masked array of the stack where NaNs and zeros are all masked im = images[eachWaveband['color']] im[im<=0] = np.nan masked = np.ma.masked_invalid(images[eachWaveband['color']]) #Median combine the images (leaving out masked pixels) finalImage[eachWaveband['color']] = np.ma.median(masked, axis=0).data #assert 1==0 print 'Done....' #assert 1==0 result = {'final':finalImage, 'stack':images} if type(outputFile) is str and len(outputFile) > 0: try: f = open(os.path.join(workingDir, outputFile), 'wb') pickle.dump(result, f) except OSError: print 'Could not write results to output file.' finally: f.close() return result