def makeDark(stackForDark=None, outPutFileName=None, verbose=True): """ takes a list or array of arrays and makes a dark. Return an array with the dark. If outPutFileName is provided, it saves the dark in an npz """ try: dark = irUtils.medianStack(stackForDark) print("Making dark hot pixel mask") if verbose: plotArray(dark, title='Dark', origin='upper') except: print( "#### Error with the stack of dark. makeDark takes a list of array of arrays to make the dark ####" ) sys.exit(0) try: np.savez(darkPath, dark=dark) except: if outPutFileName == None: print("No output file name provided. Dark not saved") else: print("Wrong output path. Dark not saved") return dark
def makeMask(run=None, date=None, basePath=None, startTimeStamp=None, stopTimeStamp=None, verbose=False, sigma=3, maxCut=2450, coldCut=False, manualArray=None): """ MaxCut sets level for initial hot pixel cut. Everything with cps>maxCut -> np.nan Sigma determines level for second cut. Everything with cps>mean+sigma*stdev -> np.nan If coldCut=True, third cut where everything with cps<mean-sigma*stdev -> np.nan manualArray is a way to give a list of bad pixels manually, in format [[row,col],[row,col],...] """ try: dataPath = basePath + str(run) + os.path.sep + str(date) + os.path.sep stack = loadIMGStack(dataPath, startTimeStamp, stopTimeStamp, nCols=nCols, nRows=nRows) except: print("Could not find dark data in ScienceData path, checking ramdisk") dataPath = '/mnt/ramdisk/' stack = loadIMGStack(dataPath, startTimeStamp, stopTimeStamp, nCols=nCols, nRows=nRows) medStack = irUtils.medianStack(stack) ##### if verbose: try: plotArray(medStack, title='Median Dark Stack') except: plt.matshow(medStack) plt.show() #initial masking, take out anything with cps > maxCut mask = np.zeros(np.shape(medStack), dtype=int) mask[np.where(medStack >= maxCut)] = 1 if verbose: try: plotArray(mask, title='cps>%i == 1' % maxCut) except: plt.matshow(mask) plt.show() medStack[np.where(mask == 1)] = np.nan medStack[np.where(medStack == 0)] = np.nan if verbose: try: plotArray(medStack, title='Median Stack with mask 1') except: plt.matshow(medStack) plt.show() #second round of masking, where cps > mean+sigma*std mask2 = np.zeros(np.shape(medStack), dtype=int) mask2[np.where( medStack >= np.nanmean(medStack) + sigma * np.nanstd(medStack))] = 1 #if coldCut is true, also mask cps < mean-sigma*std if coldCut == True: mask2[np.where(medStack <= np.nanmean(medStack) - sigma * np.nanstd(medStack))] = 1 if verbose: try: plotArray(mask2, title='cps>mean+%i-sigma == 1' % sigma) except: plt.matshow(mask2) plt.show() medStack[np.where(mask2 == 1)] = np.nan if verbose: try: plotArray(medStack, title='Median Stack with mask 2') except: plt.matshow(medStack) plt.show() finalMask = mask + mask2 # provide an easy means to pipe in an array of manually identified pixels if manualArray is not None: for pix in manualArray: finalMask[pix[0], pix[1]] = 1 if verbose: try: plotArray(finalMask, title='Final mask') except: plt.matshow(finalMask) plt.show() print("Number masked pixels = ", len(np.array(np.where(finalMask == 1)).flatten())) return finalMask
def makeFlat(flatStack=None, dark=None, outPutFileName=None, badPixelMask=None, crop=False, cropColsForMedian=[(0, 19), (60, 79)], cropRowsForMedian=[(25, 49)], verbose=True): """ Input: flatStack: array or list of arrays to make the flat dark: dark. If none the flat is not dark subtracteed outPutFileName: if not provided returns the flat array without saving it crop: if False uses the entire array to calculate the median, otherwise it crops the columns in the range specified by cropForMedian cropColsForMedian, cropRowsForMedian: are tuples with each element being the range of columns/rows to be cropped when taking the median for the flat 10/1/2017: currently crops the high frequency boards (bad performances) and FL2 (not working on Faceless) Returns: dictionary with 'weights' and 'flat' """ try: flat = irUtils.medianStack(flatStack) print("Loading flat frame") except: print("No valid flat stack provided. Exiting") sys.exit(0) if verbose: plotArray(flat, title='NotdarkSubFlat') if darkSpan[0] != '0': #dark subtract the flat print("Subtracting dark") flatSub = flat - dark else: print("Dark=None, will make a flat without dark subtraction") flatSub = flat flatSub[np.where(flatSub < 0.0)] = np.nan croppedFrame = np.copy(flatSub) if crop: if cropColsForMedian != None: cropColsForMedian = np.array(cropColsForMedian) for iCrop, cropInd in enumerate(cropColsForMedian): croppedFrame[:, cropInd[0]:cropInd[1] + 1] = np.nan if cropRowsForMedian != None: cropRowsForMedian = np.array(cropRowsForMedian) for iCrop, cropInd in enumerate(cropRowsForMedian): croppedFrame[cropInd[0]:cropInd[1] + 1, :] = np.nan #Apply bad pixel mask if badPixelMask != None: print("Applying bad pixel mask") croppedFrame[badPixelMask != 0] = np.nan if verbose: plotArray(croppedFrame, title='Cropped flat frame') #Plot the cropped flat frame med = np.nanmedian(croppedFrame.flatten()) print('median', med) flatSub[flatSub == 0] = 1 weights = med / flatSub #calculate the weights by dividing the median by the dark-subtracted flat frame if verbose: plotArray(weights, title='Weights') #Plot the weights try: np.savez(outPutFileName, weights=weights, flat=flatSub) except: if outPutFileName == None: print('No output file name provided. Not saving flat') else: print('Output file name not valid. Not saving flat') dict = {} dict['flat'] = flat dict['weights'] = weights return dict
dataDir = binPath print("Loading data from .bin files") print("This is the span of Dark Frames") print(darkSpan) print("This is the span of Flat Frames") print(flatSpan) #load dark frames if darkSpan[0]!='0': print("Loading dark frame") if useImg == True: darkStack = loadIMGStack(dataDir, darkSpan[0], darkSpan[1], nCols=numCols, nRows=numRows) else: darkStack = loadBINStack(dataDir, darkSpan[0], darkSpan[1], nCols=numCols, nRows=numRows) dark = irUtils.medianStack(darkStack) else: print("No dark provided") dark = np.zeros((numRows, numCols),dtype=int) #if dark frames are provided, generate a hot pixel mask using SR Meeker's darkHotPixMask.py if darkSpan[0]!='0': darkHPM = dhpm.makeMask(run=run, date=date, basePath=dataDir,startTimeStamp=darkSpan[0], stopTimeStamp=darkSpan[1], coldCut=False, manualArray=None) else: print("Failed to generate dark mask. Turning off hot pixel masking") #Apply the hot pixel mask and plot the hot pixel masked dark frame dark[np.where(darkHPM==1)]=np.nan plotArray(dark,title='Dark',origin='upper')
print("Loading dark frame to make Dark and Saving It") if useImg == True: darkStackImg = loadIMGStack(dataDir, darkSpanImg[0], darkSpanImg[1], nCols=numCols, nRows=numRows, verbose=False) else: darkStackImg = loadBINStack(dataDir, darkSpanImg[0], darkSpanImg[1], nCols=numCols, nRows=numRows, verbose=False) darkImg = irUtils.medianStack(darkStackImg) plotArray(darkImg, title='DarkImg', origin='upper') darkImg[np.where(darkHPMImg == 1)] = np.nan plotArray(darkImg, title='DarkImg HP Masked', origin='upper') darkFN = 'darkImg_' + target + '.npz' #Save the dark into the output directory darkPath = os.path.join(outputDir, darkFN) np.savez(darkPath, darkImg=darkImg) if useImg == True: darkStackFlat = loadIMGStack(dataDir, darkSpanFlat[0], darkSpanFlat[1], nCols=numCols, nRows=numRows, verbose=False) else:
def stackCube(h5File,npzFile, verbose=True): #load npz file with cubes and timestamps npzDict = loadCubeStack(npzFile) cubeTimes = npzDict['times'] cubeCubes = npzDict['cubes'] cubeWBEs = npzDict['wvlBinEdges'] #define bin center wavelengths (in nm) wbWidths = np.diff(cubeWBEs) centers = cubeWBEs[:-1]+wbWidths/2.0 nWvlBins = len(centers) if verbose: print("Loaded npz file...") #load h5 file with centroids, image resampling info, dark frames, and timestamps h5Dict = loadH5Stack(h5File) h5Times = h5Dict['times'] centXs = h5Dict['centX'] centYs = h5Dict['centY'] hotPix = h5Dict['hpm'] #get image stacking info from params dictionary paramsDict = h5Dict['params'] padFraction = np.float(paramsDict['padFraction'][0]) upSample = paramsDict['upSample'][0] doHPM = bool(paramsDict['doHPM'][0]) coldCut = paramsDict['coldCut'][0] if verbose: print("Loaded h5 file...") if cubeTimes.all() == h5Times.all(): print("Timestamps match. Carrying on with stacking...") else: print("Timestamp mismatch between two files!!") print(cubeTimes) print(h5Times) sys.exit(0) if doHPM: hpMask = hotPix[0] cubeStack = [] for i in range(nWvlBins): cubeStack.append([]) finalCube = [] finalTimes = [] for t in np.arange(len(cubeTimes)): time = cubeTimes[t] cube = cubeCubes[t] centX = centXs[t] centY = centYs[t] finalTimes.append(cubeTimes[t]) for w in np.arange(nWvlBins): im = np.array(cube[:,:,w],float) im = np.transpose(im) #apply hp mask to image if doHPM: im[np.where(hpMask==1)]=np.nan #cut out cold/dead pixels im[np.where(im<=coldCut)]=np.nan #pad frame with margin for shifting and stacking paddedFrame = irUtils.embedInLargerArray(im,frameSize=padFraction) #upSample frame for sub-pixel registration with fitting code upSampledFrame = irUtils.upSampleIm(paddedFrame,upSample) #conserve flux. Break each pixel into upSample^2 sub pixels, #each subpixel should have 1/(upSample^2) the flux of the original pixel upSampledFrame/=float(upSample*upSample) ### UPDATE WITH DX AND DY DETERMINED FROM ACTUAL STARTING POINT ### SHIFTS ALL TO 0 RIGHT NOW #apply dX and dY shift to frame dX = centX*-1.*upSample dY = centY*-1.*upSample if verbose: print("Shifting timestamp %i, wvl %i by x=%2.2f, y=%2.2f" % (t, w, dX, dY)) shiftedFrame = irUtils.rotateShiftImage(upSampledFrame,0,dX,dY) cubeStack[w].append(shiftedFrame) cubeStack = np.array(cubeStack) for n in np.arange(nWvlBins): finalCube.append(irUtils.medianStack(cubeStack[n])) if verbose: plotArray(finalCube[n],title="%i nm"%centers[n]) finalCube = np.array(finalCube) return {'finalCube':finalCube,'wvls':centers, 'cubeStack':cubeStack, 'times':finalTimes}