def makeDark(stackForDark=None, outPutFileName=None, verbose=True): """ takes a list or array of arrays and makes a dark. Return an array with the dark. If outPutFileName is provided, it saves the dark in an npz """ try: dark = irUtils.medianStack(stackForDark) print("Making dark hot pixel mask") if verbose: plotArray(dark, title='Dark', origin='upper') except: print( "#### Error with the stack of dark. makeDark takes a list of array of arrays to make the dark ####" ) sys.exit(0) try: np.savez(darkPath, dark=dark) except: if outPutFileName == None: print("No output file name provided. Dark not saved") else: print("Wrong output path. Dark not saved") return dark
def makeDHPMask(stack=None, outputFileName=None, verbose=False, sigma=3, maxCut=2450, coldCut=False, manualArray=None): """ MaxCut sets level for initial hot pixel cut. Everything with cps>maxCut -> np.nan Sigma determines level for second cut. Everything with cps>mean+sigma*stdev -> np.nan If coldCut=True, third cut where everything with cps<mean-sigma*stdev -> np.nan manualArray is a way to give a list of bad pixels manually, in format [[row,col],[row,col],...] """ medStack = utils.medianStack(stack) if verbose: try: plotArray(medStack, title='Median Dark Stack') except: plt.matshow(medStack) plt.show() #initial masking, take out anything with cps > maxCut mask = np.zeros(np.shape(medStack), dtype=int) mask[np.where(medStack >= maxCut)] = 1 if verbose: try: plotArray(mask, title='cps>%i == 1' % maxCut) except: plt.matshow(mask) plt.show() medStack[np.where(mask == 1)] = np.nan medStack[np.where(medStack == 0)] = np.nan if verbose: try: plotArray(medStack, title='Median Stack with mask 1') except: plt.matshow(medStack) plt.show() #second round of masking, where cps > mean+sigma*std mask2 = np.zeros(np.shape(medStack), dtype=int) with warnings.catch_warnings(): # nan values will give an unnecessary RuntimeWarning warnings.simplefilter("ignore", category=RuntimeWarning) mask2[np.where(medStack >= np.nanmean(medStack) + sigma * np.nanstd(medStack))] = 1 #if coldCut is true, also mask cps < mean-sigma*std if coldCut == True: with warnings.catch_warnings(): # nan values will give an unnecessary RuntimeWarning warnings.simplefilter("ignore", category=RuntimeWarning) mask2[np.where(medStack <= np.nanmean(medStack) - sigma * np.nanstd(medStack))] = 1 if verbose: try: plotArray(mask2, title='cps>mean+%i-sigma == 1' % sigma) except: plt.matshow(mask2) plt.show() medStack[np.where(mask2 == 1)] = np.nan if verbose: try: plotArray(medStack, title='Median Stack with mask 2') except: plt.matshow(medStack) plt.show() finalMask = mask + mask2 # provide an easy means to pipe in an array of manually identified pixels if manualArray is not None: for pix in manualArray: finalMask[pix[0], pix[1]] = 1 if verbose: try: plotArray(finalMask, title='Final mask') except: plt.matshow(finalMask) plt.show() print("Number masked pixels = ", len(np.array(np.where(finalMask == 1)).flatten())) return finalMask
def plotMask(mask): try: plotArray(mask, title='hp Mask') except: plt.matshow(mask) plt.show()
def makeMask(run=None, date=None, basePath=None, startTimeStamp=None, stopTimeStamp=None, verbose=False, sigma=3, maxCut=2450, coldCut=False, manualArray=None): """ MaxCut sets level for initial hot pixel cut. Everything with cps>maxCut -> np.nan Sigma determines level for second cut. Everything with cps>mean+sigma*stdev -> np.nan If coldCut=True, third cut where everything with cps<mean-sigma*stdev -> np.nan manualArray is a way to give a list of bad pixels manually, in format [[row,col],[row,col],...] """ try: dataPath = basePath + str(run) + os.path.sep + str(date) + os.path.sep stack = loadIMGStack(dataPath, startTimeStamp, stopTimeStamp, nCols=nCols, nRows=nRows) except: print("Could not find dark data in ScienceData path, checking ramdisk") dataPath = '/mnt/ramdisk/' stack = loadIMGStack(dataPath, startTimeStamp, stopTimeStamp, nCols=nCols, nRows=nRows) medStack = irUtils.medianStack(stack) ##### if verbose: try: plotArray(medStack, title='Median Dark Stack') except: plt.matshow(medStack) plt.show() #initial masking, take out anything with cps > maxCut mask = np.zeros(np.shape(medStack), dtype=int) mask[np.where(medStack >= maxCut)] = 1 if verbose: try: plotArray(mask, title='cps>%i == 1' % maxCut) except: plt.matshow(mask) plt.show() medStack[np.where(mask == 1)] = np.nan medStack[np.where(medStack == 0)] = np.nan if verbose: try: plotArray(medStack, title='Median Stack with mask 1') except: plt.matshow(medStack) plt.show() #second round of masking, where cps > mean+sigma*std mask2 = np.zeros(np.shape(medStack), dtype=int) mask2[np.where( medStack >= np.nanmean(medStack) + sigma * np.nanstd(medStack))] = 1 #if coldCut is true, also mask cps < mean-sigma*std if coldCut == True: mask2[np.where(medStack <= np.nanmean(medStack) - sigma * np.nanstd(medStack))] = 1 if verbose: try: plotArray(mask2, title='cps>mean+%i-sigma == 1' % sigma) except: plt.matshow(mask2) plt.show() medStack[np.where(mask2 == 1)] = np.nan if verbose: try: plotArray(medStack, title='Median Stack with mask 2') except: plt.matshow(medStack) plt.show() finalMask = mask + mask2 # provide an easy means to pipe in an array of manually identified pixels if manualArray is not None: for pix in manualArray: finalMask[pix[0], pix[1]] = 1 if verbose: try: plotArray(finalMask, title='Final mask') except: plt.matshow(finalMask) plt.show() print("Number masked pixels = ", len(np.array(np.where(finalMask == 1)).flatten())) return finalMask
darkStack = loadBINStack(dataDir, darkSpan[0], darkSpan[1], nCols=numCols, nRows=numRows) dark = irUtils.medianStack(darkStack) else: print("No dark provided") dark = np.zeros((numRows, numCols),dtype=int) #if dark frames are provided, generate a hot pixel mask using SR Meeker's darkHotPixMask.py if darkSpan[0]!='0': darkHPM = dhpm.makeMask(run=run, date=date, basePath=dataDir,startTimeStamp=darkSpan[0], stopTimeStamp=darkSpan[1], coldCut=False, manualArray=None) else: print("Failed to generate dark mask. Turning off hot pixel masking") #Apply the hot pixel mask and plot the hot pixel masked dark frame dark[np.where(darkHPM==1)]=np.nan plotArray(dark,title='Dark',origin='upper') #load flat frames if flatSpan[0]!='0': print("Loading flat frame") if useImg == True: flatStack = loadIMGStack(dataDir, flatSpan[0], flatSpan[1], nCols=numCols, nRows=numRows) else: flatStack = loadBINStack(dataDir, flatSpan[0], flatSpan[1], nCols=numCols, nRows=numRows) flat = irUtils.medianStack(flatStack) plotArray(flat,title='NotdarkSubFlat') #dark subtract the flat flatSub=flat-dark flatSub[np.where(flatSub < 0.0)]=np.nan else: print("No flat provided")
def makeFlat(flatStack=None, dark=None, outPutFileName=None, badPixelMask=None, crop=False, cropColsForMedian=[(0, 19), (60, 79)], cropRowsForMedian=[(25, 49)], verbose=True): """ Input: flatStack: array or list of arrays to make the flat dark: dark. If none the flat is not dark subtracteed outPutFileName: if not provided returns the flat array without saving it crop: if False uses the entire array to calculate the median, otherwise it crops the columns in the range specified by cropForMedian cropColsForMedian, cropRowsForMedian: are tuples with each element being the range of columns/rows to be cropped when taking the median for the flat 10/1/2017: currently crops the high frequency boards (bad performances) and FL2 (not working on Faceless) Returns: dictionary with 'weights' and 'flat' """ try: flat = irUtils.medianStack(flatStack) print("Loading flat frame") except: print("No valid flat stack provided. Exiting") sys.exit(0) if verbose: plotArray(flat, title='NotdarkSubFlat') if darkSpan[0] != '0': #dark subtract the flat print("Subtracting dark") flatSub = flat - dark else: print("Dark=None, will make a flat without dark subtraction") flatSub = flat flatSub[np.where(flatSub < 0.0)] = np.nan croppedFrame = np.copy(flatSub) if crop: if cropColsForMedian != None: cropColsForMedian = np.array(cropColsForMedian) for iCrop, cropInd in enumerate(cropColsForMedian): croppedFrame[:, cropInd[0]:cropInd[1] + 1] = np.nan if cropRowsForMedian != None: cropRowsForMedian = np.array(cropRowsForMedian) for iCrop, cropInd in enumerate(cropRowsForMedian): croppedFrame[cropInd[0]:cropInd[1] + 1, :] = np.nan #Apply bad pixel mask if badPixelMask != None: print("Applying bad pixel mask") croppedFrame[badPixelMask != 0] = np.nan if verbose: plotArray(croppedFrame, title='Cropped flat frame') #Plot the cropped flat frame med = np.nanmedian(croppedFrame.flatten()) print('median', med) flatSub[flatSub == 0] = 1 weights = med / flatSub #calculate the weights by dividing the median by the dark-subtracted flat frame if verbose: plotArray(weights, title='Weights') #Plot the weights try: np.savez(outPutFileName, weights=weights, flat=flatSub) except: if outPutFileName == None: print('No output file name provided. Not saving flat') else: print('Output file name not valid. Not saving flat') dict = {} dict['flat'] = flat dict['weights'] = weights return dict
print('Usage: {} tstampStart tstampEnd'.format(sys.argv[0])) exit(0) else: startTstamp = int(sys.argv[1]) endTstamp = int(sys.argv[2]) darkStart = 1469354906 darkEnd = 1469354926 darkStack = loadImageStack(darkStart, darkEnd) darkFrame = medianFrame(darkStack) imageStack = loadImageStack(startTstamp, endTstamp) #option to flatten stack to median combined frame mf = medianFrame(imageStack) FITSfile = str(startTstamp) + '_to_' + str( endTstamp) + '_goodBeammapOnly.fits' path = outpath + FITSfile writeFits(imageStack, path) if not mf == None: medianFile = str(startTstamp) + '_to_' + str( endTstamp) + '_goodBeammapOnly_median.fits' path = outpath + medianFile writeFits(mf, path) if verbose: plotArray(image=mf) #form = PopUp(showMe=False,title='B') #form.plotArray(np.arange(9).reshape(3,3)) #form.show()
################################################# if doHPM: print("Making HPM") darkHPMImg = dhpm.makeMask(basePath=imgDir, startTimeStamp=darkSpanImg[0], stopTimeStamp=darkSpanImg[1], coldCut=True, maxCut=maxCut, sigma=sigma, manualArray=None) hpFN = 'darkHPMImg_' + target + '.npz' #Save the hot pixel mask into the output directory hpPath = os.path.join(outputDir, hpFN) np.savez(hpPath, darkHPMImg=darkHPMImg) plotArray(darkHPMImg, title='Hot Pixel Mask Image', origin='upper') darkHPMFlat = dhpm.makeMask(basePath=imgDir, startTimeStamp=darkSpanFlat[0], stopTimeStamp=darkSpanFlat[1], coldCut=True, maxCut=maxCut, sigma=sigma, manualArray=None) hpFN = 'darkHPMFlat_' + target + '.npz' #Save the hot pixel mask into the output directory hpPath = os.path.join(outputDir, hpFN) np.savez(hpPath, darkHPMFlat=darkHPMFlat) plotArray(darkHPMFlat, title='Hot Pixel Mask Flat', origin='upper') else:
def stackCube(h5File,npzFile, verbose=True): #load npz file with cubes and timestamps npzDict = loadCubeStack(npzFile) cubeTimes = npzDict['times'] cubeCubes = npzDict['cubes'] cubeWBEs = npzDict['wvlBinEdges'] #define bin center wavelengths (in nm) wbWidths = np.diff(cubeWBEs) centers = cubeWBEs[:-1]+wbWidths/2.0 nWvlBins = len(centers) if verbose: print("Loaded npz file...") #load h5 file with centroids, image resampling info, dark frames, and timestamps h5Dict = loadH5Stack(h5File) h5Times = h5Dict['times'] centXs = h5Dict['centX'] centYs = h5Dict['centY'] hotPix = h5Dict['hpm'] #get image stacking info from params dictionary paramsDict = h5Dict['params'] padFraction = np.float(paramsDict['padFraction'][0]) upSample = paramsDict['upSample'][0] doHPM = bool(paramsDict['doHPM'][0]) coldCut = paramsDict['coldCut'][0] if verbose: print("Loaded h5 file...") if cubeTimes.all() == h5Times.all(): print("Timestamps match. Carrying on with stacking...") else: print("Timestamp mismatch between two files!!") print(cubeTimes) print(h5Times) sys.exit(0) if doHPM: hpMask = hotPix[0] cubeStack = [] for i in range(nWvlBins): cubeStack.append([]) finalCube = [] finalTimes = [] for t in np.arange(len(cubeTimes)): time = cubeTimes[t] cube = cubeCubes[t] centX = centXs[t] centY = centYs[t] finalTimes.append(cubeTimes[t]) for w in np.arange(nWvlBins): im = np.array(cube[:,:,w],float) im = np.transpose(im) #apply hp mask to image if doHPM: im[np.where(hpMask==1)]=np.nan #cut out cold/dead pixels im[np.where(im<=coldCut)]=np.nan #pad frame with margin for shifting and stacking paddedFrame = irUtils.embedInLargerArray(im,frameSize=padFraction) #upSample frame for sub-pixel registration with fitting code upSampledFrame = irUtils.upSampleIm(paddedFrame,upSample) #conserve flux. Break each pixel into upSample^2 sub pixels, #each subpixel should have 1/(upSample^2) the flux of the original pixel upSampledFrame/=float(upSample*upSample) ### UPDATE WITH DX AND DY DETERMINED FROM ACTUAL STARTING POINT ### SHIFTS ALL TO 0 RIGHT NOW #apply dX and dY shift to frame dX = centX*-1.*upSample dY = centY*-1.*upSample if verbose: print("Shifting timestamp %i, wvl %i by x=%2.2f, y=%2.2f" % (t, w, dX, dY)) shiftedFrame = irUtils.rotateShiftImage(upSampledFrame,0,dX,dY) cubeStack[w].append(shiftedFrame) cubeStack = np.array(cubeStack) for n in np.arange(nWvlBins): finalCube.append(irUtils.medianStack(cubeStack[n])) if verbose: plotArray(finalCube[n],title="%i nm"%centers[n]) finalCube = np.array(finalCube) return {'finalCube':finalCube,'wvls':centers, 'cubeStack':cubeStack, 'times':finalTimes}
#break into 64 bit words words = np.array(struct.unpack('>{:d}Q'.format(nWords), data), dtype=object) parseDict = parsePacketData(words, verbose=True) phasesDeg = parseDict['phasesDeg'] basesDeg = parseDict['basesDeg'] pixelIds = parseDict['pixelIds'] image = parseDict['image'] #selPixelId = 0#(1<<10)+23 selPixelId = (30 << 10) + 46 print('selected pixel', selPixelId) print(len(np.where(pixelIds == selPixelId)), 'photons for selected pixel') print('phase', phasesDeg[0:10]) print('base', basesDeg[0:10]) print('IDs', pixelIds[0:10]) fig, ax = plt.subplots(1, 1) ax.plot(phasesDeg[np.where(pixelIds == selPixelId)]) ax.plot(basesDeg[np.where(pixelIds == selPixelId)]) ax.set_title('phases (deg)') #ax.plot(pixelIds) plotArray(image, origin='upper') np.savez('/mnt/data0/test2/{}.npz'.format(pathTstamp), **parseDict) plt.show()
def makeMaskedImage(imagePath=None, verbose=False, sigma=None, maxCut=2450, coldCut=False): """ MaxCut sets level for initial hot pixel cut. Everything with cps>maxCut -> np.nan Sigma determines level for second cut. Everything with cps>mean+sigma*stdev -> np.nan If coldCut=True, third cut where everything with cps<mean-sigma*stdev -> np.nan imagePath: The full image path to where your .np file is verbose: Do you want it to make the plots or not sigma: How many standard deviations away from the mean is an acceptable pixel value maxCut: Initial cut, 2450 as per SR Meeker coldCut: Should pixels that are too low in value be cut (mean-sigma*st.dev) """ array = np.load(imagePath) imageDirPath, imageBasename = os.path.split(imagePath) if verbose: try: plotArray(array, title='Unmasked Image') except: plt.matshow(array) plt.show() #initial masking, take out anything with cps > maxCut mask = np.zeros(np.shape(array), dtype=int) mask[np.where(array >= maxCut)] = 1 if verbose: try: plotArray(mask, title='cps>%i == 1' % maxCut) except: plt.matshow(mask) plt.show() array[np.where(mask == 1)] = np.nan array[np.where(array == 0)] = np.nan if verbose: try: plotArray(array, title='Image with mask 1') except: plt.matshow(array) plt.show() #second round of masking, where cps > mean+sigma*std mask2 = np.zeros(np.shape(array), dtype=int) with warnings.catch_warnings(): # nan values will give an unnecessary RuntimeWarning warnings.simplefilter("ignore", category=RuntimeWarning) mask2[np.where( array >= np.nanmean(array) + sigma * np.nanstd(array))] = 1 #if coldCut is true, also mask cps < mean-sigma*std if coldCut == True: with warnings.catch_warnings(): # nan values will give an unnecessary RuntimeWarning warnings.simplefilter("ignore", category=RuntimeWarning) mask2[np.where( array <= np.nanmean(array) - sigma * np.nanstd(array))] = 1 if verbose: try: plotArray(mask2, title='cps>mean+%i-sigma == 1' % sigma) except: plt.matshow(mask2) plt.show() array[np.where(mask2 == 1)] = np.nan if verbose: try: plotArray(array, title='Image with mask 2') except: plt.matshow(array) plt.show() saveMaskBasename = os.path.splitext(imageBasename)[0] + 'Mask' saveImageBasename = os.path.splitext(imageBasename)[0] + 'HPMasked' outfileMask = os.path.join(imageDirPath, saveMaskBasename) outfileImage = os.path.join(imageDirPath, saveImageBasename) finalMask = mask + mask2 finalImage = array np.save(outfileMask, finalMask) np.save(outfileImage, finalImage)