def makeDark(stackForDark=None, outPutFileName=None, verbose=True):
"""
takes a list or array of arrays and makes a dark. Return an array with the dark. If outPutFileName is provided, it saves the dark in an npz
"""
try:
dark = irUtils.medianStack(stackForDark)
print("Making dark hot pixel mask")
if verbose:
plotArray(dark, title='Dark', origin='upper')
except:
print(
"#### Error with the stack of dark. makeDark takes a list of array of arrays to make the dark ####"
)
sys.exit(0)
try:
np.savez(darkPath, dark=dark)
except:
if outPutFileName == None:
print("No output file name provided. Dark not saved")
else:
print("Wrong output path. Dark not saved")
return dark
def makeDHPMask(stack=None,
outputFileName=None,
verbose=False,
sigma=3,
maxCut=2450,
coldCut=False,
manualArray=None):
"""
MaxCut sets level for initial hot pixel cut. Everything with cps>maxCut -> np.nan
Sigma determines level for second cut. Everything with cps>mean+sigma*stdev -> np.nan
If coldCut=True, third cut where everything with cps<mean-sigma*stdev -> np.nan
manualArray is a way to give a list of bad pixels manually, in format [[row,col],[row,col],...]
"""
medStack = utils.medianStack(stack)
if verbose:
try:
plotArray(medStack, title='Median Dark Stack')
except:
plt.matshow(medStack)
plt.show()
#initial masking, take out anything with cps > maxCut
mask = np.zeros(np.shape(medStack), dtype=int)
mask[np.where(medStack >= maxCut)] = 1
if verbose:
try:
plotArray(mask, title='cps>%i == 1' % maxCut)
except:
plt.matshow(mask)
plt.show()
medStack[np.where(mask == 1)] = np.nan
medStack[np.where(medStack == 0)] = np.nan
if verbose:
try:
plotArray(medStack, title='Median Stack with mask 1')
except:
plt.matshow(medStack)
plt.show()
#second round of masking, where cps > mean+sigma*std
mask2 = np.zeros(np.shape(medStack), dtype=int)
with warnings.catch_warnings():
# nan values will give an unnecessary RuntimeWarning
warnings.simplefilter("ignore", category=RuntimeWarning)
mask2[np.where(medStack >= np.nanmean(medStack) +
sigma * np.nanstd(medStack))] = 1
#if coldCut is true, also mask cps < mean-sigma*std
if coldCut == True:
with warnings.catch_warnings():
# nan values will give an unnecessary RuntimeWarning
warnings.simplefilter("ignore", category=RuntimeWarning)
mask2[np.where(medStack <= np.nanmean(medStack) -
sigma * np.nanstd(medStack))] = 1
if verbose:
try:
plotArray(mask2, title='cps>mean+%i-sigma == 1' % sigma)
except:
plt.matshow(mask2)
plt.show()
medStack[np.where(mask2 == 1)] = np.nan
if verbose:
try:
plotArray(medStack, title='Median Stack with mask 2')
except:
plt.matshow(medStack)
plt.show()
finalMask = mask + mask2
# provide an easy means to pipe in an array of manually identified pixels
if manualArray is not None:
for pix in manualArray:
finalMask[pix[0], pix[1]] = 1
if verbose:
try:
plotArray(finalMask, title='Final mask')
except:
plt.matshow(finalMask)
plt.show()
print("Number masked pixels = ",
len(np.array(np.where(finalMask == 1)).flatten()))
return finalMask
def plotMask(mask):
try:
plotArray(mask, title='hp Mask')
except:
plt.matshow(mask)
plt.show()
def makeMask(run=None,
date=None,
basePath=None,
startTimeStamp=None,
stopTimeStamp=None,
verbose=False,
sigma=3,
maxCut=2450,
coldCut=False,
manualArray=None):
"""
MaxCut sets level for initial hot pixel cut. Everything with cps>maxCut -> np.nan
Sigma determines level for second cut. Everything with cps>mean+sigma*stdev -> np.nan
If coldCut=True, third cut where everything with cps<mean-sigma*stdev -> np.nan
manualArray is a way to give a list of bad pixels manually, in format [[row,col],[row,col],...]
"""
try:
dataPath = basePath + str(run) + os.path.sep + str(date) + os.path.sep
stack = loadIMGStack(dataPath,
startTimeStamp,
stopTimeStamp,
nCols=nCols,
nRows=nRows)
except:
print("Could not find dark data in ScienceData path, checking ramdisk")
dataPath = '/mnt/ramdisk/'
stack = loadIMGStack(dataPath,
startTimeStamp,
stopTimeStamp,
nCols=nCols,
nRows=nRows)
medStack = irUtils.medianStack(stack) #####
if verbose:
try:
plotArray(medStack, title='Median Dark Stack')
except:
plt.matshow(medStack)
plt.show()
#initial masking, take out anything with cps > maxCut
mask = np.zeros(np.shape(medStack), dtype=int)
mask[np.where(medStack >= maxCut)] = 1
if verbose:
try:
plotArray(mask, title='cps>%i == 1' % maxCut)
except:
plt.matshow(mask)
plt.show()
medStack[np.where(mask == 1)] = np.nan
medStack[np.where(medStack == 0)] = np.nan
if verbose:
try:
plotArray(medStack, title='Median Stack with mask 1')
except:
plt.matshow(medStack)
plt.show()
#second round of masking, where cps > mean+sigma*std
mask2 = np.zeros(np.shape(medStack), dtype=int)
mask2[np.where(
medStack >= np.nanmean(medStack) + sigma * np.nanstd(medStack))] = 1
#if coldCut is true, also mask cps < mean-sigma*std
if coldCut == True:
mask2[np.where(medStack <= np.nanmean(medStack) -
sigma * np.nanstd(medStack))] = 1
if verbose:
try:
plotArray(mask2, title='cps>mean+%i-sigma == 1' % sigma)
except:
plt.matshow(mask2)
plt.show()
medStack[np.where(mask2 == 1)] = np.nan
if verbose:
try:
plotArray(medStack, title='Median Stack with mask 2')
except:
plt.matshow(medStack)
plt.show()
finalMask = mask + mask2
# provide an easy means to pipe in an array of manually identified pixels
if manualArray is not None:
for pix in manualArray:
finalMask[pix[0], pix[1]] = 1
if verbose:
try:
plotArray(finalMask, title='Final mask')
except:
plt.matshow(finalMask)
plt.show()
print("Number masked pixels = ",
len(np.array(np.where(finalMask == 1)).flatten()))
return finalMask
darkStack = loadBINStack(dataDir, darkSpan[0], darkSpan[1], nCols=numCols, nRows=numRows)
dark = irUtils.medianStack(darkStack)
else:
print("No dark provided")
dark = np.zeros((numRows, numCols),dtype=int)
#if dark frames are provided, generate a hot pixel mask using SR Meeker's darkHotPixMask.py
if darkSpan[0]!='0':
darkHPM = dhpm.makeMask(run=run, date=date, basePath=dataDir,startTimeStamp=darkSpan[0], stopTimeStamp=darkSpan[1], coldCut=False, manualArray=None)
else:
print("Failed to generate dark mask. Turning off hot pixel masking")
#Apply the hot pixel mask and plot the hot pixel masked dark frame
dark[np.where(darkHPM==1)]=np.nan
plotArray(dark,title='Dark',origin='upper')
#load flat frames
if flatSpan[0]!='0':
print("Loading flat frame")
if useImg == True:
flatStack = loadIMGStack(dataDir, flatSpan[0], flatSpan[1], nCols=numCols, nRows=numRows)
else:
flatStack = loadBINStack(dataDir, flatSpan[0], flatSpan[1], nCols=numCols, nRows=numRows)
flat = irUtils.medianStack(flatStack)
plotArray(flat,title='NotdarkSubFlat')
#dark subtract the flat
flatSub=flat-dark
flatSub[np.where(flatSub < 0.0)]=np.nan
else:
print("No flat provided")
def makeFlat(flatStack=None,
dark=None,
outPutFileName=None,
badPixelMask=None,
crop=False,
cropColsForMedian=[(0, 19), (60, 79)],
cropRowsForMedian=[(25, 49)],
verbose=True):
"""
Input:
flatStack: array or list of arrays to make the flat
dark: dark. If none the flat is not dark subtracteed
outPutFileName: if not provided returns the flat array without saving it
crop: if False uses the entire array to calculate the median, otherwise it crops the columns in the range specified by cropForMedian
cropColsForMedian, cropRowsForMedian: are tuples with each element being the range of columns/rows to be cropped when taking the median for the flat
10/1/2017: currently crops the high frequency boards (bad performances) and FL2 (not working on Faceless)
Returns:
dictionary with 'weights' and 'flat'
"""
try:
flat = irUtils.medianStack(flatStack)
print("Loading flat frame")
except:
print("No valid flat stack provided. Exiting")
sys.exit(0)
if verbose:
plotArray(flat, title='NotdarkSubFlat')
if darkSpan[0] != '0':
#dark subtract the flat
print("Subtracting dark")
flatSub = flat - dark
else:
print("Dark=None, will make a flat without dark subtraction")
flatSub = flat
flatSub[np.where(flatSub < 0.0)] = np.nan
croppedFrame = np.copy(flatSub)
if crop:
if cropColsForMedian != None:
cropColsForMedian = np.array(cropColsForMedian)
for iCrop, cropInd in enumerate(cropColsForMedian):
croppedFrame[:, cropInd[0]:cropInd[1] + 1] = np.nan
if cropRowsForMedian != None:
cropRowsForMedian = np.array(cropRowsForMedian)
for iCrop, cropInd in enumerate(cropRowsForMedian):
croppedFrame[cropInd[0]:cropInd[1] + 1, :] = np.nan
#Apply bad pixel mask
if badPixelMask != None:
print("Applying bad pixel mask")
croppedFrame[badPixelMask != 0] = np.nan
if verbose:
plotArray(croppedFrame,
title='Cropped flat frame') #Plot the cropped flat frame
med = np.nanmedian(croppedFrame.flatten())
print('median', med)
flatSub[flatSub == 0] = 1
weights = med / flatSub #calculate the weights by dividing the median by the dark-subtracted flat frame
if verbose:
plotArray(weights, title='Weights') #Plot the weights
try:
np.savez(outPutFileName, weights=weights, flat=flatSub)
except:
if outPutFileName == None:
print('No output file name provided. Not saving flat')
else:
print('Output file name not valid. Not saving flat')
dict = {}
dict['flat'] = flat
dict['weights'] = weights
return dict
print('Usage: {} tstampStart tstampEnd'.format(sys.argv[0]))
exit(0)
else:
startTstamp = int(sys.argv[1])
endTstamp = int(sys.argv[2])
darkStart = 1469354906
darkEnd = 1469354926
darkStack = loadImageStack(darkStart, darkEnd)
darkFrame = medianFrame(darkStack)
imageStack = loadImageStack(startTstamp, endTstamp)
#option to flatten stack to median combined frame
mf = medianFrame(imageStack)
FITSfile = str(startTstamp) + '_to_' + str(
endTstamp) + '_goodBeammapOnly.fits'
path = outpath + FITSfile
writeFits(imageStack, path)
if not mf == None:
medianFile = str(startTstamp) + '_to_' + str(
endTstamp) + '_goodBeammapOnly_median.fits'
path = outpath + medianFile
writeFits(mf, path)
if verbose:
plotArray(image=mf)
#form = PopUp(showMe=False,title='B')
#form.plotArray(np.arange(9).reshape(3,3))
#form.show()
#################################################
if doHPM:
print("Making HPM")
darkHPMImg = dhpm.makeMask(basePath=imgDir,
startTimeStamp=darkSpanImg[0],
stopTimeStamp=darkSpanImg[1],
coldCut=True,
maxCut=maxCut,
sigma=sigma,
manualArray=None)
hpFN = 'darkHPMImg_' + target + '.npz' #Save the hot pixel mask into the output directory
hpPath = os.path.join(outputDir, hpFN)
np.savez(hpPath, darkHPMImg=darkHPMImg)
plotArray(darkHPMImg, title='Hot Pixel Mask Image', origin='upper')
darkHPMFlat = dhpm.makeMask(basePath=imgDir,
startTimeStamp=darkSpanFlat[0],
stopTimeStamp=darkSpanFlat[1],
coldCut=True,
maxCut=maxCut,
sigma=sigma,
manualArray=None)
hpFN = 'darkHPMFlat_' + target + '.npz' #Save the hot pixel mask into the output directory
hpPath = os.path.join(outputDir, hpFN)
np.savez(hpPath, darkHPMFlat=darkHPMFlat)
plotArray(darkHPMFlat, title='Hot Pixel Mask Flat', origin='upper')
else:
def stackCube(h5File,npzFile, verbose=True):
#load npz file with cubes and timestamps
npzDict = loadCubeStack(npzFile)
cubeTimes = npzDict['times']
cubeCubes = npzDict['cubes']
cubeWBEs = npzDict['wvlBinEdges']
#define bin center wavelengths (in nm)
wbWidths = np.diff(cubeWBEs)
centers = cubeWBEs[:-1]+wbWidths/2.0
nWvlBins = len(centers)
if verbose: print("Loaded npz file...")
#load h5 file with centroids, image resampling info, dark frames, and timestamps
h5Dict = loadH5Stack(h5File)
h5Times = h5Dict['times']
centXs = h5Dict['centX']
centYs = h5Dict['centY']
hotPix = h5Dict['hpm']
#get image stacking info from params dictionary
paramsDict = h5Dict['params']
padFraction = np.float(paramsDict['padFraction'][0])
upSample = paramsDict['upSample'][0]
doHPM = bool(paramsDict['doHPM'][0])
coldCut = paramsDict['coldCut'][0]
if verbose: print("Loaded h5 file...")
if cubeTimes.all() == h5Times.all():
print("Timestamps match. Carrying on with stacking...")
else:
print("Timestamp mismatch between two files!!")
print(cubeTimes)
print(h5Times)
sys.exit(0)
if doHPM:
hpMask = hotPix[0]
cubeStack = []
for i in range(nWvlBins): cubeStack.append([])
finalCube = []
finalTimes = []
for t in np.arange(len(cubeTimes)):
time = cubeTimes[t]
cube = cubeCubes[t]
centX = centXs[t]
centY = centYs[t]
finalTimes.append(cubeTimes[t])
for w in np.arange(nWvlBins):
im = np.array(cube[:,:,w],float)
im = np.transpose(im)
#apply hp mask to image
if doHPM:
im[np.where(hpMask==1)]=np.nan
#cut out cold/dead pixels
im[np.where(im<=coldCut)]=np.nan
#pad frame with margin for shifting and stacking
paddedFrame = irUtils.embedInLargerArray(im,frameSize=padFraction)
#upSample frame for sub-pixel registration with fitting code
upSampledFrame = irUtils.upSampleIm(paddedFrame,upSample)
#conserve flux. Break each pixel into upSample^2 sub pixels,
#each subpixel should have 1/(upSample^2) the flux of the original pixel
upSampledFrame/=float(upSample*upSample)
### UPDATE WITH DX AND DY DETERMINED FROM ACTUAL STARTING POINT
### SHIFTS ALL TO 0 RIGHT NOW
#apply dX and dY shift to frame
dX = centX*-1.*upSample
dY = centY*-1.*upSample
if verbose:
print("Shifting timestamp %i, wvl %i by x=%2.2f, y=%2.2f" % (t, w, dX, dY))
shiftedFrame = irUtils.rotateShiftImage(upSampledFrame,0,dX,dY)
cubeStack[w].append(shiftedFrame)
cubeStack = np.array(cubeStack)
for n in np.arange(nWvlBins):
finalCube.append(irUtils.medianStack(cubeStack[n]))
if verbose:
plotArray(finalCube[n],title="%i nm"%centers[n])
finalCube = np.array(finalCube)
return {'finalCube':finalCube,'wvls':centers, 'cubeStack':cubeStack, 'times':finalTimes}
#break into 64 bit words
words = np.array(struct.unpack('>{:d}Q'.format(nWords), data),
dtype=object)
parseDict = parsePacketData(words, verbose=True)
phasesDeg = parseDict['phasesDeg']
basesDeg = parseDict['basesDeg']
pixelIds = parseDict['pixelIds']
image = parseDict['image']
#selPixelId = 0#(1<<10)+23
selPixelId = (30 << 10) + 46
print('selected pixel', selPixelId)
print(len(np.where(pixelIds == selPixelId)), 'photons for selected pixel')
print('phase', phasesDeg[0:10])
print('base', basesDeg[0:10])
print('IDs', pixelIds[0:10])
fig, ax = plt.subplots(1, 1)
ax.plot(phasesDeg[np.where(pixelIds == selPixelId)])
ax.plot(basesDeg[np.where(pixelIds == selPixelId)])
ax.set_title('phases (deg)')
#ax.plot(pixelIds)
plotArray(image, origin='upper')
np.savez('/mnt/data0/test2/{}.npz'.format(pathTstamp), **parseDict)
plt.show()
def makeMaskedImage(imagePath=None,
verbose=False,
sigma=None,
maxCut=2450,
coldCut=False):
"""
MaxCut sets level for initial hot pixel cut. Everything with cps>maxCut -> np.nan
Sigma determines level for second cut. Everything with cps>mean+sigma*stdev -> np.nan
If coldCut=True, third cut where everything with cps<mean-sigma*stdev -> np.nan
imagePath: The full image path to where your .np file is
verbose: Do you want it to make the plots or not
sigma: How many standard deviations away from the mean is an acceptable pixel value
maxCut: Initial cut, 2450 as per SR Meeker
coldCut: Should pixels that are too low in value be cut (mean-sigma*st.dev)
"""
array = np.load(imagePath)
imageDirPath, imageBasename = os.path.split(imagePath)
if verbose:
try:
plotArray(array, title='Unmasked Image')
except:
plt.matshow(array)
plt.show()
#initial masking, take out anything with cps > maxCut
mask = np.zeros(np.shape(array), dtype=int)
mask[np.where(array >= maxCut)] = 1
if verbose:
try:
plotArray(mask, title='cps>%i == 1' % maxCut)
except:
plt.matshow(mask)
plt.show()
array[np.where(mask == 1)] = np.nan
array[np.where(array == 0)] = np.nan
if verbose:
try:
plotArray(array, title='Image with mask 1')
except:
plt.matshow(array)
plt.show()
#second round of masking, where cps > mean+sigma*std
mask2 = np.zeros(np.shape(array), dtype=int)
with warnings.catch_warnings():
# nan values will give an unnecessary RuntimeWarning
warnings.simplefilter("ignore", category=RuntimeWarning)
mask2[np.where(
array >= np.nanmean(array) + sigma * np.nanstd(array))] = 1
#if coldCut is true, also mask cps < mean-sigma*std
if coldCut == True:
with warnings.catch_warnings():
# nan values will give an unnecessary RuntimeWarning
warnings.simplefilter("ignore", category=RuntimeWarning)
mask2[np.where(
array <= np.nanmean(array) - sigma * np.nanstd(array))] = 1
if verbose:
try:
plotArray(mask2, title='cps>mean+%i-sigma == 1' % sigma)
except:
plt.matshow(mask2)
plt.show()
array[np.where(mask2 == 1)] = np.nan
if verbose:
try:
plotArray(array, title='Image with mask 2')
except:
plt.matshow(array)
plt.show()
saveMaskBasename = os.path.splitext(imageBasename)[0] + 'Mask'
saveImageBasename = os.path.splitext(imageBasename)[0] + 'HPMasked'
outfileMask = os.path.join(imageDirPath, saveMaskBasename)
outfileImage = os.path.join(imageDirPath, saveImageBasename)
finalMask = mask + mask2
finalImage = array
np.save(outfileMask, finalMask)
np.save(outfileImage, finalImage)