def makeDark(stackForDark=None, outPutFileName=None,verbose=True):
"""
takes a list or array of arrays and makes a dark. Return an array with the dark. If outPutFileName is provided, it saves the dark in an npz
"""
try:
dark = irUtils.medianStack(stackForDark)
print "Making dark hot pixel mask"
if verbose:
plotArray(dark,title='Dark',origin='upper')
except:
print "#### Error with the stack of dark. makeDark takes a list of array of arrays to make the dark ####"
sys.exit(0)
try:
np.savez(darkPath,dark=dark)
except:
if outPutFileName==None:
print "No output file name provided. Dark not saved"
else:
print "Wrong output path. Dark not saved"
return dark
def stackCube(h5File,npzFile, verbose=True):
#load npz file with cubes and timestamps
npzDict = loadCubeStack(npzFile)
cubeTimes = npzDict['times']
cubeCubes = npzDict['cubes']
cubeWBEs = npzDict['wvlBinEdges']
#define bin center wavelengths (in nm)
wbWidths = np.diff(cubeWBEs)
centers = cubeWBEs[:-1]+wbWidths/2.0
nWvlBins = len(centers)
if verbose: print "Loaded npz file..."
#load h5 file with centroids, image resampling info, dark frames, and timestamps
h5Dict = loadH5Stack(h5File)
h5Times = h5Dict['times']
centXs = h5Dict['centX']
centYs = h5Dict['centY']
hotPix = h5Dict['hpm']
#get image stacking info from params dictionary
paramsDict = h5Dict['params']
padFraction = np.float(paramsDict['padFraction'][0])
upSample = paramsDict['upSample'][0]
doHPM = bool(paramsDict['doHPM'][0])
coldCut = paramsDict['coldCut'][0]
if verbose: print "Loaded h5 file..."
if cubeTimes.all() == h5Times.all():
print "Timestamps match. Carrying on with stacking..."
else:
print "Timestamp mismatch between two files!!"
print cubeTimes
print h5Times
sys.exit(0)
if doHPM:
hpMask = hotPix[0]
cubeStack = []
for i in range(nWvlBins): cubeStack.append([])
finalCube = []
finalTimes = []
for t in np.arange(len(cubeTimes)):
time = cubeTimes[t]
cube = cubeCubes[t]
centX = centXs[t]
centY = centYs[t]
finalTimes.append(cubeTimes[t])
for w in np.arange(nWvlBins):
im = np.array(cube[:,:,w],float)
im = np.transpose(im)
#apply hp mask to image
if doHPM:
im[np.where(hpMask==1)]=np.nan
#cut out cold/dead pixels
im[np.where(im<=coldCut)]=np.nan
#pad frame with margin for shifting and stacking
paddedFrame = irUtils.embedInLargerArray(im,frameSize=padFraction)
#upSample frame for sub-pixel registration with fitting code
upSampledFrame = irUtils.upSampleIm(paddedFrame,upSample)
#conserve flux. Break each pixel into upSample^2 sub pixels,
#each subpixel should have 1/(upSample^2) the flux of the original pixel
upSampledFrame/=float(upSample*upSample)
### UPDATE WITH DX AND DY DETERMINED FROM ACTUAL STARTING POINT
### SHIFTS ALL TO 0 RIGHT NOW
#apply dX and dY shift to frame
dX = centX*-1.*upSample
dY = centY*-1.*upSample
if verbose:
print "Shifting timestamp %i, wvl %i by x=%2.2f, y=%2.2f"%(t,w, dX, dY)
shiftedFrame = irUtils.rotateShiftImage(upSampledFrame,0,dX,dY)
cubeStack[w].append(shiftedFrame)
cubeStack = np.array(cubeStack)
for n in np.arange(nWvlBins):
finalCube.append(irUtils.medianStack(cubeStack[n]))
if verbose:
plotArray(finalCube[n],title="%i nm"%centers[n])
finalCube = np.array(finalCube)
return {'finalCube':finalCube,'wvls':centers, 'cubeStack':cubeStack, 'times':finalTimes}
def makeFlat(flatStack=None, dark=None, outPutFileName=None, badPixelMask=None, crop=False, cropColsForMedian=[(0,19),(60,79)], cropRowsForMedian=[(25,49)], verbose=True):
"""
Input:
flatStack: array or list of arrays to make the flat
dark: dark. If none the flat is not dark subtracteed
outPutFileName: if not provided returns the flat array without saving it
crop: if False uses the entire array to calculate the median, otherwise it crops the columns in the range specified by cropForMedian
cropColsForMedian, cropRowsForMedian: are tuples with each element being the range of columns/rows to be cropped when taking the median for the flat
10/1/2017: currently crops the high frequency boards (bad performances) and FL2 (not working on Faceless)
Returns:
dictionary with 'weights' and 'flat'
"""
try:
flat = irUtils.medianStack(flatStack)
print "Loading flat frame"
except:
print "No valid flat stack provided. Exiting"
sys.exit(0)
if verbose:
plotArray(flat,title='NotdarkSubFlat')
if dark==None:
#dark subtract the flat
print "Subtracting dark"
flatSub=flat-dark
else:
print "Dark=None, will make a flat without dark subtraction"
flatSub=flat
flatSub[np.where(flatSub < 0.0)]=np.nan
croppedFrame = np.copy(flatSub)
if crop:
if cropColsForMedian!=None:
cropColsForMedian=np.array(cropColsForMedian)
for iCrop, cropInd in enumerate(cropColsForMedian):
croppedFrame[:,cropInd[0]:cropInd[1]+1]=np.nan
if cropRowsForMedian!=None:
cropRowsForMedian=np.array(cropRowsForMedian)
for iCrop, cropInd in enumerate(cropRowsForMedian):
croppedFrame[cropInd[0]:cropInd[1]+1,:]=np.nan
#Apply bad pixel mask
if badPixelMask!=None:
print "Applying bad pixel mask"
croppedFrame[badPixelMask!=0] = np.nan
if verbose:
plotArray(croppedFrame,title='Cropped flat frame') #Plot the cropped flat frame
med = np.nanmedian(croppedFrame.flatten())
print 'median', med
flatSub[flatSub==0]=1
weights = med/flatSub #calculate the weights by dividing the median by the dark-subtracted flat frame
if verbose:
plotArray(weights,title='Weights') #Plot the weights
try:
np.savez(outPutFileName,weights = weights,flat=flatSub)
except:
if outPutFileName==None:
print 'No output file name provided. Not saving flat'
else:
print 'Output file name not valid. Not saving flat'
dict={}
dict['flat']=flat
dict['weights']=weights
return dict
def makeDHPMask(stack=None, outputFileName=None, verbose=False, sigma=3, maxCut=2450, coldCut=False, manualArray=None):
'''
MaxCut sets level for initial hot pixel cut. Everything with cps>maxCut -> np.nan
Sigma determines level for second cut. Everything with cps>mean+sigma*stdev -> np.nan
If coldCut=True, third cut where everything with cps<mean-sigma*stdev -> np.nan
manualArray is a way to give a list of bad pixels manually, in format [[row,col],[row,col],...]
'''
medStack = irUtils.medianStack(stack)
if verbose:
try:
plotArray(medStack,title='Median Dark Stack')
except:
plt.matshow(medStack)
plt.show()
#initial masking, take out anything with cps > maxCut
mask = np.zeros(np.shape(medStack),dtype=int)
mask[np.where(medStack>=maxCut)]=1
if verbose:
try:
plotArray(mask, title='cps>%i == 1'%maxCut)
except:
plt.matshow(mask)
plt.show()
medStack[np.where(mask==1)]=np.nan
medStack[np.where(medStack==0)]=np.nan
if verbose:
try:
plotArray(medStack, title='Median Stack with mask 1')
except:
plt.matshow(medStack)
plt.show()
#second round of masking, where cps > mean+sigma*std
mask2 = np.zeros(np.shape(medStack),dtype=int)
mask2[np.where(medStack>=np.nanmean(medStack)+sigma*np.nanstd(medStack))]=1
#if coldCut is true, also mask cps < mean-sigma*std
if coldCut==True:
mask2[np.where(medStack<=np.nanmean(medStack)-sigma*np.nanstd(medStack))]=1
if verbose:
try:
plotArray(mask2, title='cps>mean+%i-sigma == 1'%sigma)
except:
plt.matshow(mask2)
plt.show()
medStack[np.where(mask2==1)]=np.nan
if verbose:
try:
plotArray(medStack, title='Median Stack with mask 2')
except:
plt.matshow(medStack)
plt.show()
finalMask = mask+mask2
# provide an easy means to pipe in an array of manually identified pixels
if manualArray is not None:
for pix in manualArray:
finalMask[pix[0],pix[1]]=1
if verbose:
try:
plotArray(finalMask, title='Final mask')
except:
plt.matshow(finalMask)
plt.show()
print("Number masked pixels = ", len(np.array(np.where(finalMask==1)).flatten()))
return finalMask
#pad frame with margin for shifting and stacking
paddedFrame = irUtils.embedInLargerArray(processedIm,
frameSize=padFraction)
outfile = h5dir + outfileName + str(i)
np.save(outfile, paddedFrame)
#apply rough dX and dY shift to frame
print("Shifting dither %i, frame %i by x=%i, y=%i" %
(i, 0, dXs[i], dYs[i]))
shiftedFrame = irUtils.rotateShiftImage(paddedFrame, 0, dXs[i], dYs[i])
#upSample frame for sub-pixel registration with fitting code
upSampledFrame = irUtils.upSampleIm(shiftedFrame, upSample)
#conserve flux. Break each pixel into upSample^2 sub pixels,
#each subpixel should have 1/(upSample^2) the flux of the original pixel
upSampledFrame /= float(upSample * upSample)
#append upsampled, padded frame to array for storage into next part of code
ditherFrames.append(upSampledFrame)
print("Loaded dither position %i" % i)
shiftedFrames = np.array(ditherFrames)
#take median stack of all shifted frames
finalImage = irUtils.medianStack(shiftedFrames)
plotArray(finalImage, title='final', origin='upper')
outfile = h5dir + outfileName + 'Aligned'
np.save(outfile, finalImage)
plotArray(processedIm,title='Dither Pos %i ColdPix Masked'%i,origin='upper',vmin=0)
#pad frame with margin for shifting and stacking
paddedFrame = irUtils.embedInLargerArray(processedIm,frameSize=padFraction)
outfile=h5dir+outfileName+str(i)
np.save(outfile, paddedFrame)
#apply rough dX and dY shift to frame
print("Shifting dither %i, frame %i by x=%i, y=%i"%(i,0,dXs[i], dYs[i]))
shiftedFrame = irUtils.rotateShiftImage(paddedFrame,0,dXs[i],dYs[i])
#upSample frame for sub-pixel registration with fitting code
upSampledFrame = irUtils.upSampleIm(shiftedFrame,upSample)
#conserve flux. Break each pixel into upSample^2 sub pixels,
#each subpixel should have 1/(upSample^2) the flux of the original pixel
upSampledFrame/=float(upSample*upSample)
#append upsampled, padded frame to array for storage into next part of code
ditherFrames.append(upSampledFrame)
print("Loaded dither position %i"%i)
shiftedFrames = np.array(ditherFrames)
#take median stack of all shifted frames
finalImage = irUtils.medianStack(shiftedFrames)
plotArray(finalImage,title='final',origin='upper')
outfile=h5dir+outfileName+'Aligned'
np.save(outfile, finalImage)
def stackCube(h5File, npzFile, verbose=True):
#load npz file with cubes and timestamps
npzDict = loadCubeStack(npzFile)
cubeTimes = npzDict['times']
cubeCubes = npzDict['cubes']
cubeWBEs = npzDict['wvlBinEdges']
#define bin center wavelengths (in nm)
wbWidths = np.diff(cubeWBEs)
centers = cubeWBEs[:-1] + wbWidths / 2.0
nWvlBins = len(centers)
if verbose: print "Loaded npz file..."
#load h5 file with centroids, image resampling info, dark frames, and timestamps
h5Dict = loadH5Stack(h5File)
h5Times = h5Dict['times']
centXs = h5Dict['centX']
centYs = h5Dict['centY']
hotPix = h5Dict['hpm']
#get image stacking info from params dictionary
paramsDict = h5Dict['params']
padFraction = np.float(paramsDict['padFraction'][0])
upSample = paramsDict['upSample'][0]
doHPM = bool(paramsDict['doHPM'][0])
coldCut = paramsDict['coldCut'][0]
if verbose: print "Loaded h5 file..."
if cubeTimes.all() == h5Times.all():
print "Timestamps match. Carrying on with stacking..."
else:
print "Timestamp mismatch between two files!!"
print cubeTimes
print h5Times
sys.exit(0)
if doHPM:
hpMask = hotPix[0]
cubeStack = []
for i in range(nWvlBins):
cubeStack.append([])
finalCube = []
finalTimes = []
for t in np.arange(len(cubeTimes)):
time = cubeTimes[t]
cube = cubeCubes[t]
centX = centXs[t]
centY = centYs[t]
finalTimes.append(cubeTimes[t])
for w in np.arange(nWvlBins):
im = np.array(cube[:, :, w], float)
im = np.transpose(im)
#apply hp mask to image
if doHPM:
im[np.where(hpMask == 1)] = np.nan
#cut out cold/dead pixels
im[np.where(im <= coldCut)] = np.nan
#pad frame with margin for shifting and stacking
paddedFrame = irUtils.embedInLargerArray(im, frameSize=padFraction)
#upSample frame for sub-pixel registration with fitting code
upSampledFrame = irUtils.upSampleIm(paddedFrame, upSample)
#conserve flux. Break each pixel into upSample^2 sub pixels,
#each subpixel should have 1/(upSample^2) the flux of the original pixel
upSampledFrame /= float(upSample * upSample)
### UPDATE WITH DX AND DY DETERMINED FROM ACTUAL STARTING POINT
### SHIFTS ALL TO 0 RIGHT NOW
#apply dX and dY shift to frame
dX = centX * -1. * upSample
dY = centY * -1. * upSample
if verbose:
print "Shifting timestamp %i, wvl %i by x=%2.2f, y=%2.2f" % (
t, w, dX, dY)
shiftedFrame = irUtils.rotateShiftImage(upSampledFrame, 0, dX, dY)
cubeStack[w].append(shiftedFrame)
cubeStack = np.array(cubeStack)
for n in np.arange(nWvlBins):
finalCube.append(irUtils.medianStack(cubeStack[n]))
if verbose:
plotArray(finalCube[n], title="%i nm" % centers[n])
finalCube = np.array(finalCube)
return {
'finalCube': finalCube,
'wvls': centers,
'cubeStack': cubeStack,
'times': finalTimes
}
# darkHPM = dhpm.loadMask(hpFile)
# print "Loaded hot pixel mask from dark data %s"%hpFile
#load dark frames
if calibPath[0]!='0':
darkPath=os.path.join(calibPath,"dark_"+target+".npz")
darkfile=np.load(darkPath)
dark=darkfile['dark']
print "Found Dark File"
else:
print "Loading dark frame to make Dark and Saving It"
if useImg == True:
darkStackImg = loadIMGStack(dataDir, darkSpanImg[0], darkSpanImg[1], nCols=numCols, nRows=numRows)
else:
darkStackImg = loadBINStack(dataDir, darkSpanImg[0], darkSpanImg[1], nCols=numCols, nRows=numRows)
darkImg = irUtils.medianStack(darkStackImg)
darkFN='darkImg_'+target+'.npz' #Save the dark into the output directory
darkPath = os.path.join(outputDir,darkFN)
np.savez(darkPath,darkImg = darkImg)
if calibPath[0]!='0':
darkPath=os.path.join(calibPath,"dark_"+target+".npz")
darkfile=np.load(darkPath)
dark=darkfile['dark']
print "Found Dark File"
else:
print "Loading dark frame to make Dark and Saving It"
if useImg == True:
darkStackFlat = loadIMGStack(dataDir, darkSpanFlat[0], darkSpanFlat[1], nCols=numCols, nRows=numRows)