def darknessSolution():
tile0 = FreqMapTile(nRows=40, nCols=25, startFreq=4000.0, bandwidth=2000.0, verbosity=2, name="center")
# tile0.addFreqHole(holeWidth=10.,holeCenter=1000.)
tile0.minNeighborSeparation = 70
tile0.minSecondNeighborSeparation = 8
tile0.maxNeighborSeparation = 550
tile0.simpleGradientSolution()
tile1 = FreqMapTile(nRows=40, nCols=25, startFreq=6200.0, bandwidth=2000.0, verbosity=2, name="outer")
# tile0.addFreqHole(holeWidth=10.,holeCenter=1000.)
tile1.minNeighborSeparation = 70
tile1.minSecondNeighborSeparation = 8
tile1.maxNeighborSeparation = 550
tile1.simpleGradientSolution()
np.savetxt("darkness_center.txt", tile0.freqTable)
np.savetxt("darkness_outer.txt", tile1.freqTable)
plotArray(tile0[:, :], origin="upper")
tile0.runSwaps(int(1e4))
tile1.runSwaps(int(1e4))
np.savetxt(
"darkness_center_swap_{}min_{}max.txt".format(tile0.minNeighborSeparation, tile0.maxNeighborSeparation),
tile0.freqTable,
)
np.savetxt(
"darkness_outer_swap_{}min_{}max.txt".format(tile1.minNeighborSeparation, tile1.maxNeighborSeparation),
tile1.freqTable,
)
combinedTile = np.vstack([tile1[0:20, :], tile0[:, :], tile1[20:, :]])
np.savetxt("darkness_feedline.txt", combinedTile)
print "saved!"
def mecSolution():
tile0 = FreqMapTile(nRows=73, nCols=14, startFreq=0.0, bandwidth=2000.0, verbosity=2, name="center")
# tile0.addFreqHole(holeWidth=10.,holeCenter=1000.)
tile0.minNeighborSeparation = 70
tile0.minSecondNeighborSeparation = 8
tile0.maxNeighborSeparation = 550
tile0.simpleGradientSolution2()
tile1 = FreqMapTile(nRows=73, nCols=14, startFreq=2200.0, bandwidth=2000.0, verbosity=2, name="outer")
# tile0.addFreqHole(holeWidth=10.,holeCenter=1000.)
tile1.minNeighborSeparation = 70
tile1.minSecondNeighborSeparation = 8
tile1.maxNeighborSeparation = 550
tile1.simpleGradientSolution2()
np.savetxt("mec_center.txt", tile0.freqTable)
np.savetxt("mec_outer.txt", tile1.freqTable)
tile0.runSwaps(int(1e4))
tile1.runSwaps(int(1e4))
np.savetxt(
"mec_center_swap_{}min_{}max.txt".format(tile0.minNeighborSeparation, tile0.maxNeighborSeparation),
tile0.freqTable,
)
np.savetxt(
"mec_outer_swap_{}min_{}max.txt".format(tile1.minNeighborSeparation, tile1.maxNeighborSeparation),
tile1.freqTable,
)
combinedTile = np.vstack([tile1[0:36, :], tile0[:, :], tile1[36:, :]])
plotArray(combinedTile, origin="upper")
np.savetxt("mec_feedline.txt", combinedTile, fmt="%.1f")
print "saved!"
def checkTile(tile,title=''):
nRow,nCol= np.shape(tile)
freqList = np.sort(np.reshape(tile,(-1,)))
nearestDist = neighborDistClass(tile)
minDists = ndimage.generic_filter(tile, nearestDist.minFilter, footprint=footprint,mode='constant',cval=np.inf)
nearestDist = neighborDistClass(tile)
minDists2 = ndimage.generic_filter(tile, nearestDist.minFilter, footprint=secondNeighborFootprint,mode='constant',cval=np.inf)
nearestDist = neighborDistClass(tile)
minDistsWrap = ndimage.generic_filter(tile, nearestDist.minFilter, footprint=sideFootprint,mode='wrap',cval=np.inf)
nearestDist = neighborDistClass(tile)
maxDists = ndimage.generic_filter(tile, nearestDist.maxFilter, footprint=footprint,mode='reflect')
plotArray(title=title,image=tile,normNSigma=2.,origin='upper')
plotArray(title='{} min dists wrap'.format(title),image=minDistsWrap,normNSigma=2.,origin='upper')
plotArray(title='{} min dists'.format(title),image=minDists,normNSigma=2.,origin='upper')
plotArray(title='{} min dists 2nd nearest'.format(title),image=minDists2,normNSigma=2.,origin='upper')
plotArray(title='{} max dists'.format(title),image=maxDists,normNSigma=2.,origin='upper')
# def f(thing):
# thing.axes.hist(minDists.ravel(),bins=100)
# thing.axes.set_title('{} min dists'.format(title))
#
# pop = PopUp(plotFunc=f)
def f(thing):
thing.axes.hist(minDists2.ravel(),bins=100)
thing.axes.set_title('{} second neighbor min dists'.format(title))
pop = PopUp(plotFunc=f)
def simpleGradientSolution(self):
self.makeFreqList()
idxStep = self.nCols
# idxStep = int(np.floor(2.5*idxStep))+1
# idxStep = 69 # min 67 MHz, max 700 MHz
# idxStep = 87 # min 84 MHz, max 526 MHz
idxList = np.arange(self.nFreqs)
modIdxList = (idxList * idxStep) % (self.nFreqs) + ((idxList * idxStep) // self.nFreqs)
print "idxStep", idxStep
print "nFreqs", self.nFreqs
diffIdxList = np.abs(np.diff(modIdxList))
idxTable = np.reshape(modIdxList, (self.nCols, self.nRows)).T
# do some flips so neighboring columns aren't next to others with close values
nRowFlip = 4
for iRowGroup in xrange(int(np.ceil(1.0 * self.nRows / nRowFlip))):
slc = slice(iRowGroup * nRowFlip, (iRowGroup + 1) * nRowFlip)
idxTable[slc, 1::2] = idxTable[slc, 1::2][::-1, :]
nRowFlip = 8
for iRowGroup in xrange(int(np.ceil(1.0 * self.nRows / nRowFlip))):
slc = slice(iRowGroup * nRowFlip, (iRowGroup + 1) * nRowFlip)
idxTable[slc, 1::2] = idxTable[slc, 1::2][::-1, :]
# move cols around so 5 cols follows a pattern
# idxTable2 = np.array(idxTable)
# nColGroup = 5
# for i in range(self.nCols/nColGroup):
# idxTable[:,i::nColGroup] = idxTable2[:,i*nColGroup:(i+1)*nColGroup]
# move the lowest frequency rows to the center and alternate placing higher frequency rows outward from center
# idxTable2 = np.array(idxTable)
# idxTable[0:self.nRows/2,:] = idxTable2[::-2]
# idxTable[self.nRows/2:,:] = idxTable2[::2]
plotArray(idxTable)
# make sure all the frequencies appear in the table exactly once
assert np.all(np.sort(np.ravel(idxTable)) == np.arange(self.nFreqs))
self.freqTable = self.freqList[idxTable]
np.savetxt("idx.txt", idxTable)
flatImg.mask = clipMask
illumImg.mask = clipMask
print 'removed ',np.sum(clipMask)-np.sum(nanMask),'pixels'
rawList = rawImg[~rawImg.mask]
flatList = flatImg[~flatImg.mask]
illumList = illumImg[~illumImg.mask]
#plotArray(title='with flatcal',image=flatImg)
print 'raw count',len(rawList)
print 'flat count',len(flatList)
print 'illum count',len(illumList)
plotArray(rawImg,title='raw')
plotArray(illumImg,title='illumination corrected')
plotArray(flatImg,title='flatfield corrected')
binWidth = 20 #counts
nBins = int(1.*len(flatList)/binWidth)
rawHist,rawHistEdges = np.histogram(rawList,bins=nBins)
flatHist,flatHistEdges = np.histogram(flatList,bins=rawHistEdges)
illumHist,illumHistEdges = np.histogram(illumList,bins=rawHistEdges)
rawFwhm = peakWidth(rawHistEdges[0:-1],rawHist)
flatFwhm = peakWidth(flatHistEdges[0:-1],flatHist)
illumFwhm = peakWidth(illumHistEdges[0:-1],illumHist)
avgCounts = int(flatFwhm['peakX'])
rawFwhmPercent = 100.*rawFwhm['sigma']/avgCounts
def simpleGradientSolution2(self):
self.makeFreqList()
idxStep = self.nCols
# idxStep = int(np.floor(2.5*idxStep))+1
# idxStep = 69 # min 67 MHz, max 700 MHz
# idxStep = 87 # min 84 MHz, max 526 MHz
idxList = np.arange(self.nFreqs)
modIdxList = (idxList * idxStep) % (self.nFreqs) + ((idxList * idxStep) // self.nFreqs)
print "idxStep", idxStep
print "nFreqs", self.nFreqs
diffIdxList = np.abs(np.diff(modIdxList))
idxTable = np.reshape(modIdxList, (self.nCols, self.nRows)).T
# do some flips so neighboring columns aren't next to others with close values
nRowFlip = 4
for iRowGroup in xrange(int(np.ceil(1.0 * self.nRows / nRowFlip))):
slc = slice(iRowGroup * nRowFlip, (iRowGroup + 1) * nRowFlip)
idxTable[slc, 1::2] = idxTable[slc, 1::2][::-1, :]
nRowFlip = 8
for iRowGroup in xrange(int(np.ceil(1.0 * self.nRows / nRowFlip))):
slc = slice(iRowGroup * nRowFlip, (iRowGroup + 1) * nRowFlip)
idxTable[slc, 1::2] = idxTable[slc, 1::2][::-1, :]
# adjust last row
idxTable2 = np.array(idxTable)
idxTable[-1:, 1::2] = idxTable2[-10:-9, 1::2]
idxTable[-10:-9, 1::2] = idxTable2[-1:, 1::2]
idxTable[0, :] = idxTable2[8, :]
idxTable[8, :] = idxTable2[0, :]
# once more but shift down 1 row to make sure the last row gets a flip
# nRowFlip = 15
# for iRowGroup in xrange(int(np.ceil(1.*self.nRows/nRowFlip))):
# slc = slice(1+iRowGroup*nRowFlip,1+(iRowGroup+1)*nRowFlip)
# idxTable[slc,1::2] = idxTable[slc,1::2][::-1,:]
# reverse order of odd cols
# idxTable[:,:] = idxTable[:,::-1]
# regroup cols
# idxTable2 = np.array(idxTable)
# nColGroup = 2 #nCols should be divisible by this
# nColInterleaves = self.nCols/nColGroup
# for i in range(nColInterleaves):
# idxTable[:,i::nColInterleaves] = idxTable2[:,i*nColGroup:(i+1)*nColGroup]
# move the lowest frequency rows to the center and alternate placing higher frequency rows outward from center
idxTable2 = np.array(idxTable)
idxTable[0 : self.nRows / 2, :] = idxTable2[self.nRows % 2 :: 2, :][::-1, :]
idxTable[self.nRows / 2 :, :] = idxTable2[::2]
plotArray(idxTable, origin="upper", cmap="hot")
# make sure all the frequencies appear in the table exactly once
assert np.all(np.sort(np.ravel(idxTable)) == np.arange(self.nFreqs))
self.freqTable = self.freqList[idxTable]
np.savetxt("idx.txt", idxTable)
def calculateWeights(self):
"""
finds flat cal factors as medians/pixelSpectra for each pixel
"""
cubeWeightsList = []
self.averageSpectra = []
deltaWeightsList = []
for iCube,cube in enumerate(self.spectralCubes):
effIntTime = self.cubeEffIntTimes[iCube]
#for each time chunk
wvlAverages = np.zeros(self.nWvlBins)
spectra2d = np.reshape(cube,[self.nRow*self.nCol,self.nWvlBins ])
for iWvl in xrange(self.nWvlBins):
wvlSlice = spectra2d[:,iWvl]
goodPixelWvlSlice = np.array(wvlSlice[wvlSlice != 0])#dead pixels need to be taken out before calculating averages
nGoodPixels = len(goodPixelWvlSlice)
#goodPixelWvlSlice = np.sort(goodPixelWvlSlice)
#trimmedSpectrum = goodPixelWvlSlice[self.fractionOfPixelsToTrim*nGoodPixels:(1-self.fractionOfPixelsToTrim)*nGoodPixels]
#trimmedPixelWeights = 1/np.sqrt(trimmedSpectrum)
#histGood,binEdges = np.histogram(self.intTime*goodSpectrum,bins=nBins)
#histTrim,binEdges = np.histogram(self.intTime*trimmedSpectrum,bins=binEdges)
# plt.plot(binEdges[0:-1],histGood)
# def f(fig,axes):
# axes.plot(binEdges[0:-1],histGood)
# axes.plot(binEdges[0:-1],histTrim)
# pop(plotFunc=f)
# plt.plot(binEdges[0:-1],histTrim)
wvlAverages[iWvl] = np.median(goodPixelWvlSlice)
# plt.show()
weights = np.divide(wvlAverages,cube)
weights[weights==0] = np.nan
weights[weights==np.inf] = np.nan
cubeWeightsList.append(weights)
#Now to get uncertainty in weight:
#Assuming negligible uncertainty in medians compared to single pixel spectra,
#then deltaWeight=weight*deltaSpectrum/Spectrum
#deltaWeight=weight*deltaRawCounts/RawCounts
# with deltaRawCounts=sqrt(RawCounts)#Assuming Poisson noise
#deltaWeight=weight/sqrt(RawCounts)
# but 'cube' is in units cps, not raw counts
# so multiply by effIntTime before sqrt
deltaWeights = weights/np.sqrt(effIntTime*cube)
deltaWeightsList.append(deltaWeights)
self.averageSpectra.append(wvlAverages)
cubeWeights = np.array(cubeWeightsList)
deltaCubeWeights = np.array(deltaWeightsList)
cubeWeightsMask = np.isnan(cubeWeights)
self.maskedCubeWeights = np.ma.array(cubeWeights,mask=cubeWeightsMask,fill_value=1.)
self.maskedCubeDeltaWeights = np.ma.array(deltaCubeWeights,mask=cubeWeightsMask)
#sort maskedCubeWeights and rearange spectral cubes the same way
sortedIndices = np.ma.argsort(self.maskedCubeWeights,axis=0)
identityIndices = np.ma.indices(np.shape(self.maskedCubeWeights))
sortedWeights = self.maskedCubeWeights[sortedIndices,identityIndices[1],identityIndices[2],identityIndices[3]]
countCubesReordered = self.countCubes[sortedIndices,identityIndices[1],identityIndices[2],identityIndices[3]]
cubeDeltaWeightsReordered = self.maskedCubeDeltaWeights[sortedIndices,identityIndices[1],identityIndices[2],identityIndices[3]]
#trim the beginning and end off the sorted weights for each wvl for each pixel, to exclude extremes from averages
nCubes = np.shape(self.maskedCubeWeights)[0]
trimmedWeights = sortedWeights[self.fractionOfChunksToTrim*nCubes:(1-self.fractionOfChunksToTrim)*nCubes,:,:,:]
trimmedCountCubesReordered = countCubesReordered[self.fractionOfChunksToTrim*nCubes:(1-self.fractionOfChunksToTrim)*nCubes,:,:,:]
print 'trimmed cubes shape',np.shape(trimmedCountCubesReordered)
self.totalCube = np.ma.sum(trimmedCountCubesReordered,axis=0)
self.totalFrame = np.ma.sum(self.totalCube,axis=-1)
plotArray(self.totalFrame)
trimmedCubeDeltaWeightsReordered = cubeDeltaWeightsReordered[self.fractionOfChunksToTrim*nCubes:(1-self.fractionOfChunksToTrim)*nCubes,:,:,:]
self.flatWeights,summedAveragingWeights = np.ma.average(trimmedWeights,axis=0,weights=trimmedCubeDeltaWeightsReordered**-2.,returned=True)
self.deltaFlatWeights = np.sqrt(summedAveragingWeights**-1.)#Uncertainty in weighted average is sqrt(1/sum(averagingWeights))
self.flatFlags = self.flatWeights.mask
#normalize weights at each wavelength bin
wvlWeightMedians = np.ma.median(np.reshape(self.flatWeights,(-1,self.nWvlBins)),axis=0)
self.flatWeights = np.divide(self.flatWeights,wvlWeightMedians)
obsFN = FileName(run=run,date=date,tstamp=tstamp)
obsPath = obsFN.obs()
flatPath = FileName(run=run,date=date).flatSoln()
hotPath = obsFN.timeMask()
centroidPath = obsFN.centroidList()
obs = ObsFile(obsPath)
if not os.path.exists(hotPath):
hp.findHotPixels(obsPath,hotPath)
obs.loadHotPixCalFile(hotPath,switchOnMask=False)
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatPath)
obs.setWvlCutoffs(3000,8000)
centroidRa = '03:37:43.826'
centroidDec = '14:15:14.828'
haOffset = 150.
imgDict = obs.getPixelCountImage(integrationTime=60,scaleByEffInt=True)
plotArray(imgDict['image'])
xGuess = 18 #col
yGuess = 15 #row
cc.centroidCalc(obs,centroidRa,centroidDec,guessTime=300,integrationTime=30,secondMaxCountsForDisplay=2000,HA_offset=haOffset,xyapprox=[xGuess,yGuess],outputFileName=centroidPath)
iFrameList = range(len(ofs.frameObsInfos))
image_list = []
for iFrame in iFrameList:
c = data[iFrame]['cube']
c = np.sum(c, axis = 2)
t = data[iFrame]['effIntTime']
image = c/t
nanspot = np.isnan(image)
image[nanspot] = 0
image_list.append(image)
#these are all the matching stars i found in the 66 frames
#frame_list = np.array([image_list[0], image_list[10], image_list[11], image_list[18], image_list[19], image_list[28], image_list[29], image_list[34], image_list[35], image_list[36], image_list[37], image_list[42], image_list[43], image_list[65]])
#these are the frames previously used in chris's example
frame_list = np.array([image_list[0], image_list[28], image_list[29], image_list[65]])
#degPerPix, theta, raArcsecPerSec = ObsFileSeq.ObjectFinder(image_list, fd, RA, Dec)
degPerPix, theta, raArcsecPerSec = mf.ObjectFinder(frame_list, fd, RA, Dec)
ofs.setRm(degPerPix,
math.degrees((theta)),
raArcsecPerSec,
)
mosaic = ofs.makeMosaicImage(range(66))
plotArray(mosaic)
phaseFilename = os.path.join(os.path.join('/Scratch/filterData/',date),'ch_snap_r{}p{}_{}secs.dat'.format(roachNum,pixelNum,nSecs))
dataFile = open(phaseFilename,'r')
data = dataFile.read()
nQdrWords=2**19 #for firmware with qdr longsnap
nBytesPerWord=4 #bytes in each qdr row/word
nBytesPerSample=2 #bytes in a fix16_13 phase sample
nSamples = nQdrWords*(nBytesPerWord/nBytesPerSample)*nSecs
intValues = struct.unpack('>%dh'%(nSamples),data)
phaseValuesRad = np.array(intValues,dtype=np.float32)/2**13 #move binary point
phaseValuesDeg = 180./np.pi*phaseValuesRad
covDict = covFromData(phaseValuesDeg,size=800,nTrials=None)
covMatrix = covDict['covMatrix']
covMatrixInv = covDict['covMatrixInv']
print 'cov done'
plotArray(covMatrix,title='cov',origin='upper')
plotArray(covMatrixInv,title='cov^{-1}',origin='upper')
maxIndex = np.argmax(template)
if filterSize < len(template):
template = template[maxIndex-5:maxIndex-5+filterSize]
matchedFilter = makeMatchedFilter(template,covMatrixInv)
superMatchedFilter = makeSuperMatchedFilter(template,covMatrixInv)
print np.shape(superMatchedFilter)
bPlotCoeffs = True
if bPlotCoeffs:
fig,ax = plt.subplots(1,1)
ax.plot(wienerFilterCoeffs,'r.-',label='wiener')
ax.plot(matchedFilter,'b.-',label='matched')
clippedBeforeImg = sigma_clip(beforeImg,sig=3)
clipMask = clippedBeforeImg.mask
beforeImg[clipMask] = 0
afterImg[clipMask] = 0
deadBeforeImg = (beforeImg == 0)
deadAfterImg = (afterImg == 0)
beforeList = beforeImg[beforeImg != 0]
afterList = afterImg[afterImg != 0]
afterImg[deadAfterImg] = np.nan
beforeImg[deadBeforeImg] = np.nan
plotArray(title='without flatcal',image=beforeImg)
#plotArray(title='with flatcal',image=afterImg)
def plotFunc(fig,axes):
axes.plot(beforeHistEdges[0:-1],beforeHist,label='without flatcal')
axes.plot(afterHistEdges[0:-1],afterHist,label='with flatcal')
axes.set_title('Distribution of pixel counts')
axes.set_xlabel('Counts')
axes.set_ylabel('Num of Pixels')
axes.legend()
#pop(plotFunc=plotFunc)
print 'before count',len(beforeList)
print 'after count',len(afterList)
# Fit a 3rd order, 2d polynomial to the non-flatcal image
def main():
#open the sky file for hr9087
run = 'PAL2012'
date = '20121210'
wvlCal = '20121211-052230'
obsTimestamp = '20121211-051650'
flatCalDate = '20121211'
flatCalTstamp = '20121212-074700'
obsFN = FileName(run=run,date=date,tstamp=obsTimestamp)
obsFileName = obsFN.obs()
timeMaskFileName = obsFN.timeMask()
wvlFileName = FileName(run=run,date=date,tstamp=wvlCal).calSoln()
flatFileName = FileName(run=run,date=flatCalDate,tstamp=flatCalTstamp).flatSoln()
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obsFileName
hp.findHotPixels(obsFileName,timeMaskFileName)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs = ObsFile(obsFileName)
obs.loadTimeAdjustmentFile(FileName(run='PAL2012').timeAdjustments())
obs.loadWvlCalFile(wvlFileName)
obs.loadFlatCalFile(flatFileName)
obs.loadHotPixCalFile(timeMaskFileName)
#obs.setWvlCutoffs(4000,8000)
#get image before and after flat cal
print 'getting images'
beforeImgDict = obs.getPixelCountImage(weighted=False,fluxWeighted=False,scaleByEffInt=True)
rawCubeDict = obs.getSpectralCube(weighted=False)
rawCube = np.array(rawCubeDict['cube'],dtype=np.double)
effIntTime = rawCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
rawCube *= maxIntTime / effIntTime3d
rawCube[np.isnan(rawCube)] = 0
rawCube[rawCube == np.inf] = 0
beforeImg = np.sum(rawCube,axis=-1)
print 'finished first cube'
flatCubeDict = obs.getSpectralCube(weighted=True)
flatCube = np.array(flatCubeDict['cube'],dtype=np.double)
effIntTime = flatCubeDict['effIntTime']
maxIntTime = np.max(effIntTime)
#add third dimension for broadcasting
effIntTime3d = np.reshape(effIntTime,np.shape(effIntTime)+(1,))
flatCube *= maxIntTime / effIntTime3d
flatCube[np.isnan(flatCube)] = 0
flatCube[flatCube == np.inf] = 0
afterImg = np.sum(flatCube,axis=-1)
plotArray(title='before flatcal',image=beforeImg)
plotArray(title='after flatcal',image=afterImg)
print 'before sdev',np.std(beforeImg[afterImg!=0])
print 'after sdev',np.std(afterImg[afterImg!=0])
np.savez('flatCubeGem.npz',beforeImg=beforeImg,afterImg=afterImg,rawCube=rawCube,flatCube=flatCube)