def testWeightedStats(self):
"""Test that bug from #1697 (weighted stats returning NaN) stays fixed."""
rand = afwMath.Random()
mu = 10000
edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE")
badPixelMask = afwImage.MaskU.getPlaneBitMask("EDGE")
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(3.0)
statsCtrl.setNumIter(2)
statsCtrl.setAndMask(badPixelMask)
for weight in (300.0, 10.0, 1.0):
print "Testing with weight=%0.1f" % (weight,)
maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack
weightList = []
nx, ny = 256, 256
for i in range(3):
print "Processing ", i
maskedImage = afwImage.MaskedImageF(nx, ny)
maskedImageList.append(maskedImage)
afwMath.randomPoissonImage(maskedImage.getImage(), rand, mu)
maskedImage.getVariance().set(mu)
weightList.append(weight)
self.reportBadPixels(maskedImage, badPixelMask)
print "Stack: ",
coaddMaskedImage = afwMath.statisticsStack(
maskedImageList, afwMath.MEANCLIP, statsCtrl, weightList)
self.reportBadPixels(coaddMaskedImage, badPixelMask)
def stitchExposureStatisticsStack(destWcs, destBBox, expoList, warper):
# Loosely based on pipe_tasks.assembleCoadd.
# The weight of all images should be the same as they all come from the same skyMap.
# Correct values for statsCtrl
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(3.0) # Believed to be ignored due to statsFlags = afw.Mean
statsCtrl.setNumIter(2) # Believed to be ignored due to statsFlags = afw.Mean
statsCtrl.setAndMask(afwImage.MaskU.getPlaneBitMask(["EDGE", "SAT"]))
statsCtrl.setNanSafe(False) # Correct value is ???
statsCtrl.setCalcErrorFromInputVariance(False) # Correct value is ???
statsFlags = afwMath.MEAN
#coaddMaskedImage = coaddExposure.getMaskedImage()
#coaddView = afwImage.MaskedImageF(coaddMaskedImage, bbox, afwImage.PARENT, False)
destExpo = afwImage.ExposureF(destBBox, destWcs)
maskedImageList = afwImage.vectorMaskedImageF()
weightList = []
for j, expo in enumerate(expoList):
warpedExposure = warper.warpExposure(
destWcs = destExpo.getWcs(),
srcExposure = expo,
maxBBox = destExpo.getBBox())
wn = "warpStatStack{}.fits".format(j)
log.info(wn)
#warpedExposure.writeFits(wn)
j += 1
maskedImage = warpedExposure.getMaskedImage()
maskedImageList.append(maskedImage)
weightList.append(1.0)
coadd = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList)
#coadd.writeFits("coaddStatStack.fits")
#coaddView <<= coadd
destMaskedImage = destExpo.getMaskedImage()
destMaskedImage <<= coadd
return destExpo
def testWeightedStats(self):
"""Test that bug from #1697 (weighted stats returning NaN) stays fixed."""
rand = afwMath.Random()
mu = 10000
edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE")
badPixelMask = afwImage.MaskU.getPlaneBitMask("EDGE")
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(3.0)
statsCtrl.setNumIter(2)
statsCtrl.setAndMask(badPixelMask)
for weight in (300.0, 10.0, 1.0):
print "Testing with weight=%0.1f" % (weight,)
maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack
weightList = []
nx, ny = 256, 256
for i in range(3):
print "Processing ", i
maskedImage = afwImage.MaskedImageF(nx, ny)
maskedImageList.append(maskedImage)
afwMath.randomPoissonImage(maskedImage.getImage(), rand, mu)
maskedImage.getVariance().set(mu)
weightList.append(weight)
self.reportBadPixels(maskedImage, badPixelMask)
print "Stack: ",
coaddMaskedImage = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP, statsCtrl, weightList)
self.reportBadPixels(coaddMaskedImage, badPixelMask)
def testRejectedMaskPropagation(self):
"""Test that we can propagate mask bits from rejected pixels, when the amount
of rejection crosses a threshold."""
rejectedBit = 1 # use this bit to determine whether to reject a pixel
propagatedBit = 2 # propagate this bit if a pixel with it set is rejected
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setMaskPropagationThreshold(propagatedBit, 0.3)
statsCtrl.setAndMask(1 << rejectedBit)
statsCtrl.setWeighted(True)
maskedImageList = afwImage.vectorMaskedImageF()
# start with 4 images with no mask bits set
partialSum = numpy.zeros((1, 4), dtype=numpy.float32)
finalImage = numpy.array([12.0, 12.0, 12.0, 12.0], dtype=numpy.float32)
for i in range(4):
mi = afwImage.MaskedImageF(4, 1)
imArr, maskArr, varArr = mi.getArrays()
imArr[:,:] = numpy.ones((1, 4), dtype=numpy.float32)
maskedImageList.append(mi)
partialSum += imArr
# add one more image with all permutations of the first two bits set in different pixels
mi = afwImage.MaskedImageF(4, 1)
imArr, maskArr, varArr = mi.getArrays()
imArr[0,:] = finalImage
maskArr[0,1] |= (1 << rejectedBit)
maskArr[0,2] |= (1 << propagatedBit)
maskArr[0,3] |= (1 << rejectedBit)
maskArr[0,3] |= (1 << propagatedBit)
maskedImageList.append(mi)
# these will always be rejected
finalImage[1] = 0.0
finalImage[3] = 0.0
# Uniform weights: we should only see pixel 2 set with propagatedBit, because it's not rejected;
# pixel 3 is rejected, but its weight (0.2) below the propagation threshold (0.3)
stack1 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [1.0, 1.0, 1.0, 1.0, 1.0])
self.assertEqual(stack1.get(0,0)[1], 0x0)
self.assertEqual(stack1.get(1,0)[1], 0x0)
self.assertEqual(stack1.get(2,0)[1], 1 << propagatedBit)
self.assertEqual(stack1.get(3,0)[1], 0x0)
self.assertClose(stack1.getImage().getArray(),
(partialSum + finalImage) / numpy.array([5.0, 4.0, 5.0, 4.0]),
rtol=1E-7)
# Give the masked image more weight: we should see pixel 2 and pixel 3 set with propagatedBit,
# pixel 2 because it's not rejected, and pixel 3 because the weight of the rejection (0.3333)
# is above the threshold (0.3)
# Note that rejectedBit is never propagated, because we didn't include it in statsCtrl (of course,
# normally the bits we'd propagate and the bits we'd reject would be the same)
stack2 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [1.0, 1.0, 1.0, 1.0, 2.0])
self.assertEqual(stack2.get(0,0)[1], 0x0)
self.assertEqual(stack2.get(1,0)[1], 0x0)
self.assertEqual(stack2.get(2,0)[1], 1 << propagatedBit)
self.assertEqual(stack2.get(3,0)[1], 1 << propagatedBit)
self.assertClose(stack2.getImage().getArray(),
(partialSum + 2*finalImage) / numpy.array([6.0, 4.0, 6.0, 4.0]),
rtol=1E-7)
def testRejectedMaskPropagation(self):
"""Test that we can propagate mask bits from rejected pixels, when the amount
of rejection crosses a threshold."""
rejectedBit = 1 # use this bit to determine whether to reject a pixel
propagatedBit = 2 # propagate this bit if a pixel with it set is rejected
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setMaskPropagationThreshold(propagatedBit, 0.3)
statsCtrl.setAndMask(1 << rejectedBit)
statsCtrl.setWeighted(True)
maskedImageList = afwImage.vectorMaskedImageF()
# start with 4 images with no mask bits set
partialSum = numpy.zeros((1, 4), dtype=numpy.float32)
finalImage = numpy.array([12.0, 12.0, 12.0, 12.0], dtype=numpy.float32)
for i in range(4):
mi = afwImage.MaskedImageF(4, 1)
imArr, maskArr, varArr = mi.getArrays()
imArr[:,:] = numpy.ones((1, 4), dtype=numpy.float32)
maskedImageList.append(mi)
partialSum += imArr
# add one more image with all permutations of the first two bits set in different pixels
mi = afwImage.MaskedImageF(4, 1)
imArr, maskArr, varArr = mi.getArrays()
imArr[0,:] = finalImage
maskArr[0,1] |= (1 << rejectedBit)
maskArr[0,2] |= (1 << propagatedBit)
maskArr[0,3] |= (1 << rejectedBit)
maskArr[0,3] |= (1 << propagatedBit)
maskedImageList.append(mi)
# these will always be rejected
finalImage[1] = 0.0
finalImage[3] = 0.0
# Uniform weights: we should only see pixel 2 set with propagatedBit, because it's not rejected;
# pixel 3 is rejected, but its weight (0.2) below the propagation threshold (0.3)
stack1 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [1.0, 1.0, 1.0, 1.0, 1.0])
self.assertEqual(stack1.get(0,0)[1], 0x0)
self.assertEqual(stack1.get(1,0)[1], 0x0)
self.assertEqual(stack1.get(2,0)[1], 1 << propagatedBit)
self.assertEqual(stack1.get(3,0)[1], 0x0)
self.assertClose(stack1.getImage().getArray(),
(partialSum + finalImage) / numpy.array([5.0, 4.0, 5.0, 4.0]),
rtol=1E-7)
# Give the masked image more weight: we should see pixel 2 and pixel 3 set with propagatedBit,
# pixel 2 because it's not rejected, and pixel 3 because the weight of the rejection (0.3333)
# is above the threshold (0.3)
# Note that rejectedBit is never propagated, because we didn't include it in statsCtrl (of course,
# normally the bits we'd propagate and the bits we'd reject would be the same)
stack2 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [1.0, 1.0, 1.0, 1.0, 2.0])
self.assertEqual(stack2.get(0,0)[1], 0x0)
self.assertEqual(stack2.get(1,0)[1], 0x0)
self.assertEqual(stack2.get(2,0)[1], 1 << propagatedBit)
self.assertEqual(stack2.get(3,0)[1], 1 << propagatedBit)
self.assertClose(stack2.getImage().getArray(),
(partialSum + 2*finalImage) / numpy.array([6.0, 4.0, 6.0, 4.0]),
rtol=1E-7)
def combine(self, target, imageList, stats):
"""!Combine multiple images
@param target Target image to receive the combined pixels
@param imageList List of input images
@param stats Statistics control
"""
images = afwImage.vectorMaskedImageF([img for img in imageList if img is not None])
afwMath.statisticsStack(target, images, self.config.combine, stats)
def combine(self, target, imageList, stats):
"""!Combine multiple images
@param target Target image to receive the combined pixels
@param imageList List of input images
@param stats Statistics control
"""
imageList = afwImage.vectorMaskedImageF([image for image in imageList if image is not None])
if False:
# In-place stacks are now supported on LSST's afw, but not yet on HSC
afwMath.statisticsStack(target, imageList, self.config.combine, stats)
else:
stack = afwMath.statisticsStack(imageList, self.config.combine, stats)
target <<= stack.getImage()
def testWeightedStack(self):
""" Test statisticsStack() function when weighting by a variance plane"""
sctrl = afwMath.StatisticsControl()
sctrl.setWeighted(True)
mimgList = afwImage.vectorMaskedImageF()
for val in self.values:
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(self.nX, self.nY))
mimg.set(val, 0x0, val)
mimgList.push_back(mimg)
mimgStack = afwMath.statisticsStack(mimgList, afwMath.MEAN, sctrl)
wvalues = [1.0/q for q in self.values]
wmean = float(len(self.values)) / reduce(lambda x, y: x + y, wvalues)
self.assertAlmostEqual(mimgStack.getImage().get(self.nX/2, self.nY/2), wmean)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
bgInfoList, altMaskList, statsFlags, statsCtrl):
"""Assemble the coadd for a sub-region
@param coaddExposure: The target image for the coadd
@param bbox: Sub-region to coadd
@param tempExpRefList: List of data reference to tempExp
@param imageScalerList: List of image scalers
@param weightList: List of weights
@param bgInfoList: List of background data from background matching
@param altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
@param statsFlags: Statistic for coadd
@param statsCtrl: Statistics control object for coadd
"""
self.log.logdebug("Computing coadd over %s" % bbox)
tempExpName = self.getTempExpDatasetName()
coaddMaskedImage = coaddExposure.getMaskedImage()
maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack
for tempExpRef, imageScaler, bgInfo, altMask in zip(tempExpRefList, imageScalerList, bgInfoList,
altMaskList):
exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
maskedImage = exposure.getMaskedImage()
if altMask:
altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
maskedImage.getMask().swap(altMaskSub)
imageScaler.scaleMaskedImage(maskedImage)
if self.config.doMatchBackgrounds and not bgInfo.isReference:
backgroundModel = bgInfo.backgroundModel
backgroundImage = backgroundModel.getImage() if \
self.matchBackgrounds.config.usePolynomial else \
backgroundModel.getImageF()
backgroundImage.setXY0(coaddMaskedImage.getXY0())
maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT, False)
var = maskedImage.getVariance()
var += (bgInfo.fitRMS)**2
maskedImageList.append(maskedImage)
with self.timer("stack"):
coaddSubregion = afwMath.statisticsStack(
maskedImageList, statsFlags, statsCtrl, weightList)
coaddMaskedImage.assign(coaddSubregion, bbox)
def testWeightedStack(self):
""" Test statisticsStack() function when weighting by a variance plane"""
sctrl = afwMath.StatisticsControl()
sctrl.setWeighted(True)
mimgList = afwImage.vectorMaskedImageF()
for val in self.values:
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(self.nX, self.nY))
mimg.set(val, 0x0, val)
mimgList.push_back(mimg)
mimgStack = afwMath.statisticsStack(mimgList, afwMath.MEAN, sctrl)
wvalues = [1.0/q for q in self.values]
wmean = float(len(self.values)) / reduce(lambda x, y: x + y, wvalues)
self.assertAlmostEqual(mimgStack.getImage().get(self.nX/2, self.nY/2), wmean)
# Test in-place stacking
afwMath.statisticsStack(mimgStack, mimgList, afwMath.MEAN, sctrl)
self.assertAlmostEqual(mimgStack.getImage().get(self.nX/2, self.nY/2), wmean)
def test2145(self):
"""The how-to-repeat from #2145"""
Size = 5
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setCalcErrorFromInputVariance(True)
maskedImageList = afwImage.vectorMaskedImageF()
weightList = []
for i in range(3):
mi = afwImage.MaskedImageF(Size, Size)
imArr, maskArr, varArr = mi.getArrays()
imArr[:] = numpy.random.normal(10, 0.1, (Size, Size))
varArr[:] = numpy.random.normal(10, 0.1, (Size, Size))
maskedImageList.append(mi)
weightList.append(1.0)
stack = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, weightList)
if False:
print "image=", stack.getImage().getArray()
print "variance=", stack.getVariance().getArray()
self.assertNotEqual(numpy.sum(stack.getVariance().getArray()), 0.0)
def test2145(self):
"""The how-to-repeat from #2145"""
Size = 5
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setCalcErrorFromInputVariance(True)
maskedImageList = afwImage.vectorMaskedImageF()
weightList = []
for i in range(3):
mi = afwImage.MaskedImageF(Size, Size)
imArr, maskArr, varArr = mi.getArrays()
imArr[:] = numpy.random.normal(10, 0.1, (Size, Size))
varArr[:] = numpy.random.normal(10, 0.1, (Size, Size))
maskedImageList.append(mi)
weightList.append(1.0)
stack = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, weightList)
if False:
print "image=", stack.getImage().getArray()
print "variance=", stack.getVariance().getArray()
self.assertNotEqual(numpy.sum(stack.getVariance().getArray()), 0.0)
def testReturnInputs(self):
""" Make sure that a single file put into the stacker is returned unscathed"""
imgList = afwImage.vectorMaskedImageF()
img = afwImage.MaskedImageF(afwGeom.Extent2I(10, 20))
for y in range(img.getHeight()):
simg = img.Factory(
img, afwGeom.Box2I(afwGeom.Point2I(0, y), afwGeom.Extent2I(img.getWidth(), 1)), afwImage.LOCAL
)
simg.set(y)
imgList.push_back(img)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
if display:
ds9.mtv(img, frame=1, title="input")
ds9.mtv(imgStack, frame=2, title="stack")
self.assertEqual(img.get(0, 0)[0], imgStack.get(0, 0)[0])
def testReturnInputs(self):
""" Make sure that a single file put into the stacker is returned unscathed"""
imgList = afwImage.vectorMaskedImageF()
img = afwImage.MaskedImageF(afwGeom.Extent2I(10, 20))
for y in range(img.getHeight()):
simg = img.Factory(
img,
afwGeom.Box2I(afwGeom.Point2I(0, y), afwGeom.Extent2I(img.getWidth(), 1)),
afwImage.LOCAL)
simg.set(y)
imgList.push_back(img)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
if display:
ds9.mtv(img, frame=1, title="input")
ds9.mtv(imgStack, frame=2, title="stack")
self.assertEqual(img.get(0, 0)[0], imgStack.get(0, 0)[0])
def testTicket1412(self):
"""Ticket 1412: ignored mask bits are propegated to output stack."""
mimg1 = afwImage.MaskedImageF(afwGeom.Extent2I(1, 1))
mimg1.set(0, 0, (1, 0x4, 1)) # set 0100
mimg2 = afwImage.MaskedImageF(afwGeom.Extent2I(1, 1))
mimg2.set(0, 0, (2, 0x3, 1)) # set 0010 and 0001
imgList = afwImage.vectorMaskedImageF()
imgList.push_back(mimg1)
imgList.push_back(mimg2)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(0x1) # andmask only 0001
# try first with no sctrl (no andmask set), should see 0x0111 for all output mask pixels
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
self.assertEqual(imgStack.get(0, 0)[1], 0x7)
# now try with sctrl (andmask = 0x0001), should see 0x0100 for all output mask pixels
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN, sctrl)
self.assertEqual(imgStack.get(0, 0)[1], 0x4)
def testTicket1412(self):
"""Ticket 1412: ignored mask bits are propegated to output stack."""
mimg1 = afwImage.MaskedImageF(afwGeom.Extent2I(1, 1))
mimg1.set(0, 0, (1, 0x4, 1)) # set 0100
mimg2 = afwImage.MaskedImageF(afwGeom.Extent2I(1, 1))
mimg2.set(0, 0, (2, 0x3, 1)) # set 0010 and 0001
imgList = afwImage.vectorMaskedImageF()
imgList.push_back(mimg1)
imgList.push_back(mimg2)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(0x1) # andmask only 0001
# try first with no sctrl (no andmask set), should see 0x0111 for all output mask pixels
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
self.assertEqual(imgStack.get(0, 0)[1], 0x7)
# now try with sctrl (andmask = 0x0001), should see 0x0100 for all output mask pixels
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN, sctrl)
self.assertEqual(imgStack.get(0, 0)[1], 0x4)
def testStackBadPixels(self):
"""Check that we properly ignore masked pixels, and set noGoodPixelsMask where there are
no good pixels"""
mimgVec = afwImage.vectorMaskedImageF()
DETECTED = afwImage.MaskU_getPlaneBitMask("DETECTED")
EDGE = afwImage.MaskU_getPlaneBitMask("EDGE")
INTRP = afwImage.MaskU_getPlaneBitMask("INTRP")
SAT = afwImage.MaskU_getPlaneBitMask("SAT")
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
sctrl.setAndMask(INTRP | SAT)
sctrl.setNoGoodPixelsMask(EDGE)
edgeBBox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(
20, 20)) # set these pixels to EDGE
width, height = 512, 512
dim = afwGeom.Extent2I(width, height)
val, maskVal = 10, DETECTED
for i in range(4):
mimg = afwImage.MaskedImageF(dim)
mimg.set(val, maskVal, 1)
#
# Set part of the image to NaN (with the INTRP bit set)
#
llc = afwGeom.Point2I(width // 2 * (i // 2), height // 2 * (i % 2))
bbox = afwGeom.Box2I(llc, dim // 2)
smimg = mimg.Factory(mimg, bbox, afwImage.LOCAL)
del smimg
#
# And the bottom corner to SAT
#
smask = mimg.getMask().Factory(mimg.getMask(), edgeBBox,
afwImage.LOCAL)
smask |= SAT
del smask
mimgVec.push_back(mimg)
if display > 1:
ds9.mtv(mimg, frame=i, title=str(i))
mimgStack = afwMath.statisticsStack(mimgVec, afwMath.MEAN, sctrl)
if display:
i += 1
ds9.mtv(mimgStack, frame=i, title="Stack")
i += 1
ds9.mtv(mimgStack.getVariance(), frame=i, title="var(Stack)")
#
# Check the output, ignoring EDGE pixels
#
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(afwImage.MaskU_getPlaneBitMask("EDGE"))
stats = afwMath.makeStatistics(mimgStack, afwMath.MIN | afwMath.MAX,
sctrl)
self.assertEqual(stats.getValue(afwMath.MIN), val)
self.assertEqual(stats.getValue(afwMath.MAX), val)
#
# We have to clear EDGE in the known bad corner to check the mask
#
smask = mimgStack.getMask().Factory(mimgStack.getMask(), edgeBBox,
afwImage.LOCAL)
self.assertEqual(smask.get(edgeBBox.getMinX(), edgeBBox.getMinY()),
EDGE)
smask &= ~EDGE
del smask
self.assertEqual(
afwMath.makeStatistics(mimgStack.getMask(), afwMath.SUM,
sctrl).getValue(), maskVal)
def run(self, sensorRefList, calibType):
"""Process a calibration frame.
@param sensorRef: sensor-level butler data reference
@return pipe_base Struct containing these fields:
- masterExpList: amp exposures of master calibration products
"""
referenceAmps = sensorRefList[0].subItems(level="channel")
masterExpList = []
dataIdList = []
expmeta = None
for amp in referenceAmps:
if amp.dataId['snap'] == 1:
continue
self.log.info("Amp: Processing %s", amp.dataId)
print "dataid %s" % (amp.dataId)
butler = amp.butlerSubset.butler
ampMIList = afwImage.vectorMaskedImageF()
for sRef in sensorRefList:
self.log.info("Sensor: Processing %s", sRef.dataId)
ampSnapMIList = afwImage.vectorMaskedImageF()
dataId = eval(amp.dataId.__repr__())
dataId['visit'] = sRef.dataId['visit']
for snap in (0, 1):
dataId['snap'] = snap
ampExposure = sRef.butlerSubset.butler.get('raw', dataId)
if expmeta is None:
expmeta = ampExposure.getMetadata()
expfilter = ampExposure.getFilter()
expcalib = ampExposure.getCalib()
ampDetector = cameraGeom.cast_Amp(
ampExposure.getDetector())
ampExposure = self.convertIntToFloat(ampExposure)
ampExpDataView = ampExposure.Factory(
ampExposure, ampDetector.getDiskDataSec())
self.saturationDetection(ampExposure, ampDetector)
self.overscanCorrection(ampExposure, ampDetector)
if calibType in ('flat', 'dark'):
self.biasCorrection(ampExpDataView, amp)
if False:
self.darkCorrection(ampExpDataView, amp)
self.updateVariance(ampExpDataView, ampDetector)
ampSnapMIList.append(ampExpDataView.getMaskedImage())
ampMIList.append(self.combineMIList(ampSnapMIList))
masterFrame = self.combineMIList(ampMIList)
# Fix saturation too???
self.fixDefectsAndSat(masterFrame, ampDetector)
exp = afwImage.ExposureF(masterFrame)
self.copyMetadata(exp, expmeta, calibType)
exp.setDetector(ampDetector)
exp.setWcs(afwImage.Wcs())
exp.setCalib(expcalib)
if calibType is 'flat':
exp.setFilter(expfilter)
if self.config.doWrite and calibType is not 'flat':
print "writing file %s" % dataId
butler.put(exp, calibType, dataId=amp.dataId)
masterExpList.append(exp)
dataIdList.append(amp.dataId)
if self.config.doWrite and calibType is 'flat':
self.normChipAmps(masterExpList)
for exp, dataId in zip(masterExpList, dataIdList):
print "writing flat file %s" % dataId
butler.put(exp, calibType, dataId)
return pipeBase.Struct(masterFrameList=masterExpList, )
def combine(self, identList, butler, expScales=None, backgrounds=None):
"""Combine multiple exposures for a single component
@param identList List of data identifiers
@param butler Data butler
@param expScales Scales to apply for each exposure, or None
@param bgList List of background models
@return Combined image
"""
numRows = self.config['combine'][
'rows'] # Number of rows to combine at once
assert identList, "ident not provided"
assert butler, "butler not provided"
if expScales is not None:
assert len(expScales) == len(identList), \
"Lengths of inputs (%d) and scales (%d) differ" % (len(expScales), len(identList))
imageDim = None # Size of image
for i in identList:
exp = butler.get('calexp', i)
if imageDim is None:
imageDim = exp.getDimensions()
elif imageDim != exp.getDimensions():
raise RuntimeError("Dimensions don't match: %s != %s" %
(exp.getDimensions(), imageDim))
del exp
width, height = imageDim
master = afwImage.MaskedImageF(imageDim)
maskVal = ~0x0 # Mask everything, because even objects are bad
stats = afwMath.StatisticsControl()
stats.setAndMask(maskVal)
self.log.log(
self.log.INFO, "Combining image %dx%d in chunks of %d rows" %
(width, height, numRows))
for start in range(0, height, numRows):
# Read the row of interest
combine = afwImage.vectorMaskedImageF()
stop = min(start + numRows, height)
rows = stop - start
box = afwGeom.Box2I(afwGeom.Point2I(0, start),
afwGeom.Extent2I(width, rows))
for index, id in enumerate(identList):
data = afwImage.MaskedImageF(box.getDimensions())
exp = butler.get('calexp', id)
image = exp.getMaskedImage()
data <<= afwImage.MaskedImageF(image, box, afwImage.LOCAL)
# XXX This is a little sleazy, assuming the fringes aren't varying.
# What we really should do is remove the background and scale by the fringe amplitude
if self.config['do'][
'scale'] == "FRINGE" and backgrounds is not None:
bg = backgrounds[index]
if isinstance(bg, afwMath.mathLib.Background):
bg = afwImage.ImageF(bg.getImageF(), box,
afwImage.LOCAL)
data -= bg
if expScales is not None:
data /= expScales[index]
combine.push_back(data)
del exp
#gc.collect()
# Combine the inputs
data = afwMath.statisticsStack(combine, afwMath.MEANCLIP, stats)
masterChunk = afwImage.MaskedImageF(master, box, afwImage.LOCAL)
masterChunk <<= data
del data
del masterChunk
del box
del combine
#gc.collect()
self.log.log(self.log.DEBUG,
"Combined from %d --> %d" % (start, stop))
# Scale image appropriately
stats = afwMath.makeStatistics(master, afwMath.MEDIAN,
afwMath.StatisticsControl())
median = stats.getValue(afwMath.MEDIAN)
self.log.log(self.log.INFO,
"Background of combined image: %f" % (median))
return master
if self.config.doSigmaClip:
statsFlags = afwMath.MEANCLIP
else:
statsFlags = afwMath.MEAN
coaddExposure = afwImage.ExposureF(bbox, wcs)
coaddExposure.setCalib(self.scaleZeroPoint.getCalib())
coaddMaskedImage = coaddExposure.getMaskedImage()
subregionSizeArr = self.config.subregionSize
subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
didSetMetadata = False
for subBBox in _subBBoxIter(bbox, subregionSize):
try:
self.log.info("Computing coadd %s" % (subBBox,))
coaddView = afwImage.MaskedImageF(coaddMaskedImage, subBBox, afwImage.PARENT, False)
maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack
for idx, (tempExpRef, imageScaler) in enumerate(zip(tempExpRefList,imageScalerList)):
exposure = tempExpRef.get(tempExpSubName, bbox=subBBox, imageOrigin="PARENT")
maskedImage = exposure.getMaskedImage()
imageScaler.scaleMaskedImage(maskedImage)
if not didSetMetadata:
coaddExposure.setFilter(exposure.getFilter())
didSetMetadata = True
if self.config.doMatchBackgrounds and not backgroundInfoList[idx].isReference:
backgroundModel = backgroundInfoList[idx].backgroundModel
backgroundImage = backgroundModel.getImage() if \
self.matchBackgrounds.config.usePolynomial else \
backgroundModel.getImageF()
backgroundImage.setXY0(coaddMaskedImage.getXY0())
def run(self, sensorRefList, calibType):
"""Process a calibration frame.
@param sensorRef: sensor-level butler data reference
@return pipe_base Struct containing these fields:
- masterExpList: amp exposures of master calibration products
"""
referenceAmps = sensorRefList[0].subItems(level="channel")
masterExpList = []
dataIdList = []
expmeta = None
for amp in referenceAmps:
if amp.dataId['snap'] == 1:
continue
self.log.log(self.log.INFO, "Amp: Processing %s" % (amp.dataId))
print "dataid %s"%(amp.dataId)
butler = amp.butlerSubset.butler
ampMIList = afwImage.vectorMaskedImageF()
for sRef in sensorRefList:
self.log.log(self.log.INFO, "Sensor: Processing %s" % (sRef.dataId))
ampSnapMIList = afwImage.vectorMaskedImageF()
dataId = eval(amp.dataId.__repr__())
dataId['visit'] = sRef.dataId['visit']
for snap in (0,1):
dataId['snap'] = snap
ampExposure = sRef.butlerSubset.butler.get('raw', dataId)
if expmeta is None:
expmeta = ampExposure.getMetadata()
expfilter = ampExposure.getFilter()
expcalib = ampExposure.getCalib()
ampDetector = cameraGeom.cast_Amp(ampExposure.getDetector())
ampExposure = self.convertIntToFloat(ampExposure)
ampExpDataView = ampExposure.Factory(ampExposure, ampDetector.getDiskDataSec())
self.saturationDetection(ampExposure, ampDetector)
self.overscanCorrection(ampExposure, ampDetector)
if calibType in ('flat', 'dark'):
self.biasCorrection(ampExpDataView, amp)
if False:
self.darkCorrection(ampExpDataView, amp)
self.updateVariance(ampExpDataView, ampDetector)
ampSnapMIList.append(ampExpDataView.getMaskedImage())
ampMIList.append(self.combineMIList(ampSnapMIList))
masterFrame = self.combineMIList(ampMIList)
#Fix saturation too???
self.fixDefectsAndSat(masterFrame, ampDetector)
exp = afwImage.ExposureF(masterFrame)
self.copyMetadata(exp, expmeta, calibType)
exp.setDetector(ampDetector)
exp.setWcs(afwImage.Wcs())
exp.setCalib(expcalib)
if calibType is 'flat':
exp.setFilter(expfilter)
if self.config.doWrite and calibType is not 'flat':
print "writing file %s"%dataId
butler.put(exp, calibType, dataId = amp.dataId)
masterExpList.append(exp)
dataIdList.append(amp.dataId)
if self.config.doWrite and calibType is 'flat':
self.normChipAmps(masterExpList)
for exp, dataId in zip(masterExpList, dataIdList):
print "writing flat file %s"%dataId
butler.put(exp, calibType, dataId)
return pipeBase.Struct(
masterFrameList = masterExpList,
)
def testStackBadPixels(self):
"""Check that we properly ignore masked pixels, and set noGoodPixelsMask where there are
no good pixels"""
mimgVec = afwImage.vectorMaskedImageF()
DETECTED = afwImage.MaskU_getPlaneBitMask("DETECTED")
EDGE = afwImage.MaskU_getPlaneBitMask("EDGE")
INTRP = afwImage.MaskU_getPlaneBitMask("INTRP")
SAT = afwImage.MaskU_getPlaneBitMask("SAT")
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
sctrl.setAndMask(INTRP | SAT)
sctrl.setNoGoodPixelsMask(EDGE)
edgeBBox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(20, 20)) # set these pixels to EDGE
width, height = 512, 512
dim=afwGeom.Extent2I(width, height)
val, maskVal = 10, DETECTED
for i in range(4):
mimg = afwImage.MaskedImageF(dim)
mimg.set(val, maskVal, 1)
#
# Set part of the image to NaN (with the INTRP bit set)
#
llc = afwGeom.Point2I(width//2*(i//2), height//2*(i%2))
bbox = afwGeom.Box2I(llc, dim//2)
smimg = mimg.Factory(mimg, bbox, afwImage.LOCAL)
#smimg.set(numpy.nan, INTRP, numpy.nan)
del smimg
#
# And the bottom corner to SAT
#
smask = mimg.getMask().Factory(mimg.getMask(), edgeBBox, afwImage.LOCAL)
smask |= SAT
del smask
mimgVec.push_back(mimg)
if display > 1:
ds9.mtv(mimg, frame=i, title=str(i))
mimgStack = afwMath.statisticsStack(mimgVec, afwMath.MEAN, sctrl)
if display:
i += 1
ds9.mtv(mimgStack, frame=i, title="Stack")
i += 1
ds9.mtv(mimgStack.getVariance(), frame=i, title="var(Stack)")
#
# Check the output, ignoring EDGE pixels
#
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(afwImage.MaskU_getPlaneBitMask("EDGE"))
stats = afwMath.makeStatistics(mimgStack, afwMath.MIN | afwMath.MAX, sctrl)
self.assertEqual(stats.getValue(afwMath.MIN), val)
self.assertEqual(stats.getValue(afwMath.MAX), val)
#
# We have to clear EDGE in the known bad corner to check the mask
#
smask = mimgStack.getMask().Factory(mimgStack.getMask(), edgeBBox, afwImage.LOCAL)
self.assertEqual(smask.get(edgeBBox.getMinX(), edgeBBox.getMinY()), EDGE)
smask &= ~EDGE
del smask
self.assertEqual(afwMath.makeStatistics(mimgStack.getMask(), afwMath.SUM, sctrl).getValue(), maskVal)
def makeCoadd(exposurePathList, warpExposurePolicy, psfMatchPolicy,
outlierRejectionPolicy):
"""Make a coadd using psf-matching and outlierRejectionStage
Inputs:
- exposurePathList: a list of paths to calibrated science exposures
- psfMatchPolicy: policy to control psf-matching
"""
if len(exposurePathList) == 0:
print "No images specified; nothing to do"
return
# until PR 1069 is fixed one cannot actually use SimpleStageTester to process multiple files
# meanwhile just call the process method directly
# set up pipeline stages
# note: OutlierRejectionStage cannot be run with SimpleStageTester until PR 1069 is fixed.
warpExposureStage = coaddPipe.WarpExposureStage(warpExposurePolicy)
warpExposureTester = pexHarness.simpleStageTester.SimpleStageTester(
warpExposureStage)
warpExposureTester.setDebugVerbosity(Verbosity)
psfMatchStage = coaddPipe.PsfMatchStage(psfMatchPolicy)
psfMatchTester = pexHarness.simpleStageTester.SimpleStageTester(
psfMatchStage)
psfMatchTester.setDebugVerbosity(Verbosity)
outlierRejectionStage = coaddPipe.OutlierRejectionStage(
outlierRejectionPolicy)
outlierRejectionTester = pexHarness.simpleStageTester.SimpleStageTester(
outlierRejectionStage)
outlierRejectionTester.setDebugVerbosity(Verbosity)
# process exposures
referenceExposure = None
lastInd = len(exposurePathList) - 1
psfMatchedExposureList = []
for expInd, exposurePath in enumerate(exposurePathList):
isLast = (expInd == lastInd)
print "Processing exposure %d of %d: %s" % (expInd + 1, lastInd + 1,
exposurePath)
exposure = afwImage.ExposureF(exposurePath)
print "Subtract background"
subtractBackground(exposure.getMaskedImage())
clipboard = pexHarness.Clipboard.Clipboard()
# psf-match, if necessary
if not referenceExposure:
print "First exposure; simply add to coadd"
referenceExposure = exposure
psfMatchedExposureList.append(afwImage.ExposureF(exposure, BBox))
else:
clipboard.put(warpExposurePolicy.get("inputKeys.exposure"),
exposure)
clipboard.put(
warpExposurePolicy.get("inputKeys.referenceExposure"),
referenceExposure)
warpExposureTester.runWorker(clipboard)
psfMatchTester.runWorker(clipboard)
psfMatchedExposure = clipboard.get(
psfMatchPolicy.get("outputKeys.psfMatchedExposure"))
psfMatchedExposureList.append(
afwImage.ExposureF(psfMatchedExposure, BBox))
if SaveDebugImages:
exposureName = os.path.basename(exposurePath)
warpedExposure = clipboard.get(
warpExposurePolicy.get("outputKeys.warpedExposure"))
warpedExposure.writeFits("warped_%s" % (exposureName, ))
psfMatchedExposure.writeFits("psfMatched_%s" %
(exposureName, ))
clipboard = pexHarness.Clipboard.Clipboard()
psfMatchedMaskedImageList = afwImage.vectorMaskedImageF(
[e.getMaskedImage() for e in psfMatchedExposureList])
clipboard.put(outlierRejectionPolicy.get("inputKeys.maskedImageList"),
psfMatchedMaskedImageList)
outlierRejectionTester.runWorker(clipboard)
coaddMaskedImage = clipboard.get(
outlierRejectionPolicy.get("outputKeys.coadd"))
coaddExposure = afwImage.makeExposure(coaddMaskedImage,
referenceExposure.getWcs())
weightMap = clipboard.get(
outlierRejectionPolicy.get("outputKeys.weightMap"))
return (coaddExposure, weightMap)
def main():
# try out one exposure
#visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"]
visits = ["0288976", "0288935"]
ccds = []
exit(0)
for i in range(1, 61):
ccds.append(i)
filterName = 'g'
DATA_PATH = "/root/extra_home/lsst_data/"
#spathprefix = "/home/dongfang/download/lsst_data/"
spathprefix = DATA_PATH + "raw/"
#calexpsloc = "/home/dongfang/download/lsst_data/calexps/"
calexpsloc = DATA_PATH + "calexps/"
#coaddloc = "/home/dongfang/download/lsst_data/coadds/"
coaddloc = DATA_PATH + "coadds/"
#mergecoaddloc = "/home/dongfang/download/lsst_data/merge/"
mergecoaddloc = DATA_PATH + "merge/"
# Characterize Image
charImageConfig = CharacterizeImageConfig()
charImage = CharacterizeImageTask()
calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False)
calibrateTask = CalibrateTask(config=calibrateConfig)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig(
projection='STG',
decList=[-4.9325280994132905],
patchInnerDimensions=[2000, 2000],
radiusList=[4.488775723429071],
pixelScale=0.333,
rotation=0.0,
patchBorder=100,
raList=[154.10660740464786],
tractOverlap=0.0)
hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig)
tract = hits_skymap[0]
coaddTempDict = {}
calibResDict = {}
f = open("log.txt", 'wb')
start = datetime.datetime.now()
#process CCDs to create calexps.
for v in visits:
for ccd in ccds:
visit = int(v)
filename = "instcal" + v + "." + str(ccd) + ".fits"
calexpfn = calexpsloc + v + "/" + filename
source = spathprefix + v + "/" + filename
exposure = afwImg.ExposureF(source)
try:
# Characterize Image
charRes = charImage.characterize(exposure,
exposureIdInfo=None,
background=None)
except:
f.write("DFZ DEBUG at charRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
# Caliberate Image
calibRes = calibrateTask.calibrate(
charRes.exposure,
exposureIdInfo=None,
background=charRes.background,
icSourceCat=None)
except:
f.write("DFZ DEBUG at calibRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
#write out calexps
calibRes.exposure.writeFits(calexpfn)
#calbresDict.append((v,ccd),calibRes)
except:
f.write("DFZ DEBUG at calibRes.exposure: errors in visit " +
v + ", ccd " + str(ccd) + "\n")
end = datetime.datetime.now()
d = end - start
f.write("time for creating calexps: ")
f.write(str(d.total_seconds()))
f.write("\n")
#time for creating co-add tempexps.
start = datetime.datetime.now()
# map calexps to patch-ids
visit = visits[0]
ccdsPerPatch = []
for ccd in ccds:
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
exposure = afwImg.ExposureF(source)
bbox = exposure.getBBox()
wcs = exposure.getWcs()
corners = bbox.getCorners()
xIndexMax, yIndexMax = tract.findPatch(
wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex()
xIndexMin, yIndexMin = tract.findPatch(
wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex()
yy = range(yIndexMin, yIndexMax + 1)
xx = range(xIndexMin, xIndexMax + 1)
for yIdx in yy:
for xIdx in xx:
ccdsPerPatch.append((ccd, (xIdx, yIdx)))
print len(ccdsPerPatch)
#import cPickle
#cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch)
# import cPickle
# f = open("ccdsInPatch.p",'wb')
# cPickle.dump(ccdsInPatch,f)
#import cPickle
#ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb'))
df = pd.DataFrame(ccdsPerPatch)
dfgby = df.groupby(1)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
coaddTempExpDict = {}
for visit in visits:
for a in dfgby.indices:
coaddTempExpDict[a] = {}
xInd = a[0]
yInd = a[1]
skyInfo = getSkyInfo(hits_skymap, xInd, yInd)
v = int(visit)
coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(
numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"),
numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = makeCTEConfig.modelPsf.apply(
) if makeCTEConfig.doPsfMatch else None
setInputRecorder = False
for b in dfgby.get_group(a)[0].ravel():
print a
print b
if not setInputRecorder:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
try:
inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder(
v, ccdsinPatch)
except:
f.write("DFZ DEBUG at inputRecorder\n")
setInputRecorder = True
numGoodPix = 0
ccd = b
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
calExp = afwImg.ExposureF(source)
ccdId = calExp.getId()
warpedCcdExp = makeCTE.warpAndPsfMatch.run(
calExp,
modelPsf=modelPsf,
wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = calExp.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
calExp.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(),
warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask())
totGoodPix += numGoodPix
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(warpedCcdExp.getCalib())
coaddTempExp.setFilter(warpedCcdExp.getFilter())
didSetMetadata = True
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
##### End loop over ccds here:
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata:
coaddTempExp.setPsf(
modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf(
inputRecorder.coaddInputs.ccds, skyInfo.wcs))
coaddTempExpDict[a][v] = coaddTempExp
coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str(
xInd) + "_" + str(yInd) + ".fits"
coaddTempExp.writeFits(coaddfilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating co-add tempexps:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
#DFZ: stop here
exit(0)
start = datetime.datetime.now()
config = AssembleCoaddConfig()
assembleTask = AssembleCoaddTask(config=config)
mergcoadds = {}
for a in dfgby.indices:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
xInd = a[0]
yInd = a[1]
imageScalerRes = prepareInputs(coaddTempExpDict[a].values(),
coaddTempExpDict[a].keys(),
assembleTask)
mask = None
doClip = False
if mask is None:
mask = assembleTask.getBadPixelMask()
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip)
statsCtrl.setNumIter(assembleTask.config.clipIter)
statsCtrl.setAndMask(mask)
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
for plane, threshold in assembleTask.config.maskPropagationThresholds.items(
):
bit = afwImage.MaskU.getMaskPlane(plane)
statsCtrl.setMaskPropagationThreshold(bit, threshold)
if doClip:
statsFlags = afwMath.MEANCLIP
else:
statsFlags = afwMath.MEAN
coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib())
coaddExposure.getInfo().setCoaddInputs(
assembleTask.inputRecorder.makeCoaddInputs())
#remember to set metadata if you want any hope of running detection and measurement on this coadd:
#self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
#most important thing is the psf
coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter())
coaddInputs = coaddExposure.getInfo().getCoaddInputs()
for tempExp, weight in zip(coaddTempExpDict[a].values(),
imageScalerRes.weightList):
assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp,
weight)
#takes numCcds as argument
coaddInputs.ccds.reserve(ccdsinPatch)
coaddInputs.visits.reserve(len(imageScalerRes.dataIdList))
psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
coaddExposure.setPsf(psf)
maskedImageList = afwImage.vectorMaskedImageF()
coaddMaskedImage = coaddExposure.getMaskedImage()
for dataId, imageScaler, exposure in zip(
imageScalerRes.dataIdList, imageScalerRes.imageScalerList,
coaddTempExpDict[a].values()):
print dataId, imageScaler, exposure
maskedImage = exposure.getMaskedImage()
imageScaler.scaleMaskedImage(maskedImage)
maskedImageList.append(maskedImage)
maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags,
statsCtrl,
imageScalerRes.weightList)
coaddMaskedImage.assign(maskedImage, skyInfo.bbox)
coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(),
coaddMaskedImage.getVariance())
# write out Coadd!
mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits"
mergcoadds[a] = coaddExposure
coaddExposure.writeFits(mergefilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating merged co-adds:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
start = datetime.datetime.now()
config = DetectCoaddSourcesConfig()
detectCoaddSources = DetectCoaddSourcesTask(config=config)
for a in dfgby.indices:
# Detect on Coadd
exp = mergcoadds[a]
detRes = detectCoaddSources.runDetection(exp, idFactory=None)
end = datetime.datetime.now()
d = end - start
f.write("time for detecting sources:\n ")
f.write(str(d.total_seconds()))
f.close()
def mergeCoadd(a, dfgby_raveled, coaddTempExpDict_a, hits_skymap):
from lsst.pipe.tasks.assembleCoadd import AssembleCoaddTask, AssembleCoaddConfig
import lsst.afw.math as afwMath
import lsst.afw.image as afwImage
import lsst.coadd.utils as coaddUtils
import lsst.meas.algorithms as measAlg
config = AssembleCoaddConfig()
assembleTask = AssembleCoaddTask(config=config)
ccdsinPatch = len(dfgby_raveled) # len(dfgby.get_group(a)[0].ravel())
xInd = a[0]
yInd = a[1]
print xInd, yInd
if xInd == 0 and yInd == 0:
return None
imageScalerRes = prepareInputs(coaddTempExpDict_a.values(),
coaddTempExpDict_a.keys(), assembleTask)
if imageScalerRes is None:
return None
mask = assembleTask.getBadPixelMask()
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip)
statsCtrl.setNumIter(assembleTask.config.clipIter)
statsCtrl.setAndMask(mask)
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
for plane, threshold in assembleTask.config.maskPropagationThresholds.items(
):
bit = afwImage.MaskU.getMaskPlane(plane)
statsCtrl.setMaskPropagationThreshold(bit, threshold)
statsFlags = afwMath.MEAN
skyInfo = getSkyInfo(hits_skymap, xInd, yInd)
coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib())
coaddExposure.getInfo().setCoaddInputs(
assembleTask.inputRecorder.makeCoaddInputs())
# remember to set metadata if you want any hope of running detection and measurement on this coadd:
# self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
# most important thing is the psf
coaddExposure.setFilter(coaddTempExpDict_a.values()[0].getFilter())
coaddInputs = coaddExposure.getInfo().getCoaddInputs()
for tempExp, weight in zip(coaddTempExpDict_a.values(),
imageScalerRes.weightList):
assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp,
weight)
# takes numCcds as argument
coaddInputs.ccds.reserve(ccdsinPatch)
coaddInputs.visits.reserve(len(imageScalerRes.dataIdList))
psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
coaddExposure.setPsf(psf)
maskedImageList = afwImage.vectorMaskedImageF()
coaddMaskedImage = coaddExposure.getMaskedImage()
for dataId, imageScaler, exposure in zip(imageScalerRes.dataIdList,
imageScalerRes.imageScalerList,
coaddTempExpDict_a.values()):
print dataId, imageScaler, exposure
maskedImage = exposure.getMaskedImage()
imageScaler.scaleMaskedImage(maskedImage)
maskedImageList.append(maskedImage)
maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags,
statsCtrl, imageScalerRes.weightList)
coaddMaskedImage.assign(maskedImage, skyInfo.bbox)
coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(),
coaddMaskedImage.getVariance())
return coaddExposure
def outlierRejectedCoadd(idList, butler, desFwhm, coaddWcs, coaddBBox, policy):
"""PSF-match, warp and coadd images, using outlier rejection
PSF matching is to a double gaussian model with core FWHM = desFwhm
and wings of amplitude 1/10 of core and FWHM = 2.5 * core.
The size of the PSF matching kernel is the same as the size of the kernel
found in the first calibrated science exposure, since there is no benefit
to making it any other size.
PSF-matching is performed before warping so the code can use the PSF models
associated with the calibrated science exposures (without having to warp those models).
@param[in] idList: list of data identity dictionaries
@param[in] butler: data butler for input images
@param[in] desFwhm: desired PSF of coadd, but in science exposure pixels
(the coadd usually has a different scale!);
if 0 then no PSF matching is performed.
@param[in] coaddWcs: WCS for coadd
@param[in] coaddBBox: bounding box for coadd
@param[in] policy: see policy/outlierRejectedCoaddDictionary.paf
@return coaddExposure: coadd exposure
"""
if len(idList) < 1:
print "Warning: no exposures to coadd!"
sys.exit(1)
print "Coadd %s calexp" % (len(idList), )
coaddCalib, exposureMetadataList = psfMatchAndWarp(
idList=idList,
butler=butler,
desFwhm=desFwhm,
coaddWcs=coaddWcs,
coaddBBox=coaddBBox,
policy=policy,
)
edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE")
coaddPolicy = policy.getPolicy("coaddPolicy")
badPixelMask = afwImage.MaskU.getPlaneBitMask(
coaddPolicy.getArray("badMaskPlanes"))
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(3.0)
statsCtrl.setNumIter(2)
statsCtrl.setAndMask(badPixelMask)
coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs)
coaddExposure.setCalib(coaddCalib)
filterDict = {} # dict of name: Filter
for expMeta in exposureMetadataList:
filterDict.setdefault(expMeta.filter.getName(), expMeta.filter)
if len(filterDict) == 1:
coaddExposure.setFilter(filterDict.values()[0])
print "Filter=", coaddExposure.getFilter().getName()
coaddExposure.writeFits("blankCoadd.fits")
coaddMaskedImage = coaddExposure.getMaskedImage()
subregionSizeArr = policy.getArray("subregionSize")
subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
dumPS = dafBase.PropertySet()
for bbox in subBBoxIter(coaddBBox, subregionSize):
print "Computing coadd %s" % (bbox, )
coaddView = afwImage.MaskedImageF(coaddMaskedImage, bbox,
afwImage.PARENT, False)
maskedImageList = afwImage.vectorMaskedImageF(
) # [] is rejected by afwMath.statisticsStack
weightList = []
for expMeta in exposureMetadataList:
if expMeta.bbox.contains(bbox):
maskedImage = afwImage.MaskedImageF(expMeta.path, 0, dumPS,
bbox, afwImage.PARENT)
elif not bbox.overlaps(expMeta.bbox):
print "Skipping %s; no overlap" % (expMeta.path, )
continue
else:
overlapBBox = afwGeom.Box2I(expMeta.bbox)
overlapBBox.clip(bbox)
print "Processing %s; grow from %s to %s" % (expMeta.path,
overlapBBox, bbox)
maskedImage = afwImage.MaskedImageF(bbox)
maskedImage.getMask().set(edgeMask)
maskedImageView = afwImage.MaskedImageF(
maskedImage, overlapBBox, afwImage.PARENT, False)
maskedImageView <<= afwImage.MaskedImageF(
expMeta.path, 0, dumPS, overlapBBox, afwImage.PARENT)
maskedImageList.append(maskedImage)
weightList.append(expMeta.weight)
try:
coaddSubregion = afwMath.statisticsStack(maskedImageList,
afwMath.MEANCLIP,
statsCtrl, weightList)
coaddView <<= coaddSubregion
except Exception, e:
print "Outlier rejection failed; setting EDGE mask: %s" % (e, )
raise
