def process(self, clipboard):
"""Reject outliers"""
# print "***** process ******"
# print "clipboard keys:"
# for key in clipboard.getKeys():
# print "*", key
maskedImageList = self.getFromClipboard(clipboard, "maskedImageList")
print "maskedImageList =", maskedImageList
weightList = self.getFromClipboard(clipboard,
"weightList",
doRaise=False)
outlierRejectionPolicy = self.policy.get("outlierRejectionPolicy")
allowedMaskPlanes = outlierRejectionPolicy.get("allowedMaskPlanes")
badPixelMask = coaddUtils.makeBitMask(allowedMaskPlanes.split(),
doInvert=True)
statsControl = afwMath.StatisticsControl()
statsControl.setNumSigmaClip(outlierRejectionPolicy.get("numSigma"))
statsControl.setNumIter(outlierRejectionPolicy.get("numIterations"))
statsControl.setAndMask(badPixelMask)
statsControl.setNanSafe(
False) # we're using masked images, so no need for NaN detection
statsControl.setWeighted(True)
if weightList != None:
coadd = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP,
statsControl, weightList)
else:
coadd = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP,
statsControl)
self.addToClipboard(clipboard, "coadd", coadd)
def process(self, clipboard):
"""Reject outliers"""
# print "***** process ******"
# print "clipboard keys:"
# for key in clipboard.getKeys():
# print "*", key
maskedImageList = self.getFromClipboard(clipboard, "maskedImageList")
print "maskedImageList =", maskedImageList
weightList = self.getFromClipboard(clipboard, "weightList", doRaise=False)
outlierRejectionPolicy = self.policy.get("outlierRejectionPolicy")
allowedMaskPlanes = outlierRejectionPolicy.get("allowedMaskPlanes")
badPixelMask = coaddUtils.makeBitMask(allowedMaskPlanes.split(), doInvert=True)
statsControl = afwMath.StatisticsControl()
statsControl.setNumSigmaClip(outlierRejectionPolicy.get("numSigma"))
statsControl.setNumIter(outlierRejectionPolicy.get("numIterations"))
statsControl.setAndMask(badPixelMask)
statsControl.setNanSafe(False) # we're using masked images, so no need for NaN detection
statsControl.setWeighted(True)
if weightList != None:
coadd = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP, statsControl, weightList)
else:
coadd = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP, statsControl)
self.addToClipboard(clipboard, "coadd", coadd)
def testRejectedMaskPropagation(self):
"""Test that we can propagate mask bits from rejected pixels, when the amount
of rejection crosses a threshold."""
rejectedBit = 1 # use this bit to determine whether to reject a pixel
propagatedBit = 2 # propagate this bit if a pixel with it set is rejected
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setMaskPropagationThreshold(propagatedBit, 0.3)
statsCtrl.setAndMask(1 << rejectedBit)
statsCtrl.setWeighted(True)
maskedImageList = afwImage.vectorMaskedImageF()
# start with 4 images with no mask bits set
partialSum = numpy.zeros((1, 4), dtype=numpy.float32)
finalImage = numpy.array([12.0, 12.0, 12.0, 12.0], dtype=numpy.float32)
for i in range(4):
mi = afwImage.MaskedImageF(4, 1)
imArr, maskArr, varArr = mi.getArrays()
imArr[:,:] = numpy.ones((1, 4), dtype=numpy.float32)
maskedImageList.append(mi)
partialSum += imArr
# add one more image with all permutations of the first two bits set in different pixels
mi = afwImage.MaskedImageF(4, 1)
imArr, maskArr, varArr = mi.getArrays()
imArr[0,:] = finalImage
maskArr[0,1] |= (1 << rejectedBit)
maskArr[0,2] |= (1 << propagatedBit)
maskArr[0,3] |= (1 << rejectedBit)
maskArr[0,3] |= (1 << propagatedBit)
maskedImageList.append(mi)
# these will always be rejected
finalImage[1] = 0.0
finalImage[3] = 0.0
# Uniform weights: we should only see pixel 2 set with propagatedBit, because it's not rejected;
# pixel 3 is rejected, but its weight (0.2) below the propagation threshold (0.3)
stack1 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [1.0, 1.0, 1.0, 1.0, 1.0])
self.assertEqual(stack1.get(0,0)[1], 0x0)
self.assertEqual(stack1.get(1,0)[1], 0x0)
self.assertEqual(stack1.get(2,0)[1], 1 << propagatedBit)
self.assertEqual(stack1.get(3,0)[1], 0x0)
self.assertClose(stack1.getImage().getArray(),
(partialSum + finalImage) / numpy.array([5.0, 4.0, 5.0, 4.0]),
rtol=1E-7)
# Give the masked image more weight: we should see pixel 2 and pixel 3 set with propagatedBit,
# pixel 2 because it's not rejected, and pixel 3 because the weight of the rejection (0.3333)
# is above the threshold (0.3)
# Note that rejectedBit is never propagated, because we didn't include it in statsCtrl (of course,
# normally the bits we'd propagate and the bits we'd reject would be the same)
stack2 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [1.0, 1.0, 1.0, 1.0, 2.0])
self.assertEqual(stack2.get(0,0)[1], 0x0)
self.assertEqual(stack2.get(1,0)[1], 0x0)
self.assertEqual(stack2.get(2,0)[1], 1 << propagatedBit)
self.assertEqual(stack2.get(3,0)[1], 1 << propagatedBit)
self.assertClose(stack2.getImage().getArray(),
(partialSum + 2*finalImage) / numpy.array([6.0, 4.0, 6.0, 4.0]),
rtol=1E-7)
def testRejectedMaskPropagation(self):
"""Test that we can propagate mask bits from rejected pixels, when the amount
of rejection crosses a threshold."""
rejectedBit = 1 # use this bit to determine whether to reject a pixel
propagatedBit = 2 # propagate this bit if a pixel with it set is rejected
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setMaskPropagationThreshold(propagatedBit, 0.3)
statsCtrl.setAndMask(1 << rejectedBit)
statsCtrl.setWeighted(True)
maskedImageList = afwImage.vectorMaskedImageF()
# start with 4 images with no mask bits set
partialSum = numpy.zeros((1, 4), dtype=numpy.float32)
finalImage = numpy.array([12.0, 12.0, 12.0, 12.0], dtype=numpy.float32)
for i in range(4):
mi = afwImage.MaskedImageF(4, 1)
imArr, maskArr, varArr = mi.getArrays()
imArr[:,:] = numpy.ones((1, 4), dtype=numpy.float32)
maskedImageList.append(mi)
partialSum += imArr
# add one more image with all permutations of the first two bits set in different pixels
mi = afwImage.MaskedImageF(4, 1)
imArr, maskArr, varArr = mi.getArrays()
imArr[0,:] = finalImage
maskArr[0,1] |= (1 << rejectedBit)
maskArr[0,2] |= (1 << propagatedBit)
maskArr[0,3] |= (1 << rejectedBit)
maskArr[0,3] |= (1 << propagatedBit)
maskedImageList.append(mi)
# these will always be rejected
finalImage[1] = 0.0
finalImage[3] = 0.0
# Uniform weights: we should only see pixel 2 set with propagatedBit, because it's not rejected;
# pixel 3 is rejected, but its weight (0.2) below the propagation threshold (0.3)
stack1 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [1.0, 1.0, 1.0, 1.0, 1.0])
self.assertEqual(stack1.get(0,0)[1], 0x0)
self.assertEqual(stack1.get(1,0)[1], 0x0)
self.assertEqual(stack1.get(2,0)[1], 1 << propagatedBit)
self.assertEqual(stack1.get(3,0)[1], 0x0)
self.assertClose(stack1.getImage().getArray(),
(partialSum + finalImage) / numpy.array([5.0, 4.0, 5.0, 4.0]),
rtol=1E-7)
# Give the masked image more weight: we should see pixel 2 and pixel 3 set with propagatedBit,
# pixel 2 because it's not rejected, and pixel 3 because the weight of the rejection (0.3333)
# is above the threshold (0.3)
# Note that rejectedBit is never propagated, because we didn't include it in statsCtrl (of course,
# normally the bits we'd propagate and the bits we'd reject would be the same)
stack2 = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, [1.0, 1.0, 1.0, 1.0, 2.0])
self.assertEqual(stack2.get(0,0)[1], 0x0)
self.assertEqual(stack2.get(1,0)[1], 0x0)
self.assertEqual(stack2.get(2,0)[1], 1 << propagatedBit)
self.assertEqual(stack2.get(3,0)[1], 1 << propagatedBit)
self.assertClose(stack2.getImage().getArray(),
(partialSum + 2*finalImage) / numpy.array([6.0, 4.0, 6.0, 4.0]),
rtol=1E-7)
def combine(self, target, imageList, stats):
"""!Combine multiple images
@param target Target image to receive the combined pixels
@param imageList List of input images
@param stats Statistics control
"""
images = afwImage.vectorMaskedImageF([img for img in imageList if img is not None])
afwMath.statisticsStack(target, images, self.config.combine, stats)
def combine(self, target, imageList, stats):
"""!Combine multiple images
@param target Target image to receive the combined pixels
@param imageList List of input images
@param stats Statistics control
"""
images = [img for img in imageList if img is not None]
afwMath.statisticsStack(target, images,
afwMath.Property(self.config.combine), stats)
def combineMIList(self, miList, method='MEANCLIP'):
combinedFrame = miList[0].Factory()
try:
if method == 'MEANCLIP':
combinedFrame = afwMath.statisticsStack(miList, afwMath.MEANCLIP, self.statsCtrl)
elif method == 'MEDIAN':
combinedFrame = afwMath.statisticsStack(miList, afwMath.MEDIAN, self.statsCtrl)
else:
raise ValueError("Method %s is not supported for combining frames" % (method))
except Exception as e:
self.log.warn("Could not combine the frames. %s", e)
return combinedFrame
def combineMIList(self, miList, method='MEANCLIP'):
combinedFrame = miList[0].Factory()
try:
if method is 'MEANCLIP':
combinedFrame = afwMath.statisticsStack(miList, afwMath.MEANCLIP, self.statsCtrl)
elif method is 'MEDIAN':
combinedFrame = afwMath.statisticsStack(miList, afwMath.MEDIAN, self.statsCtrl)
else:
raise ValueError("Method %s is not supported for combining frames" % (method))
except Exception as e:
self.log.warn("Could not combine the frames. %s", e)
return combinedFrame
def combine(self, target, imageList, stats):
"""!Combine multiple images
@param target Target image to receive the combined pixels
@param imageList List of input images
@param stats Statistics control
"""
imageList = afwImage.vectorMaskedImageF([image for image in imageList if image is not None])
if False:
# In-place stacks are now supported on LSST's afw, but not yet on HSC
afwMath.statisticsStack(target, imageList, self.config.combine, stats)
else:
stack = afwMath.statisticsStack(imageList, self.config.combine, stats)
target <<= stack.getImage()
def testStatistics(self):
""" Test the statisticsStack() function """
imgList = afwImage.vectorImageF()
for val in self.values:
imgList.push_back(afwImage.ImageF(afwGeom.Extent2I(self.nX, self.nY), val))
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
mean = reduce(lambda x, y: x+y, self.values)/float(len(self.values))
self.assertAlmostEqual(imgStack.get(self.nX//2, self.nY//2), mean)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEDIAN)
median = sorted(self.values)[len(self.values)//2]
self.assertEqual(imgStack.get(self.nX//2, self.nY//2), median)
def testStatistics(self):
""" Test the statisticsStack() function """
imgList = afwImage.vectorImageF()
for val in self.values:
imgList.push_back(afwImage.ImageF(afwGeom.Extent2I(self.nX, self.nY), val))
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
mean = reduce(lambda x, y: x+y, self.values)/float(len(self.values))
self.assertAlmostEqual(imgStack.get(self.nX/2, self.nY/2), mean)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEDIAN)
median = sorted(self.values)[len(self.values)//2]
self.assertEqual(imgStack.get(self.nX/2, self.nY/2), median)
def testStatistics(self):
""" Test the statisticsStack() function """
imgList = []
for val in self.values:
imgList.append(afwImage.ImageF(
lsst.geom.Extent2I(self.nX, self.nY), val))
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
mean = reduce(lambda x, y: x+y, self.values)/float(len(self.values))
self.assertAlmostEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], mean)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEDIAN)
median = sorted(self.values)[len(self.values)//2]
self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], median)
def combine(self, target, expList, stats):
"""Combine multiple images.
Parameters
----------
target : `lsst.afw.image.Exposure`
Output exposure to construct.
expList : `list` [`lsst.afw.image.Exposure`]
Input exposures to combine.
stats : `lsst.afw.math.StatisticsControl`
Control explaining how to combine the input images.
"""
images = [img.getMaskedImage() for img in expList if img is not None]
afwMath.statisticsStack(target, images,
afwMath.Property(self.config.combine), stats)
def testStatistics(self):
""" Test the statisticsStack() function """
imgList = []
for val in self.values:
imgList.append(afwImage.ImageF(
lsst.geom.Extent2I(self.nX, self.nY), val))
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
mean = reduce(lambda x, y: x+y, self.values)/float(len(self.values))
self.assertAlmostEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], mean)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEDIAN)
median = sorted(self.values)[len(self.values)//2]
self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], median)
def run(self, bss_ref_list, region_name=None):
"""Read input bright star stamps and stack them together.
The stacking is done iteratively over smaller areas of the final model
image to allow for a great number of bright star stamps to be used.
Parameters
----------
bss_ref_list : `list` of
`lsst.daf.butler._deferredDatasetHandle.DeferredDatasetHandle`
List of available bright star stamps data references.
region_name : `str`, optional
Name of the focal plane region, if applicable. Only used for
logging purposes, when running over multiple such regions
(typically from `MeasureExtendedPsfTask`)
"""
if region_name:
region_message = f' for region "{region_name}".'
else:
region_message = ''
self.log.info(
'Building extended PSF from stamps extracted from %d detector images%s',
len(bss_ref_list), region_message)
# read in example set of full stamps
example_bss = bss_ref_list[0].get(datasetType="brightStarStamps",
immediate=True)
example_stamp = example_bss[0].stamp_im
# create model image
ext_psf = afwImage.MaskedImageF(example_stamp.getBBox())
# divide model image into smaller subregions
subregion_size = Extent2I(*self.config.subregion_size)
sub_bboxes = AssembleCoaddTask._subBBoxIter(ext_psf.getBBox(),
subregion_size)
# compute approximate number of subregions
n_subregions = int(ext_psf.getDimensions()[0] / subregion_size[0] +
1) * int(ext_psf.getDimensions()[1] /
subregion_size[1] + 1)
self.log.info(
"Stacking will performed iteratively over approximately %d "
"smaller areas of the final model image.", n_subregions)
# set up stacking statistic
stats_control, stats_flags = self._set_up_stacking(example_stamp)
# perform stacking
for jbbox, bbox in enumerate(sub_bboxes):
all_stars = None
for bss_ref in bss_ref_list:
read_stars = bss_ref.get(datasetType="brightStarStamps",
parameters={'bbox': bbox})
if self.config.do_mag_cut:
read_stars = read_stars.selectByMag(
magMax=self.config.mag_limit)
if all_stars:
all_stars.extend(read_stars)
else:
all_stars = read_stars
# TODO: DM-27371 add weights to bright stars for stacking
coadd_sub_bbox = afwMath.statisticsStack(
all_stars.getMaskedImages(), stats_flags, stats_control)
ext_psf.assign(coadd_sub_bbox, bbox)
return ext_psf
def superflat(files,
bias_frame=None,
outfile='superflat.fits',
bitpix=-32,
bias_subtract=True):
"""
The superflat is created by bias-offset correcting the input files
and median-ing them together.
"""
# Get overscan region.
overscan = makeAmplifierGeometry(files[0]).serial_overscan
# Use the first file as a template for the fits output.
output = fits.open(files[0])
for amp in imutils.allAmps(files[0]):
images = afwImage.vectorImageF()
for infile in files:
image = afwImage.ImageF(infile, imutils.dm_hdu(amp))
if bias_subtract:
if bias_frame:
bias_image = afwImage.ImageF(bias_frame,
imutils.dm_hdu(amp))
image = bias_subtracted_image(image, bias_image, overscan)
else:
image -= imutils.bias_image(image,
overscan,
statistic=np.median)
images.push_back(image)
median_image = afwMath.statisticsStack(images, afwMath.MEDIAN)
output[amp].data = median_image.getArray()
if bitpix is not None:
imutils.set_bitpix(output[amp], bitpix)
fitsWriteto(output, outfile, clobber=True)
return outfile
def testMean(self):
""" Test the statisticsStack() function for a MEAN"""
knownMean = 0.0
imgList = afwImage.vectorImageF()
for iImg in range(self.nImg):
imgList.push_back(afwImage.ImageF(afwGeom.Extent2I(self.nX, self.nY), iImg))
knownMean += iImg
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
knownMean /= self.nImg
self.assertEqual(imgStack.get(self.nX//2, self.nY//2), knownMean)
# Test in-place stacking
afwMath.statisticsStack(imgStack, imgList, afwMath.MEAN)
self.assertEqual(imgStack.get(self.nX//2, self.nY//2), knownMean)
def testWeightedStats(self):
"""Test that bug from #1697 (weighted stats returning NaN) stays fixed."""
rand = afwMath.Random()
mu = 10000
edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE")
badPixelMask = afwImage.MaskU.getPlaneBitMask("EDGE")
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(3.0)
statsCtrl.setNumIter(2)
statsCtrl.setAndMask(badPixelMask)
for weight in (300.0, 10.0, 1.0):
print "Testing with weight=%0.1f" % (weight,)
maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack
weightList = []
nx, ny = 256, 256
for i in range(3):
print "Processing ", i
maskedImage = afwImage.MaskedImageF(nx, ny)
maskedImageList.append(maskedImage)
afwMath.randomPoissonImage(maskedImage.getImage(), rand, mu)
maskedImage.getVariance().set(mu)
weightList.append(weight)
self.reportBadPixels(maskedImage, badPixelMask)
print "Stack: ",
coaddMaskedImage = afwMath.statisticsStack(
maskedImageList, afwMath.MEANCLIP, statsCtrl, weightList)
self.reportBadPixels(coaddMaskedImage, badPixelMask)
def superflat(files,
bias_frame=None,
outfile='superflat.fits',
bitpix=None,
bias_subtract=True,
bias_method='row'):
"""
The superflat is created by bias-offset correcting the input files
and median-ing them together.
"""
# Get overscan region.
overscan = makeAmplifierGeometry(files[0]).serial_overscan
output_images = dict()
for amp in imutils.allAmps(files[0]):
images = []
for infile in files:
image = afwImage.ImageF(infile, imutils.dm_hdu(amp))
if bias_subtract:
if bias_frame:
bias_image = afwImage.ImageF(bias_frame,
imutils.dm_hdu(amp))
image = bias_subtracted_image(image, bias_image, overscan,
bias_method)
else:
image -= imutils.bias_image(im=image,
overscan=overscan,
bias_method=bias_method)
images.append(image)
if lsst.afw.__version__.startswith('12.0'):
images = afwImage.vectorImageF(images)
output_images[amp] = afwMath.statisticsStack(images, afwMath.MEDIAN)
imutils.writeFits(output_images, outfile, files[0])
return outfile
def stitchExposureStatisticsStack(destWcs, destBBox, expoList, warper):
# Loosely based on pipe_tasks.assembleCoadd.
# The weight of all images should be the same as they all come from the same skyMap.
# Correct values for statsCtrl
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(3.0) # Believed to be ignored due to statsFlags = afw.Mean
statsCtrl.setNumIter(2) # Believed to be ignored due to statsFlags = afw.Mean
statsCtrl.setAndMask(afwImage.MaskU.getPlaneBitMask(["EDGE", "SAT"]))
statsCtrl.setNanSafe(False) # Correct value is ???
statsCtrl.setCalcErrorFromInputVariance(False) # Correct value is ???
statsFlags = afwMath.MEAN
#coaddMaskedImage = coaddExposure.getMaskedImage()
#coaddView = afwImage.MaskedImageF(coaddMaskedImage, bbox, afwImage.PARENT, False)
destExpo = afwImage.ExposureF(destBBox, destWcs)
maskedImageList = afwImage.vectorMaskedImageF()
weightList = []
for j, expo in enumerate(expoList):
warpedExposure = warper.warpExposure(
destWcs = destExpo.getWcs(),
srcExposure = expo,
maxBBox = destExpo.getBBox())
wn = "warpStatStack{}.fits".format(j)
log.info(wn)
#warpedExposure.writeFits(wn)
j += 1
maskedImage = warpedExposure.getMaskedImage()
maskedImageList.append(maskedImage)
weightList.append(1.0)
coadd = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList)
#coadd.writeFits("coaddStatStack.fits")
#coaddView <<= coadd
destMaskedImage = destExpo.getMaskedImage()
destMaskedImage <<= coadd
return destExpo
def testWeightedStats(self):
"""Test that bug from #1697 (weighted stats returning NaN) stays fixed."""
rand = afwMath.Random()
mu = 10000
edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE")
badPixelMask = afwImage.MaskU.getPlaneBitMask("EDGE")
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(3.0)
statsCtrl.setNumIter(2)
statsCtrl.setAndMask(badPixelMask)
for weight in (300.0, 10.0, 1.0):
print "Testing with weight=%0.1f" % (weight,)
maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack
weightList = []
nx, ny = 256, 256
for i in range(3):
print "Processing ", i
maskedImage = afwImage.MaskedImageF(nx, ny)
maskedImageList.append(maskedImage)
afwMath.randomPoissonImage(maskedImage.getImage(), rand, mu)
maskedImage.getVariance().set(mu)
weightList.append(weight)
self.reportBadPixels(maskedImage, badPixelMask)
print "Stack: ",
coaddMaskedImage = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP, statsCtrl, weightList)
self.reportBadPixels(coaddMaskedImage, badPixelMask)
def testMean(self):
""" Test the statisticsStack() function for a MEAN"""
knownMean = 0.0
imgList = afwImage.vectorImageF()
for iImg in range(self.nImg):
imgList.push_back(afwImage.ImageF(afwGeom.Extent2I(self.nX, self.nY), iImg))
knownMean += iImg
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
knownMean /= self.nImg
self.assertEqual(imgStack.get(self.nX/2, self.nY/2), knownMean)
# Test in-place stacking
afwMath.statisticsStack(imgStack, imgList, afwMath.MEAN)
self.assertEqual(imgStack.get(self.nX/2, self.nY/2), knownMean)
def fits_mean_file(files, outfile, overwrite=True, bitpix=32):
output = fits.HDUList()
output.append(fits.PrimaryHDU())
all_amps = allAmps()
for amp in all_amps:
images = [afwImage.ImageF(item, dm_hdu(amp)) for item in files]
if lsst.afw.__version__.startswith('12.0'):
images = afwImage.vectorImageF(images)
mean_image = afwMath.statisticsStack(images, afwMath.MEAN)
if bitpix < 0:
output.append(fits.ImageHDU(data=mean_image.getArray()))
else:
output.append(
fits.CompImageHDU(data=mean_image.getArray(),
compression_type='RICE_1'))
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=UserWarning, append=True)
warnings.filterwarnings('ignore', category=AstropyWarning, append=True)
warnings.filterwarnings('ignore',
category=AstropyUserWarning,
append=True)
with fits.open(files[0]) as template:
output[0].header.update(template[0].header)
output[0].header['FILENAME'] = os.path.basename(outfile)
for amp in all_amps:
output[amp].header.update(template[amp].header)
set_bitpix(output[amp], bitpix)
for i in (-3, -2, -1):
output.append(template[i])
fitsWriteto(output, outfile, overwrite=overwrite)
def testClipped(self):
"""Test that we set mask bits when pixels are clipped"""
box = lsst.geom.Box2I(lsst.geom.Point2I(12345, 67890), lsst.geom.Extent2I(3, 3))
num = 10
maskVal = 0xAD
value = 0.0
images = [afwImage.MaskedImageF(box) for _ in range(num)]
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(maskVal)
clipped = 1 << afwImage.Mask().addMaskPlane("CLIPPED")
# No clipping: check that vanilla is working
for img in images:
img.getImage().set(value)
img.getMask().set(0)
stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, clipped=clipped)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertFloatsAlmostEqual(stack.getMask().getArray(), 0, atol=0.0) # Not floats, but that's OK
# Clip a pixel; the CLIPPED bit should be set
images[0].getImage()[1, 1, afwImage.LOCAL] = value + 1.0
stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, clipped=clipped)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped)
# Mask a pixel; the CLIPPED bit should be set
images[0].getMask()[1, 1, afwImage.LOCAL] = maskVal
stack = afwMath.statisticsStack(images, afwMath.MEAN, statsCtrl, clipped=clipped)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped)
# Excuse that mask; the CLIPPED bit should not be set
stack = afwMath.statisticsStack(images, afwMath.MEAN, statsCtrl, clipped=clipped, excuse=maskVal)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], 0)
# Map that mask value to a different one.
rejected = 1 << afwImage.Mask().addMaskPlane("REJECTED")
maskMap = [(maskVal, rejected)]
images[0].mask[1, 1, afwImage.LOCAL] = 0 # only want to clip, not mask, this one
images[1].mask[1, 2, afwImage.LOCAL] = maskVal # only want to mask, not clip, this one
stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, statsCtrl, wvector=[], clipped=clipped,
maskMap=maskMap)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped)
self.assertEqual(stack.mask[1, 2, afwImage.LOCAL], rejected)
def testClipped(self):
"""Test that we set mask bits when pixels are clipped"""
box = lsst.geom.Box2I(lsst.geom.Point2I(12345, 67890), lsst.geom.Extent2I(3, 3))
num = 10
maskVal = 0xAD
value = 0.0
images = [afwImage.MaskedImageF(box) for _ in range(num)]
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(maskVal)
clipped = 1 << afwImage.Mask().addMaskPlane("CLIPPED")
# No clipping: check that vanilla is working
for img in images:
img.getImage().set(value)
img.getMask().set(0)
stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, clipped=clipped)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertFloatsAlmostEqual(stack.getMask().getArray(), 0, atol=0.0) # Not floats, but that's OK
# Clip a pixel; the CLIPPED bit should be set
images[0].getImage()[1, 1, afwImage.LOCAL] = value + 1.0
stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, clipped=clipped)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped)
# Mask a pixel; the CLIPPED bit should be set
images[0].getMask()[1, 1, afwImage.LOCAL] = maskVal
stack = afwMath.statisticsStack(images, afwMath.MEAN, statsCtrl, clipped=clipped)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped)
# Excuse that mask; the CLIPPED bit should not be set
stack = afwMath.statisticsStack(images, afwMath.MEAN, statsCtrl, clipped=clipped, excuse=maskVal)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], 0)
# Map that mask value to a different one.
rejected = 1 << afwImage.Mask().addMaskPlane("REJECTED")
maskMap = [(maskVal, rejected)]
images[0].mask[1, 1, afwImage.LOCAL] = 0 # only want to clip, not mask, this one
images[1].mask[1, 2, afwImage.LOCAL] = maskVal # only want to mask, not clip, this one
stack = afwMath.statisticsStack(images, afwMath.MEANCLIP, statsCtrl, wvector=[], clipped=clipped,
maskMap=maskMap)
self.assertFloatsAlmostEqual(stack.getImage().getArray(), 0.0, atol=0.0)
self.assertEqual(stack.mask[1, 1, afwImage.LOCAL], clipped)
self.assertEqual(stack.mask[1, 2, afwImage.LOCAL], rejected)
def testMean(self):
""" Test the statisticsStack() function for a MEAN"""
knownMean = 0.0
imgList = []
for iImg in range(self.nImg):
imgList.append(afwImage.ImageF(
lsst.geom.Extent2I(self.nX, self.nY), iImg))
knownMean += iImg
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
knownMean /= self.nImg
self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], knownMean)
# Test in-place stacking
afwMath.statisticsStack(imgStack, imgList, afwMath.MEAN)
self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], knownMean)
def testMean(self):
""" Test the statisticsStack() function for a MEAN"""
knownMean = 0.0
imgList = []
for iImg in range(self.nImg):
imgList.append(afwImage.ImageF(
lsst.geom.Extent2I(self.nX, self.nY), iImg))
knownMean += iImg
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
knownMean /= self.nImg
self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], knownMean)
# Test in-place stacking
afwMath.statisticsStack(imgStack, imgList, afwMath.MEAN)
self.assertEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], knownMean)
def stack(ims, statistic=afwMath.MEDIAN):
"""Stacks a list of images based on a statistic."""
images = []
for image in ims:
images.append(image)
if lsst.afw.__version__.startswith('12.0'):
images = afwImage.vectorImageF(images)
summary = afwMath.statisticsStack(images, statistic)
return summary
def testWeightedStack(self):
""" Test statisticsStack() function when weighting by a variance plane"""
sctrl = afwMath.StatisticsControl()
sctrl.setWeighted(True)
mimgList = afwImage.vectorMaskedImageF()
for val in self.values:
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(self.nX, self.nY))
mimg.set(val, 0x0, val)
mimgList.push_back(mimg)
mimgStack = afwMath.statisticsStack(mimgList, afwMath.MEAN, sctrl)
wvalues = [1.0/q for q in self.values]
wmean = float(len(self.values)) / reduce(lambda x, y: x + y, wvalues)
self.assertAlmostEqual(mimgStack.getImage().get(self.nX/2, self.nY/2), wmean)
# Test in-place stacking
afwMath.statisticsStack(mimgStack, mimgList, afwMath.MEAN, sctrl)
self.assertAlmostEqual(mimgStack.getImage().get(self.nX/2, self.nY/2), wmean)
def testWeightedStack(self):
""" Test statisticsStack() function when weighting by a variance plane"""
sctrl = afwMath.StatisticsControl()
sctrl.setWeighted(True)
mimgList = afwImage.vectorMaskedImageF()
for val in self.values:
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(self.nX, self.nY))
mimg.set(val, 0x0, val)
mimgList.push_back(mimg)
mimgStack = afwMath.statisticsStack(mimgList, afwMath.MEAN, sctrl)
wvalues = [1.0/q for q in self.values]
wmean = float(len(self.values)) / reduce(lambda x, y: x + y, wvalues)
self.assertAlmostEqual(mimgStack.getImage().get(self.nX//2, self.nY//2), wmean)
# Test in-place stacking
afwMath.statisticsStack(mimgStack, mimgList, afwMath.MEAN, sctrl)
self.assertAlmostEqual(mimgStack.getImage().get(self.nX//2, self.nY//2), wmean)
def main():
nImg = 10
nX, nY = 64, 64
imgList = afwImage.vectorImageF()
for iImg in range(nImg):
imgList.push_back(afwImage.ImageF(afwGeom.Extent2I(nX, nY), iImg))
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
print imgStack.get(nX / 2, nY / 2)
def main():
nImg = 10
nX, nY = 64, 64
imgList = afwImage.vectorImageF()
for iImg in range(nImg):
imgList.push_back(afwImage.ImageF(afwGeom.Extent2I(nX, nY), iImg))
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
print imgStack.get(nX/2, nY/2)
def setUp(self):
# fill an image with a gradient
self.n = 8
self.img = afwImage.ImageF(afwGeom.Extent2I(self.n, self.n), 0)
# these are the known answers for comparison
def nVector(n, v):
return [v for i in range(n)]
self.column = nVector(self.n, 0.0)
self.row = nVector(self.n, 0.0)
self.colPlus = nVector(self.n, 0.0)
self.colMinus = nVector(self.n, 0.0)
self.colMult = nVector(self.n, 0.0)
self.colDiv = nVector(self.n, 0.0)
self.rowPlus = nVector(self.n, 0.0)
self.rowMinus = nVector(self.n, 0.0)
self.rowMult = nVector(self.n, 0.0)
self.rowDiv = nVector(self.n, 0.0)
# set the values in the image, and keep track of the stats to verify
# things
for y in range(self.n):
for x in range(self.n):
val = 1.0 * x + 2.0 * y
self.img.set(x, y, val)
self.column[y] += val
self.row[x] += val
for i in range(self.n):
self.row[i] /= self.n
self.column[i] /= self.n
self.colPlus[i] = self.img.get(0, i) + self.column[i]
# get stats on the columns and rows
self.imgProjectCol = afwMath.statisticsStack(self.img, afwMath.MEAN,
'x')
self.imgProjectRow = afwMath.statisticsStack(self.img, afwMath.MEAN,
'y')
def test_online_coadd_input_variance_false(self):
"""Test online coaddition with calcErrorFromInputVariance=False."""
exposures, weights = self.make_test_images_to_coadd()
coadd_exposure = self.make_coadd_exposure(exposures[0])
stats_ctrl = self.make_stats_ctrl()
stats_ctrl.setCalcErrorFromInputVariance(False)
mask_map = self.make_mask_map(stats_ctrl)
stats_flags = afwMath.stringToStatisticsProperty("MEAN")
clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
masked_image_list = [exp.maskedImage for exp in exposures]
afw_masked_image = afwMath.statisticsStack(masked_image_list,
stats_flags, stats_ctrl,
weights, clipped, mask_map)
mask_threshold_dict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(
stats_ctrl)
# Make the stack with the online accumulator
# By setting no_good_pixels=None we have the same behavior
# as the default from stats_ctrl.getNoGoodPixelsMask(), but
# covers the alternate code path.
stacker = AccumulatorMeanStack(
coadd_exposure.image.array.shape,
stats_ctrl.getAndMask(),
mask_threshold_dict=mask_threshold_dict,
mask_map=mask_map,
no_good_pixels_mask=None,
calc_error_from_input_variance=stats_ctrl.
getCalcErrorFromInputVariance(),
compute_n_image=True)
for exposure, weight in zip(exposures, weights):
stacker.add_masked_image(exposure.maskedImage, weight=weight)
stacker.fill_stacked_masked_image(coadd_exposure.maskedImage)
online_masked_image = coadd_exposure.maskedImage
# The coadds match at the <1e-5 level.
testing.assert_array_almost_equal(online_masked_image.image.array,
afw_masked_image.image.array,
decimal=5)
# The computed variances match at the <1e-4 level.
testing.assert_array_almost_equal(online_masked_image.variance.array,
afw_masked_image.variance.array,
decimal=4)
testing.assert_array_equal(online_masked_image.mask.array,
afw_masked_image.mask.array)
def testTicket1412(self):
"""Ticket 1412: ignored mask bits are propegated to output stack."""
mimg1 = afwImage.MaskedImageF(afwGeom.Extent2I(1, 1))
mimg1.set(0, 0, (1, 0x4, 1)) # set 0100
mimg2 = afwImage.MaskedImageF(afwGeom.Extent2I(1, 1))
mimg2.set(0, 0, (2, 0x3, 1)) # set 0010 and 0001
imgList = afwImage.vectorMaskedImageF()
imgList.push_back(mimg1)
imgList.push_back(mimg2)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(0x1) # andmask only 0001
# try first with no sctrl (no andmask set), should see 0x0111 for all output mask pixels
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
self.assertEqual(imgStack.get(0, 0)[1], 0x7)
# now try with sctrl (andmask = 0x0001), should see 0x0100 for all output mask pixels
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN, sctrl)
self.assertEqual(imgStack.get(0, 0)[1], 0x4)
def testTicket1412(self):
"""Ticket 1412: ignored mask bits are propegated to output stack."""
mimg1 = afwImage.MaskedImageF(afwGeom.Extent2I(1, 1))
mimg1.set(0, 0, (1, 0x4, 1)) # set 0100
mimg2 = afwImage.MaskedImageF(afwGeom.Extent2I(1, 1))
mimg2.set(0, 0, (2, 0x3, 1)) # set 0010 and 0001
imgList = afwImage.vectorMaskedImageF()
imgList.push_back(mimg1)
imgList.push_back(mimg2)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(0x1) # andmask only 0001
# try first with no sctrl (no andmask set), should see 0x0111 for all output mask pixels
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
self.assertEqual(imgStack.get(0, 0)[1], 0x7)
# now try with sctrl (andmask = 0x0001), should see 0x0100 for all output mask pixels
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN, sctrl)
self.assertEqual(imgStack.get(0, 0)[1], 0x4)
def setUp(self):
# fill an image with a gradient
self.n = 8
self.img = afwImage.ImageF(afwGeom.Extent2I(self.n, self.n), 0)
# these are the known answers for comparison
def nVector(n, v):
return [v for i in range(n)]
self.column = nVector(self.n, 0.0)
self.row = nVector(self.n, 0.0)
self.colPlus = nVector(self.n, 0.0)
self.colMinus = nVector(self.n, 0.0)
self.colMult = nVector(self.n, 0.0)
self.colDiv = nVector(self.n, 0.0)
self.rowPlus = nVector(self.n, 0.0)
self.rowMinus = nVector(self.n, 0.0)
self.rowMult = nVector(self.n, 0.0)
self.rowDiv = nVector(self.n, 0.0)
# set the values in the image, and keep track of the stats to verify things
for y in range(self.n):
for x in range(self.n):
val = 1.0*x + 2.0*y
self.img.set(x, y, val)
self.column[y] += val
self.row[x] += val
for i in range(self.n):
self.row[i] /= self.n
self.column[i] /= self.n
self.colPlus[i] = self.img.get(0, i) + self.column[i]
# get stats on the columns and rows
self.imgProjectCol = afwMath.statisticsStack(self.img, afwMath.MEAN, 'x')
self.imgProjectRow = afwMath.statisticsStack(self.img, afwMath.MEAN, 'y')
def testConstantWeightedStack(self):
""" Test statisticsStack() function when weighting by a vector of weights"""
sctrl = afwMath.StatisticsControl()
imgList = afwImage.vectorImageF()
weights = afwMath.vectorF()
for val in self.values:
img = afwImage.ImageF(afwGeom.Extent2I(self.nX, self.nY), val)
imgList.push_back(img)
weights.push_back(val)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN, sctrl, weights)
wsum = reduce(lambda x, y: x + y, self.values)
wvalues = [x*x for x in self.values]
wmean = reduce(lambda x, y: x + y, wvalues)/float(wsum)
self.assertAlmostEqual(imgStack.get(self.nX//2, self.nY//2), wmean)
def testConstantWeightedStack(self):
""" Test statisticsStack() function when weighting by a vector of weights"""
sctrl = afwMath.StatisticsControl()
imgList = afwImage.vectorImageF()
weights = afwMath.vectorF()
for val in self.values:
img = afwImage.ImageF(afwGeom.Extent2I(self.nX, self.nY), val)
imgList.push_back(img)
weights.push_back(val)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN, sctrl, weights)
wsum = reduce(lambda x, y: x + y, self.values)
wvalues = [x*x for x in self.values]
wmean = reduce(lambda x, y: x + y, wvalues)/float(wsum)
self.assertAlmostEqual(imgStack.get(self.nX/2, self.nY/2), wmean)
def testConstantWeightedStack(self):
""" Test statisticsStack() function when weighting by a vector of weights"""
sctrl = afwMath.StatisticsControl()
imgList = []
weights = []
for val in self.values:
img = afwImage.ImageF(lsst.geom.Extent2I(self.nX, self.nY), val)
imgList.append(img)
weights.append(val)
imgStack = afwMath.statisticsStack(
imgList, afwMath.MEAN, sctrl, weights)
wsum = reduce(lambda x, y: x + y, self.values)
wvalues = [x*x for x in self.values]
wmean = reduce(lambda x, y: x + y, wvalues)/float(wsum)
self.assertAlmostEqual(imgStack[self.nX//2, self.nY//2, afwImage.LOCAL], wmean)
def averageBackgrounds(self, bgList):
"""Average multiple background models
The input background models should be a `BackgroundList` consisting
of a single `BackgroundMI`.
Parameters
----------
bgList : `list` of `lsst.afw.math.BackgroundList`
Background models to average.
Returns
-------
bgExp : `lsst.afw.image.Exposure`
Background model in Exposure format.
"""
assert all(len(bg) == 1 for bg in
bgList), "Mixed bgList: %s" % ([len(bg) for bg in bgList], )
images = [bg[0][0].getStatsImage() for bg in bgList]
boxes = [bg[0][0].getImageBBox() for bg in bgList]
assert len(set((box.getMinX(), box.getMinY(), box.getMaxX(), box.getMaxY()) for box in boxes)) == 1, \
"Bounding boxes not all equal"
bbox = boxes.pop(0)
# Ensure bad pixels are masked
maskVal = afwImage.Mask.getPlaneBitMask("BAD")
for img in images:
bad = numpy.isnan(img.getImage().getArray())
img.getMask().getArray()[bad] = maskVal
stats = afwMath.StatisticsControl()
stats.setAndMask(maskVal)
stats.setNanSafe(True)
combined = afwMath.statisticsStack(images, afwMath.MEANCLIP, stats)
# Set bad pixels to the median
# Specifically NOT going to attempt to interpolate the bad values because we're only working on a
# single CCD here and can't use the entire field-of-view to do the interpolation (which is what we
# would need to avoid introducing problems at the edge of CCDs).
array = combined.getImage().getArray()
bad = numpy.isnan(array)
median = numpy.median(array[~bad])
array[bad] = median
# Put it into an exposure, which is required for calibs
return self.backgroundToExposure(combined, bbox)
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
bgInfoList, altMaskList, statsFlags, statsCtrl):
"""Assemble the coadd for a sub-region
@param coaddExposure: The target image for the coadd
@param bbox: Sub-region to coadd
@param tempExpRefList: List of data reference to tempExp
@param imageScalerList: List of image scalers
@param weightList: List of weights
@param bgInfoList: List of background data from background matching
@param altMaskList: List of alternate masks to use rather than those stored with tempExp, or None
@param statsFlags: Statistic for coadd
@param statsCtrl: Statistics control object for coadd
"""
self.log.logdebug("Computing coadd over %s" % bbox)
tempExpName = self.getTempExpDatasetName()
coaddMaskedImage = coaddExposure.getMaskedImage()
maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack
for tempExpRef, imageScaler, bgInfo, altMask in zip(tempExpRefList, imageScalerList, bgInfoList,
altMaskList):
exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
maskedImage = exposure.getMaskedImage()
if altMask:
altMaskSub = altMask.Factory(altMask, bbox, afwImage.PARENT)
maskedImage.getMask().swap(altMaskSub)
imageScaler.scaleMaskedImage(maskedImage)
if self.config.doMatchBackgrounds and not bgInfo.isReference:
backgroundModel = bgInfo.backgroundModel
backgroundImage = backgroundModel.getImage() if \
self.matchBackgrounds.config.usePolynomial else \
backgroundModel.getImageF()
backgroundImage.setXY0(coaddMaskedImage.getXY0())
maskedImage += backgroundImage.Factory(backgroundImage, bbox, afwImage.PARENT, False)
var = maskedImage.getVariance()
var += (bgInfo.fitRMS)**2
maskedImageList.append(maskedImage)
with self.timer("stack"):
coaddSubregion = afwMath.statisticsStack(
maskedImageList, statsFlags, statsCtrl, weightList)
coaddMaskedImage.assign(coaddSubregion, bbox)
def test_online_coadd_image(self):
"""Test online coaddition with regular non-masked images."""
exposures, weights = self.make_test_images_to_coadd()
coadd_exposure = self.make_coadd_exposure(exposures[0])
stats_ctrl = self.make_stats_ctrl()
stats_ctrl.setAndMask(0)
stats_ctrl.setCalcErrorFromInputVariance(True)
mask_map = self.make_mask_map(stats_ctrl)
stats_flags = afwMath.stringToStatisticsProperty("MEAN")
clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
masked_image_list = [exp.maskedImage for exp in exposures]
afw_masked_image = afwMath.statisticsStack(masked_image_list,
stats_flags, stats_ctrl,
weights, clipped, mask_map)
# Make the stack with the online accumulator
stacker = AccumulatorMeanStack(
coadd_exposure.image.array.shape,
stats_ctrl.getAndMask(),
mask_map=mask_map,
no_good_pixels_mask=stats_ctrl.getNoGoodPixelsMask(),
calc_error_from_input_variance=stats_ctrl.
getCalcErrorFromInputVariance(),
compute_n_image=True)
for exposure, weight in zip(exposures, weights):
stacker.add_image(exposure.image, weight=weight)
stacker.fill_stacked_image(coadd_exposure.image)
online_image = coadd_exposure.image
# The unmasked coadd good pixels should match at the <1e-5 level
# The masked pixels will not be correct for straight image stacking.
good_pixels = np.where(afw_masked_image.mask.array == 0)
testing.assert_array_almost_equal(
online_image.array[good_pixels],
afw_masked_image.image.array[good_pixels],
decimal=5)
def fits_median(files, hdu=2, fix=True):
"""Compute the median image from a set of image FITS files."""
ims = [afwImage.ImageF(f, hdu) for f in files]
exptimes = [Metadata(f).get('EXPTIME') for f in files]
if min(exptimes) != max(exptimes):
raise RuntimeError("Error: unequal exposure times")
if fix:
medians = np.array([median(im) for im in ims])
med = sum(medians) / len(medians)
errs = medians - med
for im, err in zip(ims, errs):
im -= err
if lsst.afw.__version__.startswith('12.0'):
ims = afwImage.vectorImageF(ims)
median_image = afwMath.statisticsStack(ims, afwMath.MEDIAN)
return median_image
def test2145(self):
"""The how-to-repeat from #2145"""
Size = 5
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setCalcErrorFromInputVariance(True)
maskedImageList = afwImage.vectorMaskedImageF()
weightList = []
for i in range(3):
mi = afwImage.MaskedImageF(Size, Size)
imArr, maskArr, varArr = mi.getArrays()
imArr[:] = numpy.random.normal(10, 0.1, (Size, Size))
varArr[:] = numpy.random.normal(10, 0.1, (Size, Size))
maskedImageList.append(mi)
weightList.append(1.0)
stack = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, weightList)
if False:
print "image=", stack.getImage().getArray()
print "variance=", stack.getVariance().getArray()
self.assertNotEqual(numpy.sum(stack.getVariance().getArray()), 0.0)
def test2145(self):
"""The how-to-repeat from #2145"""
Size = 5
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setCalcErrorFromInputVariance(True)
maskedImageList = afwImage.vectorMaskedImageF()
weightList = []
for i in range(3):
mi = afwImage.MaskedImageF(Size, Size)
imArr, maskArr, varArr = mi.getArrays()
imArr[:] = numpy.random.normal(10, 0.1, (Size, Size))
varArr[:] = numpy.random.normal(10, 0.1, (Size, Size))
maskedImageList.append(mi)
weightList.append(1.0)
stack = afwMath.statisticsStack(maskedImageList, afwMath.MEAN, statsCtrl, weightList)
if False:
print "image=", stack.getImage().getArray()
print "variance=", stack.getVariance().getArray()
self.assertNotEqual(numpy.sum(stack.getVariance().getArray()), 0.0)
def testReturnInputs(self):
""" Make sure that a single file put into the stacker is returned unscathed"""
imgList = afwImage.vectorMaskedImageF()
img = afwImage.MaskedImageF(afwGeom.Extent2I(10, 20))
for y in range(img.getHeight()):
simg = img.Factory(
img, afwGeom.Box2I(afwGeom.Point2I(0, y), afwGeom.Extent2I(img.getWidth(), 1)), afwImage.LOCAL
)
simg.set(y)
imgList.push_back(img)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
if display:
ds9.mtv(img, frame=1, title="input")
ds9.mtv(imgStack, frame=2, title="stack")
self.assertEqual(img.get(0, 0)[0], imgStack.get(0, 0)[0])
def testReturnInputs(self):
""" Make sure that a single file put into the stacker is returned unscathed"""
imgList = afwImage.vectorMaskedImageF()
img = afwImage.MaskedImageF(afwGeom.Extent2I(10, 20))
for y in range(img.getHeight()):
simg = img.Factory(
img,
afwGeom.Box2I(afwGeom.Point2I(0, y), afwGeom.Extent2I(img.getWidth(), 1)),
afwImage.LOCAL)
simg.set(y)
imgList.push_back(img)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
if display:
ds9.mtv(img, frame=1, title="input")
ds9.mtv(imgStack, frame=2, title="stack")
self.assertEqual(img.get(0, 0)[0], imgStack.get(0, 0)[0])
def testReturnInputs(self):
""" Make sure that a single file put into the stacker is returned unscathed"""
imgList = []
img = afwImage.MaskedImageF(lsst.geom.Extent2I(10, 20))
for y in range(img.getHeight()):
simg = img.Factory(
img,
lsst.geom.Box2I(lsst.geom.Point2I(0, y),
lsst.geom.Extent2I(img.getWidth(), 1)),
afwImage.LOCAL)
simg.set(y)
imgList.append(img)
imgStack = afwMath.statisticsStack(imgList, afwMath.MEAN)
if display:
afwDisplay.Display(frame=1).mtv(img, title="input")
afwDisplay.Display(frame=2).mtv(imgStack, title="stack")
self.assertEqual(img[0, 0, afwImage.LOCAL][0], imgStack[0, 0, afwImage.LOCAL][0])
if self.config.doMatchBackgrounds and not backgroundInfoList[idx].isReference:
backgroundModel = backgroundInfoList[idx].backgroundModel
backgroundImage = backgroundModel.getImage() if \
self.matchBackgrounds.config.usePolynomial else \
backgroundModel.getImageF()
backgroundImage.setXY0(coaddMaskedImage.getXY0())
maskedImage += backgroundImage.Factory(backgroundImage, subBBox,
afwImage.PARENT, False)
var = maskedImage.getVariance()
var += (backgroundInfoList[idx].fitRMS)**2
maskedImageList.append(maskedImage)
with self.timer("stack"):
coaddSubregion = afwMath.statisticsStack(
maskedImageList, statsFlags, statsCtrl, weightList)
coaddView <<= coaddSubregion
except Exception, e:
self.log.fatal("Cannot compute coadd %s: %s" % (subBBox, e,))
if self.config.doMatchBackgrounds:
self.log.info("Adding exposure information to metadata")
metadata = coaddExposure.getMetadata()
metadata.addString("CTExp_SDQA1_DESCRIPTION",
"Background matching: Ratio of matchedMSE / diffImVar")
for ind, (tempExpRef, backgroundInfo) in enumerate(zip(tempExpRefList, backgroundInfoList)):
tempExpStr = '&'.join('%s=%s' % (k,v) for k,v in tempExpRef.dataId.items())
if backgroundInfo.isReference:
metadata.addString("ReferenceExp_ID", tempExpStr)
else:
def tst():
imgStackBad = afwMath.statisticsStack(imgList, afwMath.MEAN | afwMath.MEANCLIP, sctrl)
def testStackBadPixels(self):
"""Check that we properly ignore masked pixels, and set noGoodPixelsMask where there are
no good pixels"""
mimgVec = afwImage.vectorMaskedImageF()
DETECTED = afwImage.MaskU_getPlaneBitMask("DETECTED")
EDGE = afwImage.MaskU_getPlaneBitMask("EDGE")
INTRP = afwImage.MaskU_getPlaneBitMask("INTRP")
SAT = afwImage.MaskU_getPlaneBitMask("SAT")
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
sctrl.setAndMask(INTRP | SAT)
sctrl.setNoGoodPixelsMask(EDGE)
edgeBBox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(20, 20)) # set these pixels to EDGE
width, height = 512, 512
dim=afwGeom.Extent2I(width, height)
val, maskVal = 10, DETECTED
for i in range(4):
mimg = afwImage.MaskedImageF(dim)
mimg.set(val, maskVal, 1)
#
# Set part of the image to NaN (with the INTRP bit set)
#
llc = afwGeom.Point2I(width//2*(i//2), height//2*(i%2))
bbox = afwGeom.Box2I(llc, dim//2)
smimg = mimg.Factory(mimg, bbox, afwImage.LOCAL)
#smimg.set(numpy.nan, INTRP, numpy.nan)
del smimg
#
# And the bottom corner to SAT
#
smask = mimg.getMask().Factory(mimg.getMask(), edgeBBox, afwImage.LOCAL)
smask |= SAT
del smask
mimgVec.push_back(mimg)
if display > 1:
ds9.mtv(mimg, frame=i, title=str(i))
mimgStack = afwMath.statisticsStack(mimgVec, afwMath.MEAN, sctrl)
if display:
i += 1
ds9.mtv(mimgStack, frame=i, title="Stack")
i += 1
ds9.mtv(mimgStack.getVariance(), frame=i, title="var(Stack)")
#
# Check the output, ignoring EDGE pixels
#
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(afwImage.MaskU_getPlaneBitMask("EDGE"))
stats = afwMath.makeStatistics(mimgStack, afwMath.MIN | afwMath.MAX, sctrl)
self.assertEqual(stats.getValue(afwMath.MIN), val)
self.assertEqual(stats.getValue(afwMath.MAX), val)
#
# We have to clear EDGE in the known bad corner to check the mask
#
smask = mimgStack.getMask().Factory(mimgStack.getMask(), edgeBBox, afwImage.LOCAL)
self.assertEqual(smask.get(edgeBBox.getMinX(), edgeBBox.getMinY()), EDGE)
smask &= ~EDGE
del smask
self.assertEqual(afwMath.makeStatistics(mimgStack.getMask(), afwMath.SUM, sctrl).getValue(), maskVal)
def run(self, exposure, sensorRef, templateIdList=None):
"""Retrieve and mosaic a template coadd that overlaps the exposure where
the template spans multiple tracts.
The resulting template image will be an average of all the input templates from
the separate tracts.
The PSF on the template is created by combining the CoaddPsf on each template image
into a meta-CoaddPsf.
Parameters
----------
exposure: `lsst.afw.image.Exposure`
an exposure for which to generate an overlapping template
sensorRef : TYPE
a Butler data reference that can be used to obtain coadd data
templateIdList : TYPE, optional
list of data ids (unused)
Returns
-------
result : `struct`
return a pipeBase.Struct:
- ``exposure`` : a template coadd exposure assembled out of patches
- ``sources`` : None for this subtask
"""
# Table for CoaddPSF
tractsSchema = afwTable.ExposureTable.makeMinimalSchema()
tractKey = tractsSchema.addField('tract',
type=np.int32,
doc='Which tract')
patchKey = tractsSchema.addField('patch',
type=np.int32,
doc='Which patch')
weightKey = tractsSchema.addField(
'weight', type=float, doc='Weight for each tract, should be 1')
tractsCatalog = afwTable.ExposureCatalog(tractsSchema)
skyMap = sensorRef.get(datasetType=self.config.coaddName +
"Coadd_skyMap")
expWcs = exposure.getWcs()
expBoxD = geom.Box2D(exposure.getBBox())
expBoxD.grow(self.config.templateBorderSize)
ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
centralTractInfo = skyMap.findTract(ctrSkyPos)
if not centralTractInfo:
raise RuntimeError("No suitable tract found for central point")
self.log.info("Central skyMap tract %s" % (centralTractInfo.getId(), ))
skyCorners = [
expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()
]
tractPatchList = skyMap.findTractPatchList(skyCorners)
if not tractPatchList:
raise RuntimeError("No suitable tract found")
self.log.info("All overlapping skyMap tracts %s" %
([a[0].getId() for a in tractPatchList]))
# Move central tract to front of the list and use as the reference
tracts = [tract[0].getId() for tract in tractPatchList]
centralIndex = tracts.index(centralTractInfo.getId())
tracts.insert(0, tracts.pop(centralIndex))
tractPatchList.insert(0, tractPatchList.pop(centralIndex))
coaddPsf = None
coaddFilter = None
nPatchesFound = 0
maskedImageList = []
weightList = []
for itract, tract in enumerate(tracts):
tractInfo = tractPatchList[itract][0]
coaddWcs = tractInfo.getWcs()
coaddBBox = geom.Box2D()
for skyPos in skyCorners:
coaddBBox.include(coaddWcs.skyToPixel(skyPos))
coaddBBox = geom.Box2I(coaddBBox)
if itract == 0:
# Define final wcs and bounding box from the reference tract
finalWcs = coaddWcs
finalBBox = coaddBBox
patchList = tractPatchList[itract][1]
for patchInfo in patchList:
self.log.info('Adding patch %s from tract %s' %
(patchInfo.getIndex(), tract))
# Local patch information
patchSubBBox = geom.Box2I(patchInfo.getInnerBBox())
patchSubBBox.clip(coaddBBox)
patchInt = int(
f"{patchInfo.getIndex()[0]}{patchInfo.getIndex()[1]}")
innerBBox = geom.Box2I(tractInfo._minimumBoundingBox(finalWcs))
if itract == 0:
# clip to image and tract boundaries
patchSubBBox.clip(finalBBox)
patchSubBBox.clip(innerBBox)
if patchSubBBox.getArea() == 0:
self.log.debug("No ovlerap for patch %s" % patchInfo)
continue
patchArgDict = dict(
datasetType="deepCoadd_sub",
bbox=patchSubBBox,
tract=tractInfo.getId(),
patch="%s,%s" %
(patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
filter=exposure.getFilter().getName())
coaddPatch = sensorRef.get(**patchArgDict)
if coaddFilter is None:
coaddFilter = coaddPatch.getFilter()
# create full image from final bounding box
exp = afwImage.ExposureF(finalBBox, finalWcs)
exp.maskedImage.set(
np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
exp.maskedImage.assign(coaddPatch.maskedImage,
patchSubBBox)
maskedImageList.append(exp.maskedImage)
weightList.append(1)
record = tractsCatalog.addNew()
record.setPsf(coaddPatch.getPsf())
record.setWcs(coaddPatch.getWcs())
record.setPhotoCalib(coaddPatch.getPhotoCalib())
record.setBBox(patchSubBBox)
record.set(tractKey, tract)
record.set(patchKey, patchInt)
record.set(weightKey, 1.)
nPatchesFound += 1
else:
# compute the exposure bounding box in a tract that is not the reference tract
localBox = geom.Box2I()
for skyPos in skyCorners:
localBox.include(
geom.Point2I(
tractInfo.getWcs().skyToPixel(skyPos)))
# clip to patch bounding box
localBox.clip(patchSubBBox)
# grow border to deal with warping at edges
localBox.grow(self.config.templateBorderSize)
# clip to tract inner bounding box
localInnerBBox = geom.Box2I(
tractInfo._minimumBoundingBox(tractInfo.getWcs()))
localBox.clip(localInnerBBox)
patchArgDict = dict(
datasetType="deepCoadd_sub",
bbox=localBox,
tract=tractInfo.getId(),
patch="%s,%s" %
(patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
filter=exposure.getFilter().getName())
coaddPatch = sensorRef.get(**patchArgDict)
# warp to reference tract wcs
xyTransform = afwGeom.makeWcsPairTransform(
coaddPatch.getWcs(), finalWcs)
psfWarped = WarpedPsf(coaddPatch.getPsf(), xyTransform)
warped = self.warper.warpExposure(finalWcs,
coaddPatch,
maxBBox=finalBBox)
# check if warpped image is viable
if warped.getBBox().getArea() == 0:
self.log.info(
"No ovlerap for warped patch %s. Skipping" %
patchInfo)
continue
warped.setPsf(psfWarped)
exp = afwImage.ExposureF(finalBBox, finalWcs)
exp.maskedImage.set(
np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
exp.maskedImage.assign(warped.maskedImage,
warped.getBBox())
maskedImageList.append(exp.maskedImage)
weightList.append(1)
record = tractsCatalog.addNew()
record.setPsf(psfWarped)
record.setWcs(finalWcs)
record.setPhotoCalib(coaddPatch.getPhotoCalib())
record.setBBox(warped.getBBox())
record.set(tractKey, tract)
record.set(patchKey, patchInt)
record.set(weightKey, 1.)
nPatchesFound += 1
if nPatchesFound == 0:
raise RuntimeError("No patches found!")
# Combine images from individual patches together
# Do not mask any values
statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
maskMap = []
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
coaddExposure = afwImage.ExposureF(finalBBox, finalWcs)
coaddExposure.maskedImage.set(np.nan,
afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
xy0 = coaddExposure.getXY0()
coaddExposure.maskedImage = afwMath.statisticsStack(
maskedImageList, statsFlags, statsCtrl, weightList, 0, maskMap)
coaddExposure.maskedImage.setXY0(xy0)
coaddPsf = CoaddPsf(tractsCatalog, finalWcs,
self.config.coaddPsf.makeControl())
if coaddPsf is None:
raise RuntimeError("No coadd Psf found!")
coaddExposure.setPsf(coaddPsf)
coaddExposure.setFilter(coaddFilter)
return pipeBase.Struct(exposure=coaddExposure, sources=None)
def tst():
afwMath.statisticsStack(
imgList,
afwMath.Property(afwMath.MEAN | afwMath.MEANCLIP),
sctrl)
