def testMaxWithNan(self):
"""Test that we can handle NaNs correctly"""
x, y = 10, 10
for useImage in [True, False]:
if useImage:
self.image = afwImage.ImageF(100, 100)
self.image.set(self.val)
self.image.set(x, y, np.nan)
else:
self.image = afwImage.MaskedImageF(100, 100)
self.image.set(self.val, 0x0, 1.0)
self.image.set(x, y, (np.nan, 0x0, 1.0))
self.assertEqual(
afwMath.makeStatistics(self.image, afwMath.MAX).getValue(),
self.val)
self.assertEqual(
afwMath.makeStatistics(self.image, afwMath.MEAN).getValue(),
self.val)
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
self.assertFalse(
np.isfinite(
afwMath.makeStatistics(self.image, afwMath.MAX,
sctrl).getValue()))
self.assertFalse(
np.isfinite(
afwMath.makeStatistics(self.image, afwMath.MEAN,
sctrl).getValue()))
def testTicket1123(self):
"""
Ticket #1123 reported that the Statistics stack routine throws an exception
when all pixels in a stack are masked. Returning a NaN pixel in the stack is preferred
"""
ctrl = afwMath.StatisticsControl()
ctrl.setAndMask(~0x0)
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
mimg.set([self.val, 0x1, self.val])
# test the case with no valid pixels ... both mean and stdev should be nan
stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
mean = stat.getValue(afwMath.MEAN)
stdev = stat.getValue(afwMath.STDEV)
self.assertNotEqual(mean, mean) # NaN does not equal itself
self.assertNotEqual(stdev, stdev) # NaN does not equal itself
# test the case with one valid pixel ... mean is ok, but stdev should still be nan
mimg.getMask().set(1, 1, 0x0)
stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
mean = stat.getValue(afwMath.MEAN)
stdev = stat.getValue(afwMath.STDEV)
self.assertEqual(mean, self.val)
self.assertNotEqual(stdev, stdev) # NaN does not equal itself
# test the case with two valid pixels ... both mean and stdev are ok
mimg.getMask().set(1, 2, 0x0)
stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
mean = stat.getValue(afwMath.MEAN)
stdev = stat.getValue(afwMath.STDEV)
self.assertEqual(mean, self.val)
self.assertEqual(stdev, 0.0)
def testBadImage(self):
"""Test that an entirely bad image doesn't cause an absolute failure
"""
initialValue = 20
mi = afwImage.MaskedImageF(500, 200)
#
# Check that no good values don't crash (they return NaN), and that a single good value
# is enough to redeem the entire image
#
for pix00 in [np.nan, initialValue]:
mi.getImage()[:] = np.nan
mi.getImage()[0, 0] = pix00
sctrl = afwMath.StatisticsControl()
nx, ny = 17, 17
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
bkgd = afwMath.makeBackground(mi, bctrl)
afwMath.cast_BackgroundMI(bkgd).getStatsImage()
# the test is that this doesn't fail if the bug (#2297) is fixed
bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.REDUCE_INTERP_ORDER)
val = np.mean(bkgdImage[0:100, 0:100].getArray())
if np.isfinite(pix00):
self.assertEqual(val, pix00)
else:
self.assertTrue(np.isnan(val))
def testTicket987(self):
"""This code used to abort; so the test is that it doesn't"""
afwdataDir = eups.productDir("afwdata")
if not afwdataDir:
print >> sys.stderr, "Skipping testTicket987 as afwdata is not setup"
return
imagePath = os.path.join(afwdataDir, "DC3a-Sim", "sci", "v5-e0",
"v5-e0-c011-a00.sci.fits")
mimg = afwImage.MaskedImageF(imagePath)
binsize = 512
bctrl = afwMath.BackgroundControl("NATURAL_SPLINE")
### Adding this line solves the problem ###
# note: by default undersampleStyle is THROW_EXCEPTION
bctrl.setUndersampleStyle(afwMath.REDUCE_INTERP_ORDER)
################################################
nx = int(mimg.getWidth() / binsize) + 1
ny = int(mimg.getHeight() / binsize) + 1
#print 'Binning', nx, ny
bctrl.setNxSample(nx)
bctrl.setNySample(ny)
image = mimg.getImage()
backobj = afwMath.makeBackground(image, bctrl)
image -= backobj.getImageF()
def testNaNFromMaskedImage(self):
"""Check that an extensively masked image doesn't lead to NaNs in the background estimation
"""
image = afwImage.MaskedImageF(800, 800)
msk = image.getMask()
bbox = afwGeom.BoxI(afwGeom.PointI(560, 0), afwGeom.PointI(799, 335))
smsk = msk.Factory(msk, bbox)
smsk.set(msk.getPlaneBitMask("DETECTED"))
binSize = 256
nx = image.getWidth()//binSize + 1
ny = image.getHeight()//binSize + 1
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(reduce(lambda x, y: x | image.getMask().getPlaneBitMask(y),
['EDGE', 'DETECTED', 'DETECTED_NEGATIVE'], 0x0))
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, "MEANCLIP")
bkgd = afwMath.makeBackground(image, bctrl)
bkgdImage = bkgd.getImageF("NATURAL_SPLINE", "THROW_EXCEPTION")
if display:
ds9.mtv(image)
ds9.mtv(bkgdImage, frame=1)
self.assertFalse(np.isnan(bkgdImage.get(0,0)))
# Check that the non-string API works too
bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.THROW_EXCEPTION)
def testBadRows(self):
"""Test that a bad set of rows in an image doesn't cause a failure
"""
initialValue = 20
mi = afwImage.MaskedImageF(500, 200)
mi.set((initialValue, 0x0, 1.0))
im = mi.getImage()
im[:, 0:100] = np.nan
del im
msk = mi.getMask()
badBits = msk.getPlaneBitMask(['EDGE', 'DETECTED', 'DETECTED_NEGATIVE'])
msk[0:400, :] |= badBits
del msk
if display:
ds9.mtv(mi, frame=0)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(badBits)
nx, ny = 17, 17
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
bkgd = afwMath.makeBackground(mi, bctrl)
statsImage = afwMath.cast_BackgroundMI(bkgd).getStatsImage()
if display:
ds9.mtv(statsImage, frame=1)
# the test is that this doesn't fail if the bug (#2297) is fixed
bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.REDUCE_INTERP_ORDER)
self.assertEqual(np.mean(bkgdImage[0:100, 0:100].getArray()), initialValue)
if display:
ds9.mtv(bkgdImage, frame=2)
def testCFHT_oldAPI(self):
"""Test background subtraction on some real CFHT data"""
mi = afwImage.MaskedImageF(
os.path.join(AfwdataDir, "CFHT", "D4",
"cal-53535-i-797722_1.fits"))
mi = mi.Factory(
mi,
afwGeom.Box2I(afwGeom.Point2I(32, 2), afwGeom.Point2I(2079, 4609)),
afwImage.LOCAL)
bctrl = afwMath.BackgroundControl(afwMath.Interpolate.AKIMA_SPLINE)
bctrl.setNxSample(16)
bctrl.setNySample(16)
bctrl.getStatisticsControl().setNumSigmaClip(3.0)
bctrl.getStatisticsControl().setNumIter(2)
backobj = afwMath.makeBackground(mi.getImage(), bctrl)
if display:
ds9.mtv(mi, frame=0)
im = mi.getImage()
im -= backobj.getImageF()
if display:
ds9.mtv(mi, frame=1)
def _testBadValue(self, badVal):
"""Test that we can handle an instance of `badVal` in the data correctly"""
x, y = 10, 10
for useImage in [True, False]:
if useImage:
self.image = afwImage.ImageF(100, 100)
self.image.set(self.val)
self.image.set(x, y, badVal)
else:
self.image = afwImage.MaskedImageF(100, 100)
self.image.set(self.val, 0x0, 1.0)
self.image.set(x, y, (badVal, 0x0, 1.0))
self.assertEqual(
afwMath.makeStatistics(self.image, afwMath.MAX).getValue(),
self.val)
self.assertEqual(
afwMath.makeStatistics(self.image, afwMath.MEAN).getValue(),
self.val)
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
self.assertFalse(
np.isfinite(
afwMath.makeStatistics(self.image, afwMath.MAX,
sctrl).getValue()))
self.assertFalse(
np.isfinite(
afwMath.makeStatistics(self.image, afwMath.MEAN,
sctrl).getValue()))
def testErrorsFromVariance(self):
"""Test that we can estimate the errors from the incoming variances"""
weight, mean, variance = 0.1, 1.0, 10.0
ctrl = afwMath.StatisticsControl()
mi = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
npix = 10 * 10
mi.getImage().set(mean)
mi.getVariance().set(variance)
weights = afwImage.ImageF(mi.getDimensions())
weights.set(weight)
ctrl.setCalcErrorFromInputVariance(True)
weighted = afwMath.makeStatistics(
mi, weights,
afwMath.MEAN | afwMath.MEANCLIP | afwMath.SUM | afwMath.ERRORS,
ctrl)
self.assertAlmostEqual(
weighted.getValue(afwMath.SUM) / (npix * mean * weight), 1)
self.assertAlmostEqual(weighted.getValue(afwMath.MEAN), mean)
self.assertAlmostEqual(
weighted.getError(afwMath.MEAN)**2, variance / npix)
self.assertAlmostEqual(
weighted.getError(afwMath.MEANCLIP)**2, variance / npix)
def getCfhtImage(self):
"""Get a portion of a CFHT image as a MaskedImageF"""
bbox = afwGeom.Box2I(afwGeom.Point2I(500, 2000),
afwGeom.Point2I(2079, 4609))
imagePath = os.path.join(AfwdataDir, "CFHT", "D4",
"cal-53535-i-797722_1.fits")
return afwImage.MaskedImageF(imagePath, PropertySet(), bbox)
def testWeightedSum(self):
ctrl = afwMath.StatisticsControl()
mi = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
mi.getImage().set(1.0)
mi.getVariance().set(0.1)
stats = afwMath.makeStatistics(mi, afwMath.SUM, ctrl)
self.assertEqual(stats.getValue(afwMath.SUM), 100.0)
ctrl.setWeighted(True)
weighted = afwMath.makeStatistics(mi, afwMath.SUM, ctrl)
# precision at "4 places" as images are floats
# ... variance = 0.1 is stored as 0.100000001
self.assertAlmostEqual(weighted.getValue(afwMath.SUM), 1000.0, 4)
def readImage(filename=None):
"""Read an image and background subtract it"""
if not filename:
try:
afwDataDir = lsst.utils.getPackageDir("afwdata")
except Exception:
raise RuntimeError("You must provide a filename or setup afwdata to run these examples")
filename = os.path.join(afwDataDir, "CFHT", "D4", "cal-53535-i-797722_1")
bbox = afwGeom.Box2I(afwGeom.Point2I(270, 2530), afwGeom.Extent2I(512, 512))
else:
bbox = None
mi = afwImage.MaskedImageF(filename, 0, None, bbox, afwImage.LOCAL)
mi.setXY0(afwGeom.Point2I(0, 0))
#
# Subtract the background. We'd use a canned procedure, but that's in meas/utils/sourceDetection.py. We
# can't fix those pesky cosmic rays either, as that's in a dependent product (meas/algorithms) too
#
bctrl = afwMath.BackgroundControl(afwMath.Interpolate.NATURAL_SPLINE)
bctrl.setNxSample(int(mi.getWidth()/256) + 1)
bctrl.setNySample(int(mi.getHeight()/256) + 1)
sctrl = bctrl.getStatisticsControl()
sctrl.setNumSigmaClip(3.0)
sctrl.setNumIter(2)
im = mi.getImage()
try:
backobj = afwMath.makeBackground(im, bctrl)
except Exception as e:
print(e, end=' ', file=sys.stderr)
bctrl.setInterpStyle(afwMath.Interpolate.CONSTANT)
backobj = afwMath.makeBackground(im, bctrl)
im -= backobj.getImageF()
#
# Find sources
#
threshold = afwDetect.Threshold(5, afwDetect.Threshold.STDEV)
npixMin = 5 # we didn't smooth
fs = afwDetect.FootprintSet(mi, threshold, "DETECTED", npixMin)
grow, isotropic = 1, False
fs = afwDetect.FootprintSet(fs, grow, isotropic)
fs.setMask(mi.getMask(), "DETECTED")
return mi, fs
def testTicket1125(self):
"""Ticket 1125 reported that the clipped routines were aborting when called with no valid pixels. """
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
mimg.set([self.val, 0x1, self.val])
ctrl = afwMath.StatisticsControl()
ctrl.setAndMask(~0x0)
# test the case with no valid pixels ... try MEANCLIP and STDEVCLIP
stat = afwMath.makeStatistics(mimg,
afwMath.MEANCLIP | afwMath.STDEVCLIP,
ctrl)
mean = stat.getValue(afwMath.MEANCLIP)
stdev = stat.getValue(afwMath.STDEVCLIP)
self.assertNotEqual(mean, mean) # NaN does not equal itself
self.assertNotEqual(stdev, stdev) # NaN does not equal itself
def testBadPatch(self):
"""Test that a large bad patch of an image doesn't cause an absolute failure"""
initialValue = 20
mi = afwImage.MaskedImageF(500, 200)
mi.set((initialValue, 0x0, 1.0))
im = mi.getImage()
im[0:200, :] = np.nan
del im
msk = mi.getMask()
badBits = msk.getPlaneBitMask(
['EDGE', 'DETECTED', 'DETECTED_NEGATIVE'])
msk[0:400, :] |= badBits
del msk
if display:
ds9.mtv(mi, frame=0)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(badBits)
nx, ny = 17, 17
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
bkgd = afwMath.makeBackground(mi, bctrl)
statsImage = afwMath.cast_BackgroundMI(bkgd).getStatsImage()
if display:
ds9.mtv(statsImage, frame=1)
# the test is that this doesn't fail if the bug (#2297) is fixed
bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE,
afwMath.REDUCE_INTERP_ORDER)
self.assertEqual(np.mean(bkgdImage[0:100, 0:100].getArray()),
initialValue)
if display:
ds9.mtv(bkgdImage, frame=2)
#
# Check that we can fix the NaNs in the statsImage
#
sim = statsImage.getImage().getArray()
sim[np.isnan(sim)] = initialValue # replace NaN by initialValue
bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE,
afwMath.REDUCE_INTERP_ORDER)
self.assertAlmostEqual(
np.mean(bkgdImage[0:100, 0:100].getArray(), dtype=np.float64),
initialValue)
def testTicket987(self):
"""This code used to abort; so the test is that it doesn't"""
imagePath = os.path.join(AfwdataDir, "DC3a-Sim", "sci", "v5-e0",
"v5-e0-c011-a00.sci.fits")
mimg = afwImage.MaskedImageF(imagePath)
binsize = 512
bctrl = afwMath.BackgroundControl("NATURAL_SPLINE")
# note: by default undersampleStyle is THROW_EXCEPTION
bctrl.setUndersampleStyle(afwMath.REDUCE_INTERP_ORDER)
nx = int(mimg.getWidth() / binsize) + 1
ny = int(mimg.getHeight() / binsize) + 1
bctrl.setNxSample(nx)
bctrl.setNySample(ny)
image = mimg.getImage()
backobj = afwMath.makeBackground(image, bctrl)
image -= backobj.getImageF()
def testWeightedSum2(self):
"""Test using a weight image separate from the variance plane"""
weight, mean = 0.1, 1.0
ctrl = afwMath.StatisticsControl()
mi = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
npix = 10 * 10
mi.getImage().set(mean)
mi.getVariance().set(np.nan)
weights = afwImage.ImageF(mi.getDimensions())
weights.set(weight)
stats = afwMath.makeStatistics(mi, afwMath.SUM, ctrl)
self.assertEqual(stats.getValue(afwMath.SUM), mean * npix)
weighted = afwMath.makeStatistics(mi, weights, afwMath.SUM, ctrl)
# precision at "4 places" as images are floats
# ... variance = 0.1 is stored as 0.100000001
self.assertAlmostEqual(weighted.getValue(afwMath.SUM),
mean * npix * weight, 4)
def readImage(filename=None):
"""Read an image and background subtract it"""
if not filename:
try:
afwDataDir = lsst.utils.getPackageDir("afwdata")
except Exception:
raise RuntimeError(
"You must provide a filename or setup afwdata to run these examples"
)
filename = os.path.join(afwDataDir, "CFHT", "D4",
"cal-53535-i-797722_1")
bbox = afwGeom.Box2I(afwGeom.Point2I(270, 2530),
afwGeom.Extent2I(512, 512))
else:
bbox = None
mi = afwImage.MaskedImageF(filename, 0, None, bbox, afwImage.LOCAL)
mi.setXY0(afwGeom.Point2I(0, 0))
#
# Subtract the background. We'd use a canned procedure, but that's in meas/utils/sourceDetection.py. We
# can't fix those pesky cosmic rays either, as that's in a dependent product (meas/algorithms) too
#
bctrl = afwMath.BackgroundControl(afwMath.Interpolate.NATURAL_SPLINE)
bctrl.setNxSample(int(mi.getWidth() / 256) + 1)
bctrl.setNySample(int(mi.getHeight() / 256) + 1)
sctrl = bctrl.getStatisticsControl()
sctrl.setNumSigmaClip(3.0)
sctrl.setNumIter(2)
im = mi.getImage()
try:
backobj = afwMath.makeBackground(im, bctrl)
except Exception, e:
print >> sys.stderr, e,
bctrl.setInterpStyle(afwMath.Interpolate.CONSTANT)
backobj = afwMath.makeBackground(im, bctrl)
def testCFHT(self):
"""Test background subtraction on some real CFHT data"""
afwdataDir = eups.productDir("afwdata")
if not afwdataDir:
print >> sys.stderr, "Skipping testCFHT as afwdata is not setup"
return
mi = afwImage.MaskedImageF(
os.path.join(afwdataDir, "CFHT", "D4",
"cal-53535-i-797722_1.fits"))
mi = mi.Factory(
mi,
afwGeom.Box2I(afwGeom.Point2I(32, 2), afwGeom.Point2I(2079, 4609)),
afwImage.LOCAL)
bctrl = afwMath.BackgroundControl(mi.getWidth() // 128,
mi.getHeight() // 128)
bctrl.getStatisticsControl().setNumSigmaClip(3.0)
bctrl.getStatisticsControl().setNumIter(2)
backobj = afwMath.makeBackground(mi.getImage(), bctrl)
if display:
ds9.mtv(mi, frame=0)
im = mi.getImage()
im -= backobj.getImageF("AKIMA_SPLINE")
if display:
ds9.mtv(mi, frame=1)
statsImage = afwMath.cast_BackgroundMI(backobj).getStatsImage()
if display:
ds9.mtv(backobj.getStatsImage(), frame=2)
ds9.mtv(backobj.getStatsImage().getVariance(), frame=3)
import lsst.afw.display.ds9 as ds9
display = True
verbosity = 4
logUtils.traceSetAt("ip.diffim", verbosity)
defDataDir = lsst.utils.getPackageDir('afwdata')
imageProcDir = lsst.utils.getPackageDir('ip_diffim')
if len(sys.argv) == 1:
defTemplatePath = os.path.join(defDataDir, "CFHT", "D4",
"cal-53535-i-797722_2_tmpl")
defSciencePath = os.path.join(defDataDir, "CFHT", "D4",
"cal-53535-i-797722_2")
templateMaskedImage = afwImage.MaskedImageF(defTemplatePath)
scienceMaskedImage = afwImage.MaskedImageF(defSciencePath)
bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(512, 512))
templateMaskedImage = afwImage.MaskedImageF(templateMaskedImage,
bbox,
origin=afwImage.LOCAL)
scienceMaskedImage = afwImage.MaskedImageF(scienceMaskedImage,
bbox,
origin=afwImage.LOCAL)
elif len(sys.argv) == 3:
defTemplatePath = sys.argv[1]
defSciencePath = sys.argv[2]
templateMaskedImage = afwImage.MaskedImageF(defTemplatePath)
scienceMaskedImage = afwImage.MaskedImageF(defSciencePath)
else:
