def testMaxWithNan(self): """Test that we can handle NaNs correctly""" x, y = 10, 10 for useImage in [True, False]: if useImage: self.image = afwImage.ImageF(100, 100) self.image.set(self.val) self.image.set(x, y, np.nan) else: self.image = afwImage.MaskedImageF(100, 100) self.image.set(self.val, 0x0, 1.0) self.image.set(x, y, (np.nan, 0x0, 1.0)) self.assertEqual( afwMath.makeStatistics(self.image, afwMath.MAX).getValue(), self.val) self.assertEqual( afwMath.makeStatistics(self.image, afwMath.MEAN).getValue(), self.val) sctrl = afwMath.StatisticsControl() sctrl.setNanSafe(False) self.assertFalse( np.isfinite( afwMath.makeStatistics(self.image, afwMath.MAX, sctrl).getValue())) self.assertFalse( np.isfinite( afwMath.makeStatistics(self.image, afwMath.MEAN, sctrl).getValue()))
def testTicket1123(self): """ Ticket #1123 reported that the Statistics stack routine throws an exception when all pixels in a stack are masked. Returning a NaN pixel in the stack is preferred """ ctrl = afwMath.StatisticsControl() ctrl.setAndMask(~0x0) mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10)) mimg.set([self.val, 0x1, self.val]) # test the case with no valid pixels ... both mean and stdev should be nan stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl) mean = stat.getValue(afwMath.MEAN) stdev = stat.getValue(afwMath.STDEV) self.assertNotEqual(mean, mean) # NaN does not equal itself self.assertNotEqual(stdev, stdev) # NaN does not equal itself # test the case with one valid pixel ... mean is ok, but stdev should still be nan mimg.getMask().set(1, 1, 0x0) stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl) mean = stat.getValue(afwMath.MEAN) stdev = stat.getValue(afwMath.STDEV) self.assertEqual(mean, self.val) self.assertNotEqual(stdev, stdev) # NaN does not equal itself # test the case with two valid pixels ... both mean and stdev are ok mimg.getMask().set(1, 2, 0x0) stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl) mean = stat.getValue(afwMath.MEAN) stdev = stat.getValue(afwMath.STDEV) self.assertEqual(mean, self.val) self.assertEqual(stdev, 0.0)
def testBadImage(self): """Test that an entirely bad image doesn't cause an absolute failure """ initialValue = 20 mi = afwImage.MaskedImageF(500, 200) # # Check that no good values don't crash (they return NaN), and that a single good value # is enough to redeem the entire image # for pix00 in [np.nan, initialValue]: mi.getImage()[:] = np.nan mi.getImage()[0, 0] = pix00 sctrl = afwMath.StatisticsControl() nx, ny = 17, 17 bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP) bkgd = afwMath.makeBackground(mi, bctrl) afwMath.cast_BackgroundMI(bkgd).getStatsImage() # the test is that this doesn't fail if the bug (#2297) is fixed bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.REDUCE_INTERP_ORDER) val = np.mean(bkgdImage[0:100, 0:100].getArray()) if np.isfinite(pix00): self.assertEqual(val, pix00) else: self.assertTrue(np.isnan(val))
def testTicket987(self): """This code used to abort; so the test is that it doesn't""" afwdataDir = eups.productDir("afwdata") if not afwdataDir: print >> sys.stderr, "Skipping testTicket987 as afwdata is not setup" return imagePath = os.path.join(afwdataDir, "DC3a-Sim", "sci", "v5-e0", "v5-e0-c011-a00.sci.fits") mimg = afwImage.MaskedImageF(imagePath) binsize = 512 bctrl = afwMath.BackgroundControl("NATURAL_SPLINE") ### Adding this line solves the problem ### # note: by default undersampleStyle is THROW_EXCEPTION bctrl.setUndersampleStyle(afwMath.REDUCE_INTERP_ORDER) ################################################ nx = int(mimg.getWidth() / binsize) + 1 ny = int(mimg.getHeight() / binsize) + 1 #print 'Binning', nx, ny bctrl.setNxSample(nx) bctrl.setNySample(ny) image = mimg.getImage() backobj = afwMath.makeBackground(image, bctrl) image -= backobj.getImageF()
def testNaNFromMaskedImage(self): """Check that an extensively masked image doesn't lead to NaNs in the background estimation """ image = afwImage.MaskedImageF(800, 800) msk = image.getMask() bbox = afwGeom.BoxI(afwGeom.PointI(560, 0), afwGeom.PointI(799, 335)) smsk = msk.Factory(msk, bbox) smsk.set(msk.getPlaneBitMask("DETECTED")) binSize = 256 nx = image.getWidth()//binSize + 1 ny = image.getHeight()//binSize + 1 sctrl = afwMath.StatisticsControl() sctrl.setAndMask(reduce(lambda x, y: x | image.getMask().getPlaneBitMask(y), ['EDGE', 'DETECTED', 'DETECTED_NEGATIVE'], 0x0)) bctrl = afwMath.BackgroundControl(nx, ny, sctrl, "MEANCLIP") bkgd = afwMath.makeBackground(image, bctrl) bkgdImage = bkgd.getImageF("NATURAL_SPLINE", "THROW_EXCEPTION") if display: ds9.mtv(image) ds9.mtv(bkgdImage, frame=1) self.assertFalse(np.isnan(bkgdImage.get(0,0))) # Check that the non-string API works too bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.THROW_EXCEPTION)
def testBadRows(self): """Test that a bad set of rows in an image doesn't cause a failure """ initialValue = 20 mi = afwImage.MaskedImageF(500, 200) mi.set((initialValue, 0x0, 1.0)) im = mi.getImage() im[:, 0:100] = np.nan del im msk = mi.getMask() badBits = msk.getPlaneBitMask(['EDGE', 'DETECTED', 'DETECTED_NEGATIVE']) msk[0:400, :] |= badBits del msk if display: ds9.mtv(mi, frame=0) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(badBits) nx, ny = 17, 17 bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP) bkgd = afwMath.makeBackground(mi, bctrl) statsImage = afwMath.cast_BackgroundMI(bkgd).getStatsImage() if display: ds9.mtv(statsImage, frame=1) # the test is that this doesn't fail if the bug (#2297) is fixed bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.REDUCE_INTERP_ORDER) self.assertEqual(np.mean(bkgdImage[0:100, 0:100].getArray()), initialValue) if display: ds9.mtv(bkgdImage, frame=2)
def testCFHT_oldAPI(self): """Test background subtraction on some real CFHT data""" mi = afwImage.MaskedImageF( os.path.join(AfwdataDir, "CFHT", "D4", "cal-53535-i-797722_1.fits")) mi = mi.Factory( mi, afwGeom.Box2I(afwGeom.Point2I(32, 2), afwGeom.Point2I(2079, 4609)), afwImage.LOCAL) bctrl = afwMath.BackgroundControl(afwMath.Interpolate.AKIMA_SPLINE) bctrl.setNxSample(16) bctrl.setNySample(16) bctrl.getStatisticsControl().setNumSigmaClip(3.0) bctrl.getStatisticsControl().setNumIter(2) backobj = afwMath.makeBackground(mi.getImage(), bctrl) if display: ds9.mtv(mi, frame=0) im = mi.getImage() im -= backobj.getImageF() if display: ds9.mtv(mi, frame=1)
def _testBadValue(self, badVal): """Test that we can handle an instance of `badVal` in the data correctly""" x, y = 10, 10 for useImage in [True, False]: if useImage: self.image = afwImage.ImageF(100, 100) self.image.set(self.val) self.image.set(x, y, badVal) else: self.image = afwImage.MaskedImageF(100, 100) self.image.set(self.val, 0x0, 1.0) self.image.set(x, y, (badVal, 0x0, 1.0)) self.assertEqual( afwMath.makeStatistics(self.image, afwMath.MAX).getValue(), self.val) self.assertEqual( afwMath.makeStatistics(self.image, afwMath.MEAN).getValue(), self.val) sctrl = afwMath.StatisticsControl() sctrl.setNanSafe(False) self.assertFalse( np.isfinite( afwMath.makeStatistics(self.image, afwMath.MAX, sctrl).getValue())) self.assertFalse( np.isfinite( afwMath.makeStatistics(self.image, afwMath.MEAN, sctrl).getValue()))
def testErrorsFromVariance(self): """Test that we can estimate the errors from the incoming variances""" weight, mean, variance = 0.1, 1.0, 10.0 ctrl = afwMath.StatisticsControl() mi = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10)) npix = 10 * 10 mi.getImage().set(mean) mi.getVariance().set(variance) weights = afwImage.ImageF(mi.getDimensions()) weights.set(weight) ctrl.setCalcErrorFromInputVariance(True) weighted = afwMath.makeStatistics( mi, weights, afwMath.MEAN | afwMath.MEANCLIP | afwMath.SUM | afwMath.ERRORS, ctrl) self.assertAlmostEqual( weighted.getValue(afwMath.SUM) / (npix * mean * weight), 1) self.assertAlmostEqual(weighted.getValue(afwMath.MEAN), mean) self.assertAlmostEqual( weighted.getError(afwMath.MEAN)**2, variance / npix) self.assertAlmostEqual( weighted.getError(afwMath.MEANCLIP)**2, variance / npix)
def getCfhtImage(self): """Get a portion of a CFHT image as a MaskedImageF""" bbox = afwGeom.Box2I(afwGeom.Point2I(500, 2000), afwGeom.Point2I(2079, 4609)) imagePath = os.path.join(AfwdataDir, "CFHT", "D4", "cal-53535-i-797722_1.fits") return afwImage.MaskedImageF(imagePath, PropertySet(), bbox)
def testWeightedSum(self): ctrl = afwMath.StatisticsControl() mi = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10)) mi.getImage().set(1.0) mi.getVariance().set(0.1) stats = afwMath.makeStatistics(mi, afwMath.SUM, ctrl) self.assertEqual(stats.getValue(afwMath.SUM), 100.0) ctrl.setWeighted(True) weighted = afwMath.makeStatistics(mi, afwMath.SUM, ctrl) # precision at "4 places" as images are floats # ... variance = 0.1 is stored as 0.100000001 self.assertAlmostEqual(weighted.getValue(afwMath.SUM), 1000.0, 4)
def readImage(filename=None): """Read an image and background subtract it""" if not filename: try: afwDataDir = lsst.utils.getPackageDir("afwdata") except Exception: raise RuntimeError("You must provide a filename or setup afwdata to run these examples") filename = os.path.join(afwDataDir, "CFHT", "D4", "cal-53535-i-797722_1") bbox = afwGeom.Box2I(afwGeom.Point2I(270, 2530), afwGeom.Extent2I(512, 512)) else: bbox = None mi = afwImage.MaskedImageF(filename, 0, None, bbox, afwImage.LOCAL) mi.setXY0(afwGeom.Point2I(0, 0)) # # Subtract the background. We'd use a canned procedure, but that's in meas/utils/sourceDetection.py. We # can't fix those pesky cosmic rays either, as that's in a dependent product (meas/algorithms) too # bctrl = afwMath.BackgroundControl(afwMath.Interpolate.NATURAL_SPLINE) bctrl.setNxSample(int(mi.getWidth()/256) + 1) bctrl.setNySample(int(mi.getHeight()/256) + 1) sctrl = bctrl.getStatisticsControl() sctrl.setNumSigmaClip(3.0) sctrl.setNumIter(2) im = mi.getImage() try: backobj = afwMath.makeBackground(im, bctrl) except Exception as e: print(e, end=' ', file=sys.stderr) bctrl.setInterpStyle(afwMath.Interpolate.CONSTANT) backobj = afwMath.makeBackground(im, bctrl) im -= backobj.getImageF() # # Find sources # threshold = afwDetect.Threshold(5, afwDetect.Threshold.STDEV) npixMin = 5 # we didn't smooth fs = afwDetect.FootprintSet(mi, threshold, "DETECTED", npixMin) grow, isotropic = 1, False fs = afwDetect.FootprintSet(fs, grow, isotropic) fs.setMask(mi.getMask(), "DETECTED") return mi, fs
def testTicket1125(self): """Ticket 1125 reported that the clipped routines were aborting when called with no valid pixels. """ mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10)) mimg.set([self.val, 0x1, self.val]) ctrl = afwMath.StatisticsControl() ctrl.setAndMask(~0x0) # test the case with no valid pixels ... try MEANCLIP and STDEVCLIP stat = afwMath.makeStatistics(mimg, afwMath.MEANCLIP | afwMath.STDEVCLIP, ctrl) mean = stat.getValue(afwMath.MEANCLIP) stdev = stat.getValue(afwMath.STDEVCLIP) self.assertNotEqual(mean, mean) # NaN does not equal itself self.assertNotEqual(stdev, stdev) # NaN does not equal itself
def testBadPatch(self): """Test that a large bad patch of an image doesn't cause an absolute failure""" initialValue = 20 mi = afwImage.MaskedImageF(500, 200) mi.set((initialValue, 0x0, 1.0)) im = mi.getImage() im[0:200, :] = np.nan del im msk = mi.getMask() badBits = msk.getPlaneBitMask( ['EDGE', 'DETECTED', 'DETECTED_NEGATIVE']) msk[0:400, :] |= badBits del msk if display: ds9.mtv(mi, frame=0) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(badBits) nx, ny = 17, 17 bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP) bkgd = afwMath.makeBackground(mi, bctrl) statsImage = afwMath.cast_BackgroundMI(bkgd).getStatsImage() if display: ds9.mtv(statsImage, frame=1) # the test is that this doesn't fail if the bug (#2297) is fixed bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.REDUCE_INTERP_ORDER) self.assertEqual(np.mean(bkgdImage[0:100, 0:100].getArray()), initialValue) if display: ds9.mtv(bkgdImage, frame=2) # # Check that we can fix the NaNs in the statsImage # sim = statsImage.getImage().getArray() sim[np.isnan(sim)] = initialValue # replace NaN by initialValue bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.REDUCE_INTERP_ORDER) self.assertAlmostEqual( np.mean(bkgdImage[0:100, 0:100].getArray(), dtype=np.float64), initialValue)
def testTicket987(self): """This code used to abort; so the test is that it doesn't""" imagePath = os.path.join(AfwdataDir, "DC3a-Sim", "sci", "v5-e0", "v5-e0-c011-a00.sci.fits") mimg = afwImage.MaskedImageF(imagePath) binsize = 512 bctrl = afwMath.BackgroundControl("NATURAL_SPLINE") # note: by default undersampleStyle is THROW_EXCEPTION bctrl.setUndersampleStyle(afwMath.REDUCE_INTERP_ORDER) nx = int(mimg.getWidth() / binsize) + 1 ny = int(mimg.getHeight() / binsize) + 1 bctrl.setNxSample(nx) bctrl.setNySample(ny) image = mimg.getImage() backobj = afwMath.makeBackground(image, bctrl) image -= backobj.getImageF()
def testWeightedSum2(self): """Test using a weight image separate from the variance plane""" weight, mean = 0.1, 1.0 ctrl = afwMath.StatisticsControl() mi = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10)) npix = 10 * 10 mi.getImage().set(mean) mi.getVariance().set(np.nan) weights = afwImage.ImageF(mi.getDimensions()) weights.set(weight) stats = afwMath.makeStatistics(mi, afwMath.SUM, ctrl) self.assertEqual(stats.getValue(afwMath.SUM), mean * npix) weighted = afwMath.makeStatistics(mi, weights, afwMath.SUM, ctrl) # precision at "4 places" as images are floats # ... variance = 0.1 is stored as 0.100000001 self.assertAlmostEqual(weighted.getValue(afwMath.SUM), mean * npix * weight, 4)
def readImage(filename=None): """Read an image and background subtract it""" if not filename: try: afwDataDir = lsst.utils.getPackageDir("afwdata") except Exception: raise RuntimeError( "You must provide a filename or setup afwdata to run these examples" ) filename = os.path.join(afwDataDir, "CFHT", "D4", "cal-53535-i-797722_1") bbox = afwGeom.Box2I(afwGeom.Point2I(270, 2530), afwGeom.Extent2I(512, 512)) else: bbox = None mi = afwImage.MaskedImageF(filename, 0, None, bbox, afwImage.LOCAL) mi.setXY0(afwGeom.Point2I(0, 0)) # # Subtract the background. We'd use a canned procedure, but that's in meas/utils/sourceDetection.py. We # can't fix those pesky cosmic rays either, as that's in a dependent product (meas/algorithms) too # bctrl = afwMath.BackgroundControl(afwMath.Interpolate.NATURAL_SPLINE) bctrl.setNxSample(int(mi.getWidth() / 256) + 1) bctrl.setNySample(int(mi.getHeight() / 256) + 1) sctrl = bctrl.getStatisticsControl() sctrl.setNumSigmaClip(3.0) sctrl.setNumIter(2) im = mi.getImage() try: backobj = afwMath.makeBackground(im, bctrl) except Exception, e: print >> sys.stderr, e, bctrl.setInterpStyle(afwMath.Interpolate.CONSTANT) backobj = afwMath.makeBackground(im, bctrl)
def testCFHT(self): """Test background subtraction on some real CFHT data""" afwdataDir = eups.productDir("afwdata") if not afwdataDir: print >> sys.stderr, "Skipping testCFHT as afwdata is not setup" return mi = afwImage.MaskedImageF( os.path.join(afwdataDir, "CFHT", "D4", "cal-53535-i-797722_1.fits")) mi = mi.Factory( mi, afwGeom.Box2I(afwGeom.Point2I(32, 2), afwGeom.Point2I(2079, 4609)), afwImage.LOCAL) bctrl = afwMath.BackgroundControl(mi.getWidth() // 128, mi.getHeight() // 128) bctrl.getStatisticsControl().setNumSigmaClip(3.0) bctrl.getStatisticsControl().setNumIter(2) backobj = afwMath.makeBackground(mi.getImage(), bctrl) if display: ds9.mtv(mi, frame=0) im = mi.getImage() im -= backobj.getImageF("AKIMA_SPLINE") if display: ds9.mtv(mi, frame=1) statsImage = afwMath.cast_BackgroundMI(backobj).getStatsImage() if display: ds9.mtv(backobj.getStatsImage(), frame=2) ds9.mtv(backobj.getStatsImage().getVariance(), frame=3)
import lsst.afw.display.ds9 as ds9 display = True verbosity = 4 logUtils.traceSetAt("ip.diffim", verbosity) defDataDir = lsst.utils.getPackageDir('afwdata') imageProcDir = lsst.utils.getPackageDir('ip_diffim') if len(sys.argv) == 1: defTemplatePath = os.path.join(defDataDir, "CFHT", "D4", "cal-53535-i-797722_2_tmpl") defSciencePath = os.path.join(defDataDir, "CFHT", "D4", "cal-53535-i-797722_2") templateMaskedImage = afwImage.MaskedImageF(defTemplatePath) scienceMaskedImage = afwImage.MaskedImageF(defSciencePath) bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(512, 512)) templateMaskedImage = afwImage.MaskedImageF(templateMaskedImage, bbox, origin=afwImage.LOCAL) scienceMaskedImage = afwImage.MaskedImageF(scienceMaskedImage, bbox, origin=afwImage.LOCAL) elif len(sys.argv) == 3: defTemplatePath = sys.argv[1] defSciencePath = sys.argv[2] templateMaskedImage = afwImage.MaskedImageF(defTemplatePath) scienceMaskedImage = afwImage.MaskedImageF(defSciencePath) else: