def testLinearRamp(self): """Fit a ramp""" binsize = 1 ramp, rampCoeffs, xVec, yVec = self.makeRamp(binsize) # Add a (labelled) bad value ramp.set(ramp.getWidth()//2, ramp.getHeight()//2, (0, 0x1, np.nan)) if display: ds9.mtv(ramp, title="Input", frame=0) # Here's the range that the approximation should be valid (and also the # bbox of the image returned by getImage) bbox = afwGeom.BoxI(afwGeom.PointI(0, 0), afwGeom.PointI(binsize*ramp.getWidth() - 1, binsize*ramp.getHeight() - 1)) order = 3 # 1 would be enough to fit the ramp actrl = afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, order) approx = afwMath.makeApproximate(xVec, yVec, ramp, bbox, actrl) for i, aim in enumerate([approx.getImage(), approx.getMaskedImage().getImage(), ]): if i == 0 and display: ds9.mtv(aim, title="interpolated", frame=1) with ds9.Buffering(): for x in xVec: for y in yVec: ds9.dot('+', x, y, size=0.4, frame=1) for x, y in aim.getBBox().getCorners(): self.assertEqual( aim.get(x, y), rampCoeffs[0] + rampCoeffs[1]*x + rampCoeffs[1]*y)
def testChebyshevEqualOrder(self): """Check that we enforce the condition orderX == orderY""" self.assertRaises( pexExcept.InvalidParameterError, lambda: afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, 1, 2))
def testLinearRampAsBackground(self): """Fit a ramp""" ramp, rampCoeffs = self.makeRamp()[0:2] if display: ds9.mtv(ramp, title="Input", frame=0) # Here's the range that the approximation should be valid (and also the # bbox of the image returned by getImage) bkgd = afwMath.makeBackground(ramp, afwMath.BackgroundControl(10, 10)) orderMax = 3 # 1 would be enough to fit the ramp for order in range(orderMax + 1): actrl = afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, order) approx = bkgd.getApproximate(actrl) # Get the Image, the MaskedImage, and the Image with a truncated expansion for i, aim in enumerate([ approx.getImage(), approx.getMaskedImage().getImage(), approx.getImage(order - 1 if order > 1 else -1), ]): if display and (i == 0 and order == 1): ds9.mtv(aim, title="Interpolated", frame=1) for x, y in aim.getBBox().getCorners(): val = np.mean(aim.getArray()) if order == 0 else \ rampCoeffs[0] + rampCoeffs[1]*x + rampCoeffs[1]*y self.assertEqual(aim.get(x, y), val) # Check that we can't "truncate" the expansion to a higher order than we requested self.assertRaises(pexExcept.InvalidParameterError, lambda: approx.getImage(orderMax + 1, orderMax + 1))
def testApproximate(self): """Test I/O for BackgroundLists with Approximate""" # approx and interp should be very close, but not the same img = self.getParabolaImage(256, 256) # try regular interpolated image (the default) interpStyle = afwMath.Interpolate.AKIMA_SPLINE undersampleStyle = afwMath.REDUCE_INTERP_ORDER bgCtrl = afwMath.BackgroundControl(6, 6) bgCtrl.setInterpStyle(interpStyle) bgCtrl.setUndersampleStyle(undersampleStyle) bkgd = afwMath.makeBackground(img, bgCtrl) interpImage = bkgd.getImageF() with lsst.utils.tests.getTempFilePath("_bgi.fits") as bgiFile, \ lsst.utils.tests.getTempFilePath("_bga.fits") as bgaFile: bglInterp = afwMath.BackgroundList() bglInterp.append((bkgd, interpStyle, undersampleStyle, afwMath.ApproximateControl.UNKNOWN, 0, 0, True)) bglInterp.writeFits(bgiFile) # try an approx background approxStyle = afwMath.ApproximateControl.CHEBYSHEV approxOrder = 2 actrl = afwMath.ApproximateControl(approxStyle, approxOrder) bkgd.getBackgroundControl().setApproximateControl(actrl) approxImage = bkgd.getImageF() bglApprox = afwMath.BackgroundList() bglApprox.append((bkgd, interpStyle, undersampleStyle, approxStyle, approxOrder, approxOrder, True)) bglApprox.writeFits(bgaFile) # take a difference and make sure the two are very similar interpNp = interpImage.getArray() diff = np.abs(interpNp - approxImage.getArray()) / interpNp # the image and interp/approx parameters are chosen so these limits # will be greater than machine precision for float. The two methods # should be measurably different (so we know we're not just getting the # same thing from the getImage() method. But they should be very close # since they're both doing the same sort of thing. tolSame = 1.0e-3 # should be the same to this order tolDiff = 1.0e-4 # should be different here self.assertLess(diff.max(), tolSame) self.assertGreater(diff.max(), tolDiff) # now see if we can reload them from files and get the same images # we wrote interpImage2 = afwMath.BackgroundList().readFits( bgiFile).getImage() approxImage2 = afwMath.BackgroundList().readFits( bgaFile).getImage() idiff = interpImage.getArray() - interpImage2.getArray() adiff = approxImage.getArray() - approxImage2.getArray() self.assertEqual(idiff.max(), 0.0) self.assertEqual(adiff.max(), 0.0)
def getBackground(image, backgroundConfig, nx=0, ny=0, algorithm=None): """ Make a new Exposure which is exposure - background """ backgroundConfig.validate() if not nx: nx = image.getWidth() // backgroundConfig.binSize + 1 if not ny: ny = image.getHeight() // backgroundConfig.binSize + 1 displayBackground = lsstDebug.Info(__name__).displayBackground if displayBackground: import itertools ds9.mtv(image, frame=1) xPosts = numpy.rint( numpy.linspace(0, image.getWidth() + 1, num=nx, endpoint=True)) yPosts = numpy.rint( numpy.linspace(0, image.getHeight() + 1, num=ny, endpoint=True)) with ds9.Buffering(): for (xMin, xMax), (yMin, yMax) in itertools.product( zip(xPosts[:-1], xPosts[1:]), zip(yPosts[:-1], yPosts[1:])): ds9.line([(xMin, yMin), (xMin, yMax), (xMax, yMax), (xMax, yMin), (xMin, yMin)], frame=1) sctrl = afwMath.StatisticsControl() sctrl.setAndMask( reduce(lambda x, y: x | image.getMask().getPlaneBitMask(y), backgroundConfig.ignoredPixelMask, 0x0)) sctrl.setNanSafe(backgroundConfig.isNanSafe) pl = pexLogging.Debug("meas.utils.sourceDetection.getBackground") pl.debug( 3, "Ignoring mask planes: %s" % ", ".join(backgroundConfig.ignoredPixelMask)) if not algorithm: algorithm = backgroundConfig.algorithm bctrl = afwMath.BackgroundControl(algorithm, nx, ny, backgroundConfig.undersampleStyle, sctrl, backgroundConfig.statisticsProperty) if backgroundConfig.useApprox: actrl = afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, backgroundConfig.approxOrder) bctrl.setApproximateControl(actrl) return afwMath.makeBackground(image, bctrl)
def testNoFinitePoints(self): """Check that makeApproximate throws a RuntimeError if grid has no finite points and weights to fit """ binsize = 1 for badValue in [(3, 0x1, 0), (np.nan, 0x1, 1)]: ramp, rampCoeffs, xVec, yVec = self.makeRamp(binsize) ramp.set(badValue) bbox = afwGeom.BoxI(afwGeom.PointI(0, 0), afwGeom.PointI(binsize*ramp.getWidth() - 1, binsize*ramp.getHeight() - 1)) order = 2 actrl = afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, order) self.assertRaises(pexExcept.RuntimeError, lambda: afwMath.makeApproximate(xVec, yVec, ramp, bbox, actrl))
def testBackgroundListIO(self): """Test I/O for BackgroundLists""" bgCtrl = afwMath.BackgroundControl(10, 10) interpStyle = afwMath.Interpolate.AKIMA_SPLINE undersampleStyle = afwMath.REDUCE_INTERP_ORDER approxOrderX = 6 approxOrderY = 6 im = self.image.Factory(self.image, self.image.getBBox(afwImage.PARENT)) arr = im.getArray() arr += numpy.random.normal(size=(im.getHeight(), im.getWidth())) for astyle in afwMath.ApproximateControl.UNKNOWN, afwMath.ApproximateControl.CHEBYSHEV: actrl = afwMath.ApproximateControl(astyle, approxOrderX) bgCtrl.setApproximateControl(actrl) backgroundList = afwMath.BackgroundList() backImage = afwImage.ImageF(im.getDimensions()) for i in range(2): bkgd = afwMath.makeBackground(im, bgCtrl) if i == 0: # no need to call getImage backgroundList.append((bkgd, interpStyle, undersampleStyle, astyle, approxOrderX, approxOrderY)) else: backgroundList.append( bkgd) # Relies on having called getImage; deprecated backImage += bkgd.getImageF(interpStyle, undersampleStyle) fileName = "backgroundList.fits" try: backgroundList.writeFits(fileName) backgrounds = afwMath.BackgroundList.readFits(fileName) finally: if os.path.exists(fileName): os.unlink(fileName) img = backgrounds.getImage() # # Check that the read-back image is identical to that generated from the backgroundList # round-tripped to disk # backImage -= img self.assertEqual(np.min(backImage.getArray()), 0.0) self.assertEqual(np.max(backImage.getArray()), 0.0)
def testBackgroundListIO(self): """Test I/O for BackgroundLists""" bgCtrl = afwMath.BackgroundControl(10, 10) interpStyle = afwMath.Interpolate.AKIMA_SPLINE undersampleStyle = afwMath.REDUCE_INTERP_ORDER approxOrderX = 6 approxOrderY = 6 approxWeighting = True im = self.image.Factory(self.image, self.image.getBBox()) arr = im.getArray() arr += np.random.normal(size=(im.getHeight(), im.getWidth())) for astyle in afwMath.ApproximateControl.UNKNOWN, afwMath.ApproximateControl.CHEBYSHEV: actrl = afwMath.ApproximateControl(astyle, approxOrderX) bgCtrl.setApproximateControl(actrl) backgroundList = afwMath.BackgroundList() backImage = afwImage.ImageF(im.getDimensions()) for i in range(2): bkgd = afwMath.makeBackground(im, bgCtrl) if i == 0: # no need to call getImage backgroundList.append( (bkgd, interpStyle, undersampleStyle, astyle, approxOrderX, approxOrderY, approxWeighting)) else: # Relies on having called getImage; deprecated with self.assertWarns(FutureWarning): backgroundList.append(bkgd) backImage += bkgd.getImageF(interpStyle, undersampleStyle) with lsst.utils.tests.getTempFilePath(".fits") as fileName: backgroundList.writeFits(fileName) backgrounds = afwMath.BackgroundList.readFits(fileName) img = backgrounds.getImage() # Check that the read-back image is identical to that generated from the backgroundList # round-tripped to disk backImage -= img self.assertEqual(np.min(backImage.getArray()), 0.0) self.assertEqual(np.max(backImage.getArray()), 0.0)
def main(): image = getImage() if display: ds9.mtv(image, frame=0) bkgd = simpleBackground(image) image = getImage() bkgd = complexBackground(image) if display: ds9.mtv(image, frame=1) ds9.mtv(bkgd.getStatsImage(), frame=2) order = 2 actrl = afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, order, order) approx = bkgd.getApproximate(actrl) approx.getImage() approx.getMaskedImage() approx.getImage(order - 1)
def main(): image = getImage() if display: afwDisplay.Display(frame=0).mtv(image, title="Image") bkgd = simpleBackground(image) image = getImage() bkgd = complexBackground(image) if display: afwDisplay.Display(frame=1).mtv(image, title="image") afwDisplay.Display(frame=2).mtv(bkgd.getStatsImage(), title="background") order = 2 actrl = afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, order, order) approx = bkgd.getApproximate(actrl) approx.getImage() approx.getMaskedImage() approx.getImage(order - 1)
def fitBackground(self, maskedImage, nx=0, ny=0, algorithm=None): """!Estimate the background of a masked image @param[in] maskedImage masked image whose background is to be computed @param[in] nx number of x bands; if 0 compute from width and config.binSizeX @param[in] ny number of y bands; if 0 compute from height and config.binSizeY @param[in] algorithm name of interpolation algorithm; if None use self.config.algorithm @return fit background as an lsst.afw.math.Background @throw RuntimeError if lsst.afw.math.makeBackground returns None, which is apparently one way it indicates failure """ binSizeX = self.config.binSize if self.config.binSizeX == 0 else self.config.binSizeX binSizeY = self.config.binSize if self.config.binSizeY == 0 else self.config.binSizeY if not nx: nx = maskedImage.getWidth() // binSizeX + 1 if not ny: ny = maskedImage.getHeight() // binSizeY + 1 unsubFrame = getDebugFrame(self._display, "unsubtracted") if unsubFrame: unsubDisp = afwDisplay.getDisplay(frame=unsubFrame) unsubDisp.mtv(maskedImage, title="unsubtracted") xPosts = numpy.rint( numpy.linspace(0, maskedImage.getWidth() + 1, num=nx, endpoint=True)) yPosts = numpy.rint( numpy.linspace(0, maskedImage.getHeight() + 1, num=ny, endpoint=True)) with unsubDisp.Buffering(): for (xMin, xMax), (yMin, yMax) in itertools.product( zip(xPosts[:-1], xPosts[1:]), zip(yPosts[:-1], yPosts[1:])): unsubDisp.line([(xMin, yMin), (xMin, yMax), (xMax, yMax), (xMax, yMin), (xMin, yMin)]) sctrl = afwMath.StatisticsControl() badMask = maskedImage.mask.getPlaneBitMask( self.config.ignoredPixelMask) sctrl.setAndMask(badMask) sctrl.setNanSafe(self.config.isNanSafe) self.log.debug("Ignoring mask planes: %s" % ", ".join(self.config.ignoredPixelMask)) if (maskedImage.mask.getArray() & badMask).all(): raise pipeBase.TaskError( "All pixels masked. Cannot estimate background") if algorithm is None: algorithm = self.config.algorithm # TODO: DM-22814. This call to a deprecated BackgroundControl constructor # is necessary to support the algorithm parameter; it # should be replaced with # # afwMath.BackgroundControl(nx, ny, sctrl, self.config.statisticsProperty) # # when algorithm has been deprecated and removed. with suppress_deprecations(): bctrl = afwMath.BackgroundControl(algorithm, nx, ny, self.config.undersampleStyle, sctrl, self.config.statisticsProperty) # TODO: The following check should really be done within lsst.afw.math. # With the current code structure, it would need to be accounted for in the doGetImage() # function in BackgroundMI.cc (which currently only checks against the interpolation settings, # which is not appropriate when useApprox=True) # and/or the makeApproximate() function in afw/Approximate.cc. # See ticket DM-2920: "Clean up code in afw for Approximate background # estimation" (which includes a note to remove the following and the # similar checks in pipe_tasks/matchBackgrounds.py once implemented) # # Check that config setting of approxOrder/binSize make sense # (i.e. ngrid (= shortDimension/binSize) > approxOrderX) and perform # appropriate undersampleStlye behavior. if self.config.useApprox: if self.config.approxOrderY not in (self.config.approxOrderX, -1): raise ValueError( "Error: approxOrderY not in (approxOrderX, -1)") order = self.config.approxOrderX minNumberGridPoints = order + 1 if min(nx, ny) <= order: self.log.warn( "Too few points in grid to constrain fit: min(nx, ny) < approxOrder) " "[min(%d, %d) < %d]" % (nx, ny, order)) if self.config.undersampleStyle == "THROW_EXCEPTION": raise ValueError( "Too few points in grid (%d, %d) for order (%d) and binSize (%d, %d)" % (nx, ny, order, binSizeX, binSizeY)) elif self.config.undersampleStyle == "REDUCE_INTERP_ORDER": if order < 1: raise ValueError( "Cannot reduce approxOrder below 0. " "Try using undersampleStyle = \"INCREASE_NXNYSAMPLE\" instead?" ) order = min(nx, ny) - 1 self.log.warn("Reducing approxOrder to %d" % order) elif self.config.undersampleStyle == "INCREASE_NXNYSAMPLE": # Reduce bin size to the largest acceptable square bins newBinSize = min( maskedImage.getWidth(), maskedImage.getHeight()) // (minNumberGridPoints - 1) if newBinSize < 1: raise ValueError("Binsize must be greater than 0") newNx = maskedImage.getWidth() // newBinSize + 1 newNy = maskedImage.getHeight() // newBinSize + 1 bctrl.setNxSample(newNx) bctrl.setNySample(newNy) self.log.warn( "Decreasing binSize from (%d, %d) to %d for a grid of (%d, %d)" % (binSizeX, binSizeY, newBinSize, newNx, newNy)) actrl = afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, order, order, self.config.weighting) bctrl.setApproximateControl(actrl) bg = afwMath.makeBackground(maskedImage, bctrl) if bg is None: raise RuntimeError( "lsst.afw.math.makeBackground failed to fit a background model" ) return bg
def matchBackgrounds(self, refExposure, sciExposure): """ Match science exposure's background level to that of reference exposure. Process creates a difference image of the reference exposure minus the science exposure, and then generates an afw.math.Background object. It assumes (but does not require/check) that the mask plane already has detections set. If detections have not been set/masked, sources will bias the background estimation. The 'background' of the difference image is smoothed by spline interpolation (by the Background class) or by polynomial interpolation by the Approximate class. This model of difference image is added to the science exposure in memory. Fit diagnostics are also calculated and returned. @param[in] refExposure: reference exposure @param[in,out] sciExposure: science exposure; modified by changing the background level to match that of the reference exposure @returns a pipBase.Struct with fields: - backgroundModel: an afw.math.Approximate or an afw.math.Background. - fitRMS: rms of the fit. This is the sqrt(mean(residuals**2)). - matchedMSE: the MSE of the reference and matched images: mean((refImage - matchedSciImage)**2); should be comparable to difference image's mean variance. - diffImVar: the mean variance of the difference image. """ if lsstDebug.Info(__name__).savefits: refExposure.writeFits( lsstDebug.Info(__name__).figpath + 'refExposure.fits') sciExposure.writeFits( lsstDebug.Info(__name__).figpath + 'sciExposure.fits') # Check Configs for polynomials: if self.config.usePolynomial: x, y = sciExposure.getDimensions() shortSideLength = min(x, y) if shortSideLength < self.config.binSize: raise ValueError( "%d = config.binSize > shorter dimension = %d" % (self.config.binSize, shortSideLength)) npoints = shortSideLength // self.config.binSize if shortSideLength % self.config.binSize != 0: npoints += 1 if self.config.order > npoints - 1: raise ValueError("%d = config.order > npoints - 1 = %d" % (self.config.order, npoints - 1)) # Check that exposures are same shape if (sciExposure.getDimensions() != refExposure.getDimensions()): wSci, hSci = sciExposure.getDimensions() wRef, hRef = refExposure.getDimensions() raise RuntimeError( "Exposures are different dimensions. sci:(%i, %i) vs. ref:(%i, %i)" % (wSci, hSci, wRef, hRef)) statsFlag = getattr(afwMath, self.config.gridStatistic) self.sctrl.setNumSigmaClip(self.config.numSigmaClip) self.sctrl.setNumIter(self.config.numIter) im = refExposure.getMaskedImage() diffMI = im.Factory(im, True) diffMI -= sciExposure.getMaskedImage() width = diffMI.getWidth() height = diffMI.getHeight() nx = width // self.config.binSize if width % self.config.binSize != 0: nx += 1 ny = height // self.config.binSize if height % self.config.binSize != 0: ny += 1 bctrl = afwMath.BackgroundControl(nx, ny, self.sctrl, statsFlag) bctrl.setUndersampleStyle(self.config.undersampleStyle) bctrl.setInterpStyle(self.config.interpStyle) bkgd = afwMath.makeBackground(diffMI, bctrl) # Some config and input checks if config.usePolynomial: # 1) Check that order/bin size make sense: # 2) Change binsize or order if underconstrained. if self.config.usePolynomial: order = self.config.order bgX, bgY, bgZ, bgdZ = self._gridImage(diffMI, self.config.binSize, statsFlag) minNumberGridPoints = min(len(set(bgX)), len(set(bgY))) if len(bgZ) == 0: raise ValueError("No overlap with reference. Nothing to match") elif minNumberGridPoints <= self.config.order: # must either lower order or raise number of bins or throw exception if self.config.undersampleStyle == "THROW_EXCEPTION": raise ValueError( "Image does not cover enough of ref image for order and binsize" ) elif self.config.undersampleStyle == "REDUCE_INTERP_ORDER": self.log.warn("Reducing order to %d" % (minNumberGridPoints - 1)) order = minNumberGridPoints - 1 elif self.config.undersampleStyle == "INCREASE_NXNYSAMPLE": newBinSize = (minNumberGridPoints * self.config.binSize ) // (self.config.order + 1) bctrl.setNxSample(newBinSize) bctrl.setNySample(newBinSize) bkgd = afwMath.makeBackground(diffMI, bctrl) # do over self.log.warn("Decreasing binsize to %d" % (newBinSize)) # If there is no variance in any image pixels, do not weight bins by inverse variance isUniformImageDiff = not numpy.any( bgdZ > self.config.gridStdevEpsilon) weightByInverseVariance = False if isUniformImageDiff else self.config.approxWeighting # Add offset to sciExposure try: if self.config.usePolynomial: actrl = afwMath.ApproximateControl( afwMath.ApproximateControl.CHEBYSHEV, order, order, weightByInverseVariance) undersampleStyle = getattr(afwMath, self.config.undersampleStyle) approx = bkgd.getApproximate(actrl, undersampleStyle) bkgdImage = approx.getImage() else: bkgdImage = bkgd.getImageF() except Exception as e: raise RuntimeError( "Background/Approximation failed to interp image %s: %s" % (self.debugDataIdString, e)) sciMI = sciExposure.getMaskedImage() sciMI += bkgdImage del sciMI # Need RMS from fit: 2895 will replace this: rms = 0.0 X, Y, Z, dZ = self._gridImage(diffMI, self.config.binSize, statsFlag) x0, y0 = diffMI.getXY0() modelValueArr = numpy.empty(len(Z)) for i in range(len(X)): modelValueArr[i] = bkgdImage.get(int(X[i] - x0), int(Y[i] - y0)) resids = Z - modelValueArr rms = numpy.sqrt(numpy.mean(resids[~numpy.isnan(resids)]**2)) if lsstDebug.Info(__name__).savefits: sciExposure.writeFits( lsstDebug.Info(__name__).figpath + 'sciMatchedExposure.fits') if lsstDebug.Info(__name__).savefig: bbox = afwGeom.Box2D(refExposure.getMaskedImage().getBBox()) try: self._debugPlot(X, Y, Z, dZ, bkgdImage, bbox, modelValueArr, resids) except Exception as e: self.log.warn('Debug plot not generated: %s' % (e)) meanVar = afwMath.makeStatistics(diffMI.getVariance(), diffMI.getMask(), afwMath.MEANCLIP, self.sctrl).getValue() diffIm = diffMI.getImage() diffIm -= bkgdImage # diffMI should now have a mean ~ 0 del diffIm mse = afwMath.makeStatistics(diffMI, afwMath.MEANSQUARE, self.sctrl).getValue() outBkgd = approx if self.config.usePolynomial else bkgd return pipeBase.Struct(backgroundModel=outBkgd, fitRMS=rms, matchedMSE=mse, diffImVar=meanVar)