def getBackground(image, backgroundConfig, nx=0, ny=0, algorithm=None): """ Make a new Exposure which is exposure - background """ backgroundConfig.validate() if not nx: nx = image.getWidth() // backgroundConfig.binSize + 1 if not ny: ny = image.getHeight() // backgroundConfig.binSize + 1 sctrl = afwMath.StatisticsControl() sctrl.setAndMask( reduce(lambda x, y: x | image.getMask().getPlaneBitMask(y), backgroundConfig.ignoredPixelMask, 0x0)) sctrl.setNanSafe(backgroundConfig.isNanSafe) pl = pexLogging.Debug("meas.utils.sourceDetection.getBackground") pl.debug( 3, "Ignoring mask planes: %s" % ", ".join(backgroundConfig.ignoredPixelMask)) if not algorithm: algorithm = backgroundConfig.algorithm bctrl = afwMath.BackgroundControl(algorithm, nx, ny, backgroundConfig.undersampleStyle, sctrl, backgroundConfig.statisticsProperty) return afwMath.makeBackground(image, bctrl)
def testNaNFromMaskedImage(self): """Check that an extensively masked image doesn't lead to NaNs in the background estimation""" image = afwImage.MaskedImageF(800, 800) msk = image.getMask() bbox = lsst.geom.BoxI(lsst.geom.PointI(560, 0), lsst.geom.PointI(799, 335)) smsk = msk.Factory(msk, bbox) smsk.set(msk.getPlaneBitMask("DETECTED")) binSize = 256 nx = image.getWidth() // binSize + 1 ny = image.getHeight() // binSize + 1 sctrl = afwMath.StatisticsControl() sctrl.setAndMask( reduce(lambda x, y: x | image.getMask().getPlaneBitMask(y), ['EDGE', 'DETECTED', 'DETECTED_NEGATIVE'], 0x0)) bctrl = afwMath.BackgroundControl(nx, ny, sctrl, "MEANCLIP") bkgd = afwMath.makeBackground(image, bctrl) bkgdImage = bkgd.getImageF("NATURAL_SPLINE", "THROW_EXCEPTION") if debugMode: afwDisplay.Display(frame=0).mtv(image, title=self._testMethodName + " image") afwDisplay.Display(frame=1).mtv(bkgdImage, title=self._testMethodName + " bkgdImage") self.assertFalse(np.isnan(bkgdImage[0, 0, afwImage.LOCAL])) # Check that the non-string API works too bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.THROW_EXCEPTION)
def complexBackground(image): MaskPixel = afwImage.MaskPixel binsize = 128 nx = int(image.getWidth()/binsize) + 1 ny = int(image.getHeight()/binsize) + 1 sctrl = afwMath.StatisticsControl() sctrl.setNumSigmaClip(3) sctrl.setNumIter(4) sctrl.setAndMask(afwImage.Mask[MaskPixel].getPlaneBitMask(["INTRP", "EDGE"])) sctrl.setNoGoodPixelsMask(afwImage.Mask[MaskPixel].getPlaneBitMask("BAD")) sctrl.setNanSafe(True) if False: sctrl.setWeighted(True) sctrl.setCalcErrorFromInputVariance(True) bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP) bkgd = afwMath.makeBackground(image, bctrl) statsImage = bkgd.getStatsImage() ds9.mtv(statsImage.getVariance()) return bkgd
def setUp(self): self.val = 10 self.nRow, self.nCol = 100, 200 self.sctrl = afwMath.StatisticsControl() # Integers self.mimgI = afwImage.MaskedImageI( afwGeom.Extent2I(self.nRow, self.nCol)) self.mimgI.set(self.val, 0x0, self.val) self.imgI = afwImage.ImageI(afwGeom.Extent2I(self.nRow, self.nCol), self.val) # TODO: pybind11, this should probably be ndarray self.vecI = [self.val for i in range(self.nRow * self.nCol)] # floats self.mimgF = afwImage.MaskedImageF( afwGeom.Extent2I(self.nRow, self.nCol)) self.mimgF.set(self.val, 0x0, self.val) self.imgF = afwImage.ImageF(afwGeom.Extent2I(self.nRow, self.nCol), self.val) # TODO: pybind11, this should probably be ndarray self.vecF = [float(self.val) for i in range(self.nRow * self.nCol)] # doubles self.mimgD = afwImage.MaskedImageD( afwGeom.Extent2I(self.nRow, self.nCol)) self.mimgD.set(self.val, 0x0, self.val) self.imgD = afwImage.ImageD(afwGeom.Extent2I(self.nRow, self.nCol), self.val) # TODO: pybind11, this should probably be ndarray self.vecD = [float(self.val) for i in range(self.nRow * self.nCol)] self.imgList = [self.imgI, self.imgF, self.imgD] self.mimgList = [self.mimgI, self.mimgF, self.mimgD] self.vecList = [self.vecI, self.vecF, self.vecD]
def testWeightedSimple(self): mimg = afwImage.MaskedImageF(afwGeom.Extent2I(1, 2)) mimg.set(0, 0, (self.valR, 0x0, self.valR)) mimg.set(0, 1, (self.valL, 0x0, self.valL)) sctrl = afwMath.StatisticsControl() sctrl.setWeighted(True) stats = afwMath.makeStatistics( mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV, sctrl) vsum = 2.0 vsum2 = self.valR + self.valL wsum = 1.0 / self.valR + 1.0 / self.valL wwsum = 1.0 / self.valR**2 + 1.0 / self.valL**2 mean = vsum / wsum variance = vsum2 / wsum - mean**2 # biased variance n = 2 # original estimate; just a rewrite of the usual n/(n - 1) correction stddev = (1.0 * (vsum2) / (wsum * (1.0 - 1.0 / n)) - (vsum**2) / (wsum**2 * (1.0 - 1.0 / n)))**0.5 self.assertAlmostEqual(stddev, np.sqrt(variance * n / (n - 1))) # # The correct formula: stddev = np.sqrt(variance * wsum**2 / (wsum**2 - wwsum)) # get the stats for the image with two values self.assertAlmostEqual(stats.getValue(afwMath.MEAN), mean, 10) self.assertAlmostEqual(stats.getValue(afwMath.STDEV), stddev, 10)
def testBadRows(self): """Test that a bad set of rows in an image doesn't cause a failure""" initialValue = 20 mi = afwImage.MaskedImageF(500, 200) mi.set((initialValue, 0x0, 1.0)) mi.image[:, 0:100] = np.nan badBits = mi.mask.getPlaneBitMask( ['EDGE', 'DETECTED', 'DETECTED_NEGATIVE']) mi.mask[0:400, :] |= badBits if debugMode: afwDisplay.Display(frame=0).mtv(mi, title=self._testMethodName + " image") sctrl = afwMath.StatisticsControl() sctrl.setAndMask(badBits) nx, ny = 17, 17 bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP) bkgd = afwMath.makeBackground(mi, bctrl) statsImage = bkgd.getStatsImage() if debugMode: afwDisplay.Display(frame=1).mtv(statsImage, title=self._testMethodName + " bkgd StatsImage") # the test is that this doesn't fail if the bug (#2297) is fixed for frame, interpStyle in enumerate([afwMath.Interpolate.CONSTANT, afwMath.Interpolate.LINEAR, afwMath.Interpolate.NATURAL_SPLINE, afwMath.Interpolate.AKIMA_SPLINE], 2): bkgdImage = bkgd.getImageF( interpStyle, afwMath.REDUCE_INTERP_ORDER) self.assertEqual( np.mean(bkgdImage[0:100, 0:100].array), initialValue) if debugMode: afwDisplay.Display(frame=frame).mtv(bkgdImage, title=self._testMethodName + " bkgdImage: " + interpStyle.__str__())
def _testBadValue(self, badVal): """Test that we can handle an instance of `badVal` in the data correctly Note that we only test ImageF here (as ImageI can't contain a NaN) """ mean = self.images[0][1] x, y = 10, 10 for useImage in [True, False]: if useImage: image = afwImage.ImageF(100, 100) image.set(mean) image[x, y] = badVal else: image = afwImage.MaskedImageF(100, 100) image.set(mean, 0x0, 1.0) image[x, y] = (badVal, 0x0, 1.0) self.assertEqual( afwMath.makeStatistics(image, afwMath.MAX).getValue(), mean) self.assertEqual( afwMath.makeStatistics(image, afwMath.MEAN).getValue(), mean) sctrl = afwMath.StatisticsControl() sctrl.setNanSafe(False) self.assertFalse( np.isfinite( afwMath.makeStatistics(image, afwMath.MAX, sctrl).getValue())) self.assertFalse( np.isfinite( afwMath.makeStatistics(image, afwMath.MEAN, sctrl).getValue()))
def makeThreshold(self, image, thresholdParity): """Make an afw.detection.Threshold object corresponding to the task's configuration and the statistics of the given image. Parameters ---------- image : `afw.image.MaskedImage` Image to measure noise statistics from if needed. thresholdParity: `str` One of "positive" or "negative", to set the kind of fluctuations the Threshold will detect. """ parity = False if thresholdParity == "negative" else True threshold = afwDet.createThreshold(self.config.thresholdValue, self.config.thresholdType, parity) threshold.setIncludeMultiplier(self.config.includeThresholdMultiplier) if self.config.thresholdType == 'stdev': bad = image.getMask().getPlaneBitMask([ 'BAD', 'SAT', 'EDGE', 'NO_DATA', ]) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(bad) stats = afwMath.makeStatistics(image, afwMath.STDEVCLIP, sctrl) thres = (stats.getValue(afwMath.STDEVCLIP) * self.config.thresholdValue) threshold = afwDet.createThreshold(thres, 'value', parity) threshold.setIncludeMultiplier( self.config.includeThresholdMultiplier) return threshold
def process(self, clipboard): """Reject outliers""" # print "***** process ******" # print "clipboard keys:" # for key in clipboard.getKeys(): # print "*", key maskedImageList = self.getFromClipboard(clipboard, "maskedImageList") print "maskedImageList =", maskedImageList weightList = self.getFromClipboard(clipboard, "weightList", doRaise=False) outlierRejectionPolicy = self.policy.get("outlierRejectionPolicy") allowedMaskPlanes = outlierRejectionPolicy.get("allowedMaskPlanes") badPixelMask = coaddUtils.makeBitMask(allowedMaskPlanes.split(), doInvert=True) statsControl = afwMath.StatisticsControl() statsControl.setNumSigmaClip(outlierRejectionPolicy.get("numSigma")) statsControl.setNumIter(outlierRejectionPolicy.get("numIterations")) statsControl.setAndMask(badPixelMask) statsControl.setNanSafe( False) # we're using masked images, so no need for NaN detection statsControl.setWeighted(True) if weightList != None: coadd = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP, statsControl, weightList) else: coadd = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP, statsControl) self.addToClipboard(clipboard, "coadd", coadd)
def showSetAndMask(): nX, nY = 10, 10 mimg = afwImage.MaskedImageF(afwGeom.Extent2I(nX, nY)) # decide what we believe to be 'bad' mask bits mask = mimg.getMask() maskbitBad = mask.getPlaneBitMask('BAD') maskbitSat = mask.getPlaneBitMask('SAT') maskbits = maskbitBad | maskbitSat # four possibilities ... none, bad, sat, bad+sat # we'll just set the pixel value to equal the mask value for simplicity values = [(0, 0x0, 0), (maskbitBad, maskbitBad, 0), (maskbitSat, maskbitSat, 0), (maskbits, maskbits, 0)] for iY in range(nY): for iX in range(nX): mod = (iY * nX + iX) % len(values) mimg.set(iX, iY, values[mod]) # create our statisticsControl object and change the andMask to reject # the bits we deem bad. masks = [0x0, maskbitBad, maskbitSat, maskbits] explanations = [ "all values accepted ... avg of 0,1,2,3 = 1.5", "reject 'BAD' ... avg of 0,2 = 1.0", "reject 'SAT' ... avg of 0,1 = 0.5", "reject BAD | SAT ... avg of 0 = 0.0" ] for i in range(len(masks)): sctrl = afwMath.StatisticsControl() sctrl.setAndMask(masks[i]) stat = afwMath.makeStatistics(mimg, afwMath.MEAN, sctrl) answer = stat.getValue(afwMath.MEAN) print(explanations[i], " (got %.1f)" % (answer))
def makeThreshold(self, image, thresholdParity, factor=1.0): """Make an afw.detection.Threshold object corresponding to the task's configuration and the statistics of the given image. Parameters ---------- image : `afw.image.MaskedImage` Image to measure noise statistics from if needed. thresholdParity: `str` One of "positive" or "negative", to set the kind of fluctuations the Threshold will detect. factor : `float` Factor by which to multiply the configured detection threshold. This is useful for tweaking the detection threshold slightly. Returns ------- threshold : `lsst.afw.detection.Threshold` Detection threshold. """ parity = False if thresholdParity == "negative" else True thresholdValue = self.config.thresholdValue thresholdType = self.config.thresholdType if self.config.thresholdType == 'stdev': bad = image.getMask().getPlaneBitMask(self.config.statsMask) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(bad) stats = afwMath.makeStatistics(image, afwMath.STDEVCLIP, sctrl) thresholdValue *= stats.getValue(afwMath.STDEVCLIP) thresholdType = 'value' threshold = afwDet.createThreshold(thresholdValue*factor, thresholdType, parity) threshold.setIncludeMultiplier(self.config.includeThresholdMultiplier) return threshold
def computeClippedAfwStats(im, numSigmaClip=3., numIter=3, maskIm=None): """! Utility function for sigma-clipped array statistics on an image or exposure. @param im An afw.Exposure, masked image, or image. @return sigma-clipped mean, std, and variance of input array """ statsControl = afwMath.StatisticsControl() statsControl.setNumSigmaClip(numSigmaClip) statsControl.setNumIter(numIter) ignoreMaskPlanes = [ "INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE" ] statsControl.setAndMask(afwImage.MaskU.getPlaneBitMask(ignoreMaskPlanes)) if maskIm is None: statObj = afwMath.makeStatistics( im, afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.VARIANCECLIP, statsControl) else: statObj = afwMath.makeStatistics( im, maskIm, afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.VARIANCECLIP, statsControl) mean = statObj.getValue(afwMath.MEANCLIP) std = statObj.getValue(afwMath.STDEVCLIP) var = statObj.getValue(afwMath.VARIANCECLIP) return mean, std, var
def writeThumbnail(self, dataRef, dataset, exposure): """Write out exposure to a snapshot file named outfile in the given size. """ filename = dataRef.get(dataset + "_filename")[0] directory = os.path.dirname(filename) if not os.path.exists(directory): try: os.makedirs(directory) except OSError as e: # Don't fail if directory exists due to race if e.errno != errno.EEXIST: raise e binning = self.config.thumbnailBinning binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning, binning, afwMath.MEAN) statsCtrl = afwMath.StatisticsControl() statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(["SAT", "BAD", "INTRP"])) stats = afwMath.makeStatistics(binnedImage, afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX, statsCtrl) low = stats.getValue(afwMath.MEDIAN) - self.config.thumbnailStdev*stats.getValue(afwMath.STDEVCLIP) try: makeRGB(binnedImage, binnedImage, binnedImage, minimum=low, dataRange=self.config.thumbnailRange, Q=self.config.thumbnailQ, fileName=filename, saturatedBorderWidth=self.config.thumbnailSatBorder, saturatedPixelValue=stats.getValue(afwMath.MAX)) except Exception as exc: # makeRGB can fail with: # SystemError: <built-in method flush of _io.BufferedWriter object at 0x2b2a0081c938> # returned a result with an error set # This unhelpful error comes from the bowels of matplotlib, so not much we can do about it # except keep it from being fatal. self.log.warn("Unable to write thumbnail for %s: %s", dataRef.dataId, exc)
def run(self, exposureOrImage): """Measure a particular statistic on an image (of some sort). Parameters ---------- exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image` Exposure or image to calculate statistics on. Returns ------- results : float Resulting statistic value. """ stats = afwMath.StatisticsControl( self.config.clip, self.config.nIter, afwImage.Mask.getPlaneBitMask(self.config.mask)) try: image = exposureOrImage.getMaskedImage() except Exception: try: image = exposureOrImage.getImage() except Exception: image = exposureOrImage return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
def __init__(self, *args, **kwargs): pipeBase.Task.__init__(self, *args, **kwargs) self.sctrl = afwMath.StatisticsControl() self.sctrl.setAndMask( afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes)) self.sctrl.setNanSafe(True)
def _testBadValue(self, badVal): """Test that we can handle an instance of `badVal` in the data correctly""" x, y = 10, 10 for useImage in [True, False]: if useImage: self.image = afwImage.ImageF(100, 100) self.image.set(self.val) self.image.set(x, y, badVal) else: self.image = afwImage.MaskedImageF(100, 100) self.image.set(self.val, 0x0, 1.0) self.image.set(x, y, (badVal, 0x0, 1.0)) self.assertEqual( afwMath.makeStatistics(self.image, afwMath.MAX).getValue(), self.val) self.assertEqual( afwMath.makeStatistics(self.image, afwMath.MEAN).getValue(), self.val) sctrl = afwMath.StatisticsControl() sctrl.setNanSafe(False) self.assertFalse( np.isfinite( afwMath.makeStatistics(self.image, afwMath.MAX, sctrl).getValue())) self.assertFalse( np.isfinite( afwMath.makeStatistics(self.image, afwMath.MEAN, sctrl).getValue()))
def testWeightedStats(self): """Test that bug from #1697 (weighted stats returning NaN) stays fixed.""" rand = afwMath.Random() mu = 10000 edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE") badPixelMask = afwImage.MaskU.getPlaneBitMask("EDGE") statsCtrl = afwMath.StatisticsControl() statsCtrl.setNumSigmaClip(3.0) statsCtrl.setNumIter(2) statsCtrl.setAndMask(badPixelMask) for weight in (300.0, 10.0, 1.0): print "Testing with weight=%0.1f" % (weight,) maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack weightList = [] nx, ny = 256, 256 for i in range(3): print "Processing ", i maskedImage = afwImage.MaskedImageF(nx, ny) maskedImageList.append(maskedImage) afwMath.randomPoissonImage(maskedImage.getImage(), rand, mu) maskedImage.getVariance().set(mu) weightList.append(weight) self.reportBadPixels(maskedImage, badPixelMask) print "Stack: ", coaddMaskedImage = afwMath.statisticsStack( maskedImageList, afwMath.MEANCLIP, statsCtrl, weightList) self.reportBadPixels(coaddMaskedImage, badPixelMask)
def testTicket1123(self): """ Ticket #1123 reported that the Statistics stack routine throws an exception when all pixels in a stack are masked. Returning a NaN pixel in the stack is preferred """ ctrl = afwMath.StatisticsControl() ctrl.setAndMask(~0x0) mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10)) mimg.set([self.val, 0x1, self.val]) # test the case with no valid pixels ... both mean and stdev should be nan stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl) mean = stat.getValue(afwMath.MEAN) stdev = stat.getValue(afwMath.STDEV) self.assertNotEqual(mean, mean) # NaN does not equal itself self.assertNotEqual(stdev, stdev) # NaN does not equal itself # test the case with one valid pixel ... mean is ok, but stdev should still be nan mimg.getMask().set(1, 1, 0x0) stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl) mean = stat.getValue(afwMath.MEAN) stdev = stat.getValue(afwMath.STDEV) self.assertEqual(mean, self.val) self.assertNotEqual(stdev, stdev) # NaN does not equal itself # test the case with two valid pixels ... both mean and stdev are ok mimg.getMask().set(1, 2, 0x0) stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl) mean = stat.getValue(afwMath.MEAN) stdev = stat.getValue(afwMath.STDEV) self.assertEqual(mean, self.val) self.assertEqual(stdev, 0.0)
def __init__(self, bbox, wcs, badMaskPlanes, logName="coadd.utils.Coadd"): """ Creating a coadd Parameters ---------- bbox : `lsst.afw.geom.Box2I` bounding box of coadd Exposure with respect to parent coadd dimensions = bbox.getDimensions(); xy0 = bbox.getMin() wcs : `lsst.afw.geom.SKyWcs` WCS of coadd exposure badMaskPlanes : `Collection` mask planes to pay attention to when rejecting masked pixels. Specify as a collection of names. badMaskPlanes should always include "NO_DATA". logName : `str` name by which messages are logged """ self._log = log.getLogger(logName) self._bbox = bbox self._wcs = wcs self._badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes) self._coadd = afwImage.ExposureF(bbox, wcs) self._weightMap = afwImage.ImageF(bbox) # dict of filter name: filter object for all filters seen so far self._filterDict = dict() self._statsControl = afwMath.StatisticsControl() self._statsControl.setNumSigmaClip(3.0) self._statsControl.setNumIter(2) self._statsControl.setAndMask(self._badPixelMask)
def setUp(self): self.val = 10 self.nRow, self.nCol = 100, 200 self.sctrl = afwMath.StatisticsControl() # Integers self.mimgI = afwImage.MaskedImageI(afwGeom.Extent2I(self.nRow, self.nCol)) self.mimgI.set(self.val, 0x0, self.val) self.imgI = afwImage.ImageI(afwGeom.Extent2I(self.nRow, self.nCol), self.val) self.vecI = afwMath.vectorI(self.nRow*self.nCol, self.val) # floats self.mimgF = afwImage.MaskedImageF(afwGeom.Extent2I(self.nRow, self.nCol)) self.mimgF.set(self.val, 0x0, self.val) self.imgF = afwImage.ImageF(afwGeom.Extent2I(self.nRow, self.nCol), self.val) self.vecF = afwMath.vectorF(self.nRow*self.nCol, self.val) # doubles self.mimgD = afwImage.MaskedImageD(afwGeom.Extent2I(self.nRow, self.nCol)) self.mimgD.set(self.val, 0x0, self.val) self.imgD = afwImage.ImageD(afwGeom.Extent2I(self.nRow, self.nCol), self.val) self.vecD = afwMath.vectorD(self.nRow*self.nCol, self.val) self.imgList = [self.imgI, self.imgF, self.imgD] self.mimgList = [self.mimgI, self.mimgF, self.mimgD] self.vecList = [self.vecI, self.vecF, self.vecD]
def testErrorsFromVariance(self): """Test that we can estimate the errors from the incoming variances""" weight, mean, variance = 0.1, 1.0, 10.0 ctrl = afwMath.StatisticsControl() mi = afwImage.MaskedImageF(lsst.geom.Extent2I(10, 10)) npix = 10 * 10 mi.getImage().set(mean) mi.getVariance().set(variance) weights = afwImage.ImageF(mi.getDimensions()) weights.set(weight) ctrl.setCalcErrorFromInputVariance(True) weighted = afwMath.makeStatistics( mi, weights, afwMath.MEAN | afwMath.MEANCLIP | afwMath.SUM | afwMath.ERRORS, ctrl) self.assertAlmostEqual( weighted.getValue(afwMath.SUM) / (npix * mean * weight), 1) self.assertAlmostEqual(weighted.getValue(afwMath.MEAN), mean) self.assertAlmostEqual( weighted.getError(afwMath.MEAN)**2, variance / npix) self.assertAlmostEqual( weighted.getError(afwMath.MEANCLIP)**2, variance / npix)
def setBadRegions(self, exposure): """Set all BAD areas of the chip to the average of the rest of the exposure @param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels """ if self.config.badStatistic == "MEDIAN": statistic = afwMath.MEDIAN elif self.config.badStatistic == "MEANCLIP": statistic = afwMath.MEANCLIP else: raise RuntimeError( "Impossible method %s of bad region correction" % self.config.badStatistic) mi = exposure.getMaskedImage() mask = mi.getMask() BAD = mask.getPlaneBitMask("BAD") INTRP = mask.getPlaneBitMask("INTRP") sctrl = afwMath.StatisticsControl() sctrl.setAndMask(BAD) value = afwMath.makeStatistics(mi, statistic, sctrl).getValue() maskArray = mask.getArray() imageArray = mi.getImage().getArray() badPixels = numpy.logical_and((maskArray & BAD) > 0, (maskArray & INTRP) == 0) imageArray[:] = numpy.where(badPixels, value, imageArray) self.log.info("Set %d BAD pixels to %.2f" % (badPixels.sum(), value))
def testMasked(self): # get the stats for the image with two values self.mimgR.set(self.valR, 0x0, self.valR) stats = afwMath.makeStatistics( self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV) mean = 0.5 * (self.valL + self.valR) nL, nR = self.mimgL.getWidth() * self.mimgL.getHeight( ), self.mimgR.getWidth() * self.mimgR.getHeight() stdev = ((nL * (self.valL - mean)**2 + nR * (self.valR - mean)**2) / (nL + nR - 1))**0.5 self.assertEqual(stats.getValue(afwMath.NPOINT), self.n) self.assertEqual(stats.getValue(afwMath.MEAN), mean) self.assertEqual(stats.getValue(afwMath.STDEV), stdev) # set the right side Mask and the StatisticsControl andMask to 0x1 # Stats should be just for the left side! maskBit = 0x1 self.mimgR.getMask().set(maskBit) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(maskBit) statsNaN = afwMath.makeStatistics( self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV, sctrl) mean = self.valL stdev = 0.0 self.assertEqual(statsNaN.getValue(afwMath.NPOINT), nL) self.assertEqual(statsNaN.getValue(afwMath.MEAN), mean) self.assertEqual(statsNaN.getValue(afwMath.STDEV), stdev)
def writeThumbnail(self, dataRef, dataset, exposure): """Write out exposure to a snapshot file named outfile in the given size. """ filename = dataRef.get(dataset + "_filename")[0] directory = os.path.dirname(filename) if not os.path.exists(directory): try: os.makedirs(directory) except OSError as e: # Don't fail if directory exists due to race if e.errno != errno.EEXIST: raise e binning = self.config.thumbnailBinning binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning, binning, afwMath.MEAN) statsCtrl = afwMath.StatisticsControl() statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask( ["SAT", "BAD", "INTRP"])) stats = afwMath.makeStatistics( binnedImage, afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX, statsCtrl) low = stats.getValue( afwMath.MEDIAN) - self.config.thumbnailStdev * stats.getValue( afwMath.STDEVCLIP) makeRGB(binnedImage, binnedImage, binnedImage, minimum=low, dataRange=self.config.thumbnailRange, Q=self.config.thumbnailQ, fileName=filename, saturatedBorderWidth=self.config.thumbnailSatBorder, saturatedPixelValue=stats.getValue(afwMath.MAX))
def testBadRows(self): """Test that a bad set of rows in an image doesn't cause a failure """ initialValue = 20 mi = afwImage.MaskedImageF(500, 200) mi.set((initialValue, 0x0, 1.0)) im = mi.getImage() im[:, 0:100] = np.nan del im msk = mi.getMask() badBits = msk.getPlaneBitMask(['EDGE', 'DETECTED', 'DETECTED_NEGATIVE']) msk[0:400, :] |= badBits del msk if display: ds9.mtv(mi, frame=0) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(badBits) nx, ny = 17, 17 bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP) bkgd = afwMath.makeBackground(mi, bctrl) statsImage = afwMath.cast_BackgroundMI(bkgd).getStatsImage() if display: ds9.mtv(statsImage, frame=1) # the test is that this doesn't fail if the bug (#2297) is fixed bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.REDUCE_INTERP_ORDER) self.assertEqual(np.mean(bkgdImage[0:100, 0:100].getArray()), initialValue) if display: ds9.mtv(bkgdImage, frame=2)
def __init__(self, *args, **kwargs): pipeBase.Task.__init__(self, *args, **kwargs) self._badPixelMask = Mask.getPlaneBitMask(self.config.badMaskPlanes) self._statsControl = afwMath.StatisticsControl() self._statsControl.setNumSigmaClip(self.config.numSigmaClip) self._statsControl.setNumIter(self.config.numIter) self._statsControl.setAndMask(self._badPixelMask)
def testBadImage(self): """Test that an entirely bad image doesn't cause an absolute failure""" initialValue = 20 mi = afwImage.MaskedImageF(500, 200) # Check that no good values don't crash (they return NaN), and that a single good value # is enough to redeem the entire image for pix00 in [np.nan, initialValue]: mi.image[:] = np.nan mi.image[0, 0] = pix00 sctrl = afwMath.StatisticsControl() nx, ny = 17, 17 bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP) bkgd = afwMath.makeBackground(mi, bctrl) for interpStyle in [ afwMath.Interpolate.CONSTANT, afwMath.Interpolate.LINEAR, afwMath.Interpolate.NATURAL_SPLINE, afwMath.Interpolate.AKIMA_SPLINE ]: # the test is that this doesn't fail if the bug (#2297) is # fixed bkgdImage = bkgd.getImageF(interpStyle, afwMath.REDUCE_INTERP_ORDER) val = np.mean(bkgdImage[0:100, 0:100].array) if np.isfinite(pix00): self.assertEqual(val, pix00) else: self.assertTrue(np.isnan(val))
def main(): gaussFunction = afwMath.GaussianFunction2D(3, 2, 0.5) gaussKernel = afwMath.AnalyticKernel(10, 10, gaussFunction) inImage = afwImage.ImageF(afwGeom.Extent2I(100, 100)) inImage.set(1) if disp: ds9.mtv(inImage, frame = 0) # works outImage = afwImage.ImageF(afwGeom.Extent2I(100, 100)) afwMath.convolve(outImage, inImage, gaussKernel, False, True) if disp: ds9.mtv(outImage, frame = 1) print("Should be a number: ", afwMath.makeStatistics( outImage, afwMath.MEAN).getValue()) print("Should be a number: ", afwMath.makeStatistics( outImage, afwMath.STDEV).getValue()) # not works ... now does work outImage = afwImage.ImageF(afwGeom.Extent2I(100, 100)) afwMath.convolve(outImage, inImage, gaussKernel, False, False) if disp: ds9.mtv(outImage, frame = 2) print("Should be a number: ", afwMath.makeStatistics( outImage, afwMath.MEAN).getValue()) print("Should be a number: ", afwMath.makeStatistics( outImage, afwMath.STDEV).getValue()) # This will print nan sctrl = afwMath.StatisticsControl() sctrl.setNanSafe(False) print ("Should be a nan (nanSafe set to False): " + str(afwMath.makeStatistics(outImage, afwMath.MEAN, sctrl).getValue()))
def thresholdImage(self, image, thresholdParity, maskName="DETECTED"): """!Threshold the convolved image, returning a FootprintSet. Helper function for detect(). \param image The (optionally convolved) MaskedImage to threshold \param thresholdParity Parity of threshold \param maskName Name of mask to set \return FootprintSet """ parity = False if thresholdParity == "negative" else True threshold = afwDet.createThreshold(self.config.thresholdValue, self.config.thresholdType, parity) threshold.setIncludeMultiplier(self.config.includeThresholdMultiplier) if self.config.thresholdType == 'stdev': bad = image.getMask().getPlaneBitMask([ 'BAD', 'SAT', 'EDGE', 'NO_DATA', ]) sctrl = afwMath.StatisticsControl() sctrl.setAndMask(bad) stats = afwMath.makeStatistics(image, afwMath.STDEVCLIP, sctrl) thres = stats.getValue( afwMath.STDEVCLIP) * self.config.thresholdValue threshold = afwDet.createThreshold(thres, 'value', parity) threshold.setIncludeMultiplier( self.config.includeThresholdMultiplier) fpSet = afwDet.FootprintSet(image, threshold, maskName, self.config.minPixels) return fpSet
def __init__(self, imfile, mask_files=(), bias_frame=None, applyMasks=True, linearity_correction=None): super(MaskedCCD, self).__init__() self.imfile = imfile self.md = imutils.Metadata(imfile) self.amp_geom = makeAmplifierGeometry(imfile) all_amps = imutils.allAmps(imfile) for amp in all_amps: image = afwImage.ImageF(imfile, imutils.dm_hdu(amp)) mask = afwImage_Mask(image.getDimensions()) self[amp] = afwImage.MaskedImageF(image, mask) self._added_mask_types = [] for mask_file in mask_files: self.add_masks(mask_file) self.stat_ctrl = afwMath.StatisticsControl() if mask_files: self.setAllMasks() if bias_frame is not None: self.bias_frame = MaskedCCD(bias_frame) else: self.bias_frame = None self._applyMasks = applyMasks self._linearity_correction = linearity_correction