def getBackground(image, backgroundConfig, nx=0, ny=0, algorithm=None):
"""
Make a new Exposure which is exposure - background
"""
backgroundConfig.validate()
if not nx:
nx = image.getWidth() // backgroundConfig.binSize + 1
if not ny:
ny = image.getHeight() // backgroundConfig.binSize + 1
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(
reduce(lambda x, y: x | image.getMask().getPlaneBitMask(y),
backgroundConfig.ignoredPixelMask, 0x0))
sctrl.setNanSafe(backgroundConfig.isNanSafe)
pl = pexLogging.Debug("meas.utils.sourceDetection.getBackground")
pl.debug(
3, "Ignoring mask planes: %s" %
", ".join(backgroundConfig.ignoredPixelMask))
if not algorithm:
algorithm = backgroundConfig.algorithm
bctrl = afwMath.BackgroundControl(algorithm, nx, ny,
backgroundConfig.undersampleStyle, sctrl,
backgroundConfig.statisticsProperty)
return afwMath.makeBackground(image, bctrl)
def testNaNFromMaskedImage(self):
"""Check that an extensively masked image doesn't lead to NaNs in the background estimation"""
image = afwImage.MaskedImageF(800, 800)
msk = image.getMask()
bbox = lsst.geom.BoxI(lsst.geom.PointI(560, 0),
lsst.geom.PointI(799, 335))
smsk = msk.Factory(msk, bbox)
smsk.set(msk.getPlaneBitMask("DETECTED"))
binSize = 256
nx = image.getWidth() // binSize + 1
ny = image.getHeight() // binSize + 1
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(
reduce(lambda x, y: x | image.getMask().getPlaneBitMask(y),
['EDGE', 'DETECTED', 'DETECTED_NEGATIVE'], 0x0))
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, "MEANCLIP")
bkgd = afwMath.makeBackground(image, bctrl)
bkgdImage = bkgd.getImageF("NATURAL_SPLINE", "THROW_EXCEPTION")
if debugMode:
afwDisplay.Display(frame=0).mtv(image,
title=self._testMethodName +
" image")
afwDisplay.Display(frame=1).mtv(bkgdImage,
title=self._testMethodName +
" bkgdImage")
self.assertFalse(np.isnan(bkgdImage[0, 0, afwImage.LOCAL]))
# Check that the non-string API works too
bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE,
afwMath.THROW_EXCEPTION)
def complexBackground(image):
MaskPixel = afwImage.MaskPixel
binsize = 128
nx = int(image.getWidth()/binsize) + 1
ny = int(image.getHeight()/binsize) + 1
sctrl = afwMath.StatisticsControl()
sctrl.setNumSigmaClip(3)
sctrl.setNumIter(4)
sctrl.setAndMask(afwImage.Mask[MaskPixel].getPlaneBitMask(["INTRP",
"EDGE"]))
sctrl.setNoGoodPixelsMask(afwImage.Mask[MaskPixel].getPlaneBitMask("BAD"))
sctrl.setNanSafe(True)
if False:
sctrl.setWeighted(True)
sctrl.setCalcErrorFromInputVariance(True)
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
bkgd = afwMath.makeBackground(image, bctrl)
statsImage = bkgd.getStatsImage()
ds9.mtv(statsImage.getVariance())
return bkgd
def setUp(self):
self.val = 10
self.nRow, self.nCol = 100, 200
self.sctrl = afwMath.StatisticsControl()
# Integers
self.mimgI = afwImage.MaskedImageI(
afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgI.set(self.val, 0x0, self.val)
self.imgI = afwImage.ImageI(afwGeom.Extent2I(self.nRow, self.nCol),
self.val)
# TODO: pybind11, this should probably be ndarray
self.vecI = [self.val for i in range(self.nRow * self.nCol)]
# floats
self.mimgF = afwImage.MaskedImageF(
afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgF.set(self.val, 0x0, self.val)
self.imgF = afwImage.ImageF(afwGeom.Extent2I(self.nRow, self.nCol),
self.val)
# TODO: pybind11, this should probably be ndarray
self.vecF = [float(self.val) for i in range(self.nRow * self.nCol)]
# doubles
self.mimgD = afwImage.MaskedImageD(
afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgD.set(self.val, 0x0, self.val)
self.imgD = afwImage.ImageD(afwGeom.Extent2I(self.nRow, self.nCol),
self.val)
# TODO: pybind11, this should probably be ndarray
self.vecD = [float(self.val) for i in range(self.nRow * self.nCol)]
self.imgList = [self.imgI, self.imgF, self.imgD]
self.mimgList = [self.mimgI, self.mimgF, self.mimgD]
self.vecList = [self.vecI, self.vecF, self.vecD]
def testWeightedSimple(self):
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(1, 2))
mimg.set(0, 0, (self.valR, 0x0, self.valR))
mimg.set(0, 1, (self.valL, 0x0, self.valL))
sctrl = afwMath.StatisticsControl()
sctrl.setWeighted(True)
stats = afwMath.makeStatistics(
mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV, sctrl)
vsum = 2.0
vsum2 = self.valR + self.valL
wsum = 1.0 / self.valR + 1.0 / self.valL
wwsum = 1.0 / self.valR**2 + 1.0 / self.valL**2
mean = vsum / wsum
variance = vsum2 / wsum - mean**2 # biased variance
n = 2
# original estimate; just a rewrite of the usual n/(n - 1) correction
stddev = (1.0 * (vsum2) / (wsum * (1.0 - 1.0 / n)) - (vsum**2) /
(wsum**2 * (1.0 - 1.0 / n)))**0.5
self.assertAlmostEqual(stddev, np.sqrt(variance * n / (n - 1)))
#
# The correct formula:
stddev = np.sqrt(variance * wsum**2 / (wsum**2 - wwsum))
# get the stats for the image with two values
self.assertAlmostEqual(stats.getValue(afwMath.MEAN), mean, 10)
self.assertAlmostEqual(stats.getValue(afwMath.STDEV), stddev, 10)
def testBadRows(self):
"""Test that a bad set of rows in an image doesn't cause a failure"""
initialValue = 20
mi = afwImage.MaskedImageF(500, 200)
mi.set((initialValue, 0x0, 1.0))
mi.image[:, 0:100] = np.nan
badBits = mi.mask.getPlaneBitMask(
['EDGE', 'DETECTED', 'DETECTED_NEGATIVE'])
mi.mask[0:400, :] |= badBits
if debugMode:
afwDisplay.Display(frame=0).mtv(mi, title=self._testMethodName + " image")
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(badBits)
nx, ny = 17, 17
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
bkgd = afwMath.makeBackground(mi, bctrl)
statsImage = bkgd.getStatsImage()
if debugMode:
afwDisplay.Display(frame=1).mtv(statsImage, title=self._testMethodName + " bkgd StatsImage")
# the test is that this doesn't fail if the bug (#2297) is fixed
for frame, interpStyle in enumerate([afwMath.Interpolate.CONSTANT, afwMath.Interpolate.LINEAR,
afwMath.Interpolate.NATURAL_SPLINE,
afwMath.Interpolate.AKIMA_SPLINE], 2):
bkgdImage = bkgd.getImageF(
interpStyle, afwMath.REDUCE_INTERP_ORDER)
self.assertEqual(
np.mean(bkgdImage[0:100, 0:100].array), initialValue)
if debugMode:
afwDisplay.Display(frame=frame).mtv(bkgdImage, title=self._testMethodName + " bkgdImage: " +
interpStyle.__str__())
def _testBadValue(self, badVal):
"""Test that we can handle an instance of `badVal` in the data correctly
Note that we only test ImageF here (as ImageI can't contain a NaN)
"""
mean = self.images[0][1]
x, y = 10, 10
for useImage in [True, False]:
if useImage:
image = afwImage.ImageF(100, 100)
image.set(mean)
image[x, y] = badVal
else:
image = afwImage.MaskedImageF(100, 100)
image.set(mean, 0x0, 1.0)
image[x, y] = (badVal, 0x0, 1.0)
self.assertEqual(
afwMath.makeStatistics(image, afwMath.MAX).getValue(), mean)
self.assertEqual(
afwMath.makeStatistics(image, afwMath.MEAN).getValue(), mean)
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
self.assertFalse(
np.isfinite(
afwMath.makeStatistics(image, afwMath.MAX,
sctrl).getValue()))
self.assertFalse(
np.isfinite(
afwMath.makeStatistics(image, afwMath.MEAN,
sctrl).getValue()))
def makeThreshold(self, image, thresholdParity):
"""Make an afw.detection.Threshold object corresponding to the task's
configuration and the statistics of the given image.
Parameters
----------
image : `afw.image.MaskedImage`
Image to measure noise statistics from if needed.
thresholdParity: `str`
One of "positive" or "negative", to set the kind of fluctuations
the Threshold will detect.
"""
parity = False if thresholdParity == "negative" else True
threshold = afwDet.createThreshold(self.config.thresholdValue,
self.config.thresholdType, parity)
threshold.setIncludeMultiplier(self.config.includeThresholdMultiplier)
if self.config.thresholdType == 'stdev':
bad = image.getMask().getPlaneBitMask([
'BAD',
'SAT',
'EDGE',
'NO_DATA',
])
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(bad)
stats = afwMath.makeStatistics(image, afwMath.STDEVCLIP, sctrl)
thres = (stats.getValue(afwMath.STDEVCLIP) *
self.config.thresholdValue)
threshold = afwDet.createThreshold(thres, 'value', parity)
threshold.setIncludeMultiplier(
self.config.includeThresholdMultiplier)
return threshold
def process(self, clipboard):
"""Reject outliers"""
# print "***** process ******"
# print "clipboard keys:"
# for key in clipboard.getKeys():
# print "*", key
maskedImageList = self.getFromClipboard(clipboard, "maskedImageList")
print "maskedImageList =", maskedImageList
weightList = self.getFromClipboard(clipboard,
"weightList",
doRaise=False)
outlierRejectionPolicy = self.policy.get("outlierRejectionPolicy")
allowedMaskPlanes = outlierRejectionPolicy.get("allowedMaskPlanes")
badPixelMask = coaddUtils.makeBitMask(allowedMaskPlanes.split(),
doInvert=True)
statsControl = afwMath.StatisticsControl()
statsControl.setNumSigmaClip(outlierRejectionPolicy.get("numSigma"))
statsControl.setNumIter(outlierRejectionPolicy.get("numIterations"))
statsControl.setAndMask(badPixelMask)
statsControl.setNanSafe(
False) # we're using masked images, so no need for NaN detection
statsControl.setWeighted(True)
if weightList != None:
coadd = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP,
statsControl, weightList)
else:
coadd = afwMath.statisticsStack(maskedImageList, afwMath.MEANCLIP,
statsControl)
self.addToClipboard(clipboard, "coadd", coadd)
def showSetAndMask():
nX, nY = 10, 10
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(nX, nY))
# decide what we believe to be 'bad' mask bits
mask = mimg.getMask()
maskbitBad = mask.getPlaneBitMask('BAD')
maskbitSat = mask.getPlaneBitMask('SAT')
maskbits = maskbitBad | maskbitSat
# four possibilities ... none, bad, sat, bad+sat
# we'll just set the pixel value to equal the mask value for simplicity
values = [(0, 0x0, 0), (maskbitBad, maskbitBad, 0),
(maskbitSat, maskbitSat, 0), (maskbits, maskbits, 0)]
for iY in range(nY):
for iX in range(nX):
mod = (iY * nX + iX) % len(values)
mimg.set(iX, iY, values[mod])
# create our statisticsControl object and change the andMask to reject
# the bits we deem bad.
masks = [0x0, maskbitBad, maskbitSat, maskbits]
explanations = [
"all values accepted ... avg of 0,1,2,3 = 1.5",
"reject 'BAD' ... avg of 0,2 = 1.0",
"reject 'SAT' ... avg of 0,1 = 0.5",
"reject BAD | SAT ... avg of 0 = 0.0"
]
for i in range(len(masks)):
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(masks[i])
stat = afwMath.makeStatistics(mimg, afwMath.MEAN, sctrl)
answer = stat.getValue(afwMath.MEAN)
print(explanations[i], " (got %.1f)" % (answer))
def makeThreshold(self, image, thresholdParity, factor=1.0):
"""Make an afw.detection.Threshold object corresponding to the task's
configuration and the statistics of the given image.
Parameters
----------
image : `afw.image.MaskedImage`
Image to measure noise statistics from if needed.
thresholdParity: `str`
One of "positive" or "negative", to set the kind of fluctuations
the Threshold will detect.
factor : `float`
Factor by which to multiply the configured detection threshold.
This is useful for tweaking the detection threshold slightly.
Returns
-------
threshold : `lsst.afw.detection.Threshold`
Detection threshold.
"""
parity = False if thresholdParity == "negative" else True
thresholdValue = self.config.thresholdValue
thresholdType = self.config.thresholdType
if self.config.thresholdType == 'stdev':
bad = image.getMask().getPlaneBitMask(self.config.statsMask)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(bad)
stats = afwMath.makeStatistics(image, afwMath.STDEVCLIP, sctrl)
thresholdValue *= stats.getValue(afwMath.STDEVCLIP)
thresholdType = 'value'
threshold = afwDet.createThreshold(thresholdValue*factor, thresholdType, parity)
threshold.setIncludeMultiplier(self.config.includeThresholdMultiplier)
return threshold
def computeClippedAfwStats(im, numSigmaClip=3., numIter=3, maskIm=None):
"""! Utility function for sigma-clipped array statistics on an image or exposure.
@param im An afw.Exposure, masked image, or image.
@return sigma-clipped mean, std, and variance of input array
"""
statsControl = afwMath.StatisticsControl()
statsControl.setNumSigmaClip(numSigmaClip)
statsControl.setNumIter(numIter)
ignoreMaskPlanes = [
"INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA",
"DETECTED_NEGATIVE"
]
statsControl.setAndMask(afwImage.MaskU.getPlaneBitMask(ignoreMaskPlanes))
if maskIm is None:
statObj = afwMath.makeStatistics(
im, afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.VARIANCECLIP,
statsControl)
else:
statObj = afwMath.makeStatistics(
im, maskIm,
afwMath.MEANCLIP | afwMath.STDEVCLIP | afwMath.VARIANCECLIP,
statsControl)
mean = statObj.getValue(afwMath.MEANCLIP)
std = statObj.getValue(afwMath.STDEVCLIP)
var = statObj.getValue(afwMath.VARIANCECLIP)
return mean, std, var
def writeThumbnail(self, dataRef, dataset, exposure):
"""Write out exposure to a snapshot file named outfile in the given size.
"""
filename = dataRef.get(dataset + "_filename")[0]
directory = os.path.dirname(filename)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
# Don't fail if directory exists due to race
if e.errno != errno.EEXIST:
raise e
binning = self.config.thumbnailBinning
binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning, binning, afwMath.MEAN)
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(["SAT", "BAD", "INTRP"]))
stats = afwMath.makeStatistics(binnedImage,
afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX, statsCtrl)
low = stats.getValue(afwMath.MEDIAN) - self.config.thumbnailStdev*stats.getValue(afwMath.STDEVCLIP)
try:
makeRGB(binnedImage, binnedImage, binnedImage, minimum=low, dataRange=self.config.thumbnailRange,
Q=self.config.thumbnailQ, fileName=filename,
saturatedBorderWidth=self.config.thumbnailSatBorder,
saturatedPixelValue=stats.getValue(afwMath.MAX))
except Exception as exc:
# makeRGB can fail with:
# SystemError: <built-in method flush of _io.BufferedWriter object at 0x2b2a0081c938>
# returned a result with an error set
# This unhelpful error comes from the bowels of matplotlib, so not much we can do about it
# except keep it from being fatal.
self.log.warn("Unable to write thumbnail for %s: %s", dataRef.dataId, exc)
def run(self, exposureOrImage):
"""Measure a particular statistic on an image (of some sort).
Parameters
----------
exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image`
Exposure or image to calculate statistics on.
Returns
-------
results : float
Resulting statistic value.
"""
stats = afwMath.StatisticsControl(
self.config.clip, self.config.nIter,
afwImage.Mask.getPlaneBitMask(self.config.mask))
try:
image = exposureOrImage.getMaskedImage()
except Exception:
try:
image = exposureOrImage.getImage()
except Exception:
image = exposureOrImage
return afwMath.makeStatistics(image, self.config.stat,
stats).getValue()
def __init__(self, *args, **kwargs):
pipeBase.Task.__init__(self, *args, **kwargs)
self.sctrl = afwMath.StatisticsControl()
self.sctrl.setAndMask(
afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes))
self.sctrl.setNanSafe(True)
def _testBadValue(self, badVal):
"""Test that we can handle an instance of `badVal` in the data correctly"""
x, y = 10, 10
for useImage in [True, False]:
if useImage:
self.image = afwImage.ImageF(100, 100)
self.image.set(self.val)
self.image.set(x, y, badVal)
else:
self.image = afwImage.MaskedImageF(100, 100)
self.image.set(self.val, 0x0, 1.0)
self.image.set(x, y, (badVal, 0x0, 1.0))
self.assertEqual(
afwMath.makeStatistics(self.image, afwMath.MAX).getValue(),
self.val)
self.assertEqual(
afwMath.makeStatistics(self.image, afwMath.MEAN).getValue(),
self.val)
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
self.assertFalse(
np.isfinite(
afwMath.makeStatistics(self.image, afwMath.MAX,
sctrl).getValue()))
self.assertFalse(
np.isfinite(
afwMath.makeStatistics(self.image, afwMath.MEAN,
sctrl).getValue()))
def testWeightedStats(self):
"""Test that bug from #1697 (weighted stats returning NaN) stays fixed."""
rand = afwMath.Random()
mu = 10000
edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE")
badPixelMask = afwImage.MaskU.getPlaneBitMask("EDGE")
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(3.0)
statsCtrl.setNumIter(2)
statsCtrl.setAndMask(badPixelMask)
for weight in (300.0, 10.0, 1.0):
print "Testing with weight=%0.1f" % (weight,)
maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack
weightList = []
nx, ny = 256, 256
for i in range(3):
print "Processing ", i
maskedImage = afwImage.MaskedImageF(nx, ny)
maskedImageList.append(maskedImage)
afwMath.randomPoissonImage(maskedImage.getImage(), rand, mu)
maskedImage.getVariance().set(mu)
weightList.append(weight)
self.reportBadPixels(maskedImage, badPixelMask)
print "Stack: ",
coaddMaskedImage = afwMath.statisticsStack(
maskedImageList, afwMath.MEANCLIP, statsCtrl, weightList)
self.reportBadPixels(coaddMaskedImage, badPixelMask)
def testTicket1123(self):
"""
Ticket #1123 reported that the Statistics stack routine throws an exception
when all pixels in a stack are masked. Returning a NaN pixel in the stack is preferred
"""
ctrl = afwMath.StatisticsControl()
ctrl.setAndMask(~0x0)
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
mimg.set([self.val, 0x1, self.val])
# test the case with no valid pixels ... both mean and stdev should be nan
stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
mean = stat.getValue(afwMath.MEAN)
stdev = stat.getValue(afwMath.STDEV)
self.assertNotEqual(mean, mean) # NaN does not equal itself
self.assertNotEqual(stdev, stdev) # NaN does not equal itself
# test the case with one valid pixel ... mean is ok, but stdev should still be nan
mimg.getMask().set(1, 1, 0x0)
stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
mean = stat.getValue(afwMath.MEAN)
stdev = stat.getValue(afwMath.STDEV)
self.assertEqual(mean, self.val)
self.assertNotEqual(stdev, stdev) # NaN does not equal itself
# test the case with two valid pixels ... both mean and stdev are ok
mimg.getMask().set(1, 2, 0x0)
stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
mean = stat.getValue(afwMath.MEAN)
stdev = stat.getValue(afwMath.STDEV)
self.assertEqual(mean, self.val)
self.assertEqual(stdev, 0.0)
def __init__(self, bbox, wcs, badMaskPlanes, logName="coadd.utils.Coadd"):
""" Creating a coadd
Parameters
----------
bbox : `lsst.afw.geom.Box2I`
bounding box of coadd Exposure with respect to parent
coadd dimensions = bbox.getDimensions(); xy0 = bbox.getMin()
wcs : `lsst.afw.geom.SKyWcs`
WCS of coadd exposure
badMaskPlanes : `Collection`
mask planes to pay attention to when rejecting masked pixels.
Specify as a collection of names.
badMaskPlanes should always include "NO_DATA".
logName : `str`
name by which messages are logged
"""
self._log = log.getLogger(logName)
self._bbox = bbox
self._wcs = wcs
self._badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
self._coadd = afwImage.ExposureF(bbox, wcs)
self._weightMap = afwImage.ImageF(bbox)
# dict of filter name: filter object for all filters seen so far
self._filterDict = dict()
self._statsControl = afwMath.StatisticsControl()
self._statsControl.setNumSigmaClip(3.0)
self._statsControl.setNumIter(2)
self._statsControl.setAndMask(self._badPixelMask)
def setUp(self):
self.val = 10
self.nRow, self.nCol = 100, 200
self.sctrl = afwMath.StatisticsControl()
# Integers
self.mimgI = afwImage.MaskedImageI(afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgI.set(self.val, 0x0, self.val)
self.imgI = afwImage.ImageI(afwGeom.Extent2I(self.nRow, self.nCol), self.val)
self.vecI = afwMath.vectorI(self.nRow*self.nCol, self.val)
# floats
self.mimgF = afwImage.MaskedImageF(afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgF.set(self.val, 0x0, self.val)
self.imgF = afwImage.ImageF(afwGeom.Extent2I(self.nRow, self.nCol), self.val)
self.vecF = afwMath.vectorF(self.nRow*self.nCol, self.val)
# doubles
self.mimgD = afwImage.MaskedImageD(afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgD.set(self.val, 0x0, self.val)
self.imgD = afwImage.ImageD(afwGeom.Extent2I(self.nRow, self.nCol), self.val)
self.vecD = afwMath.vectorD(self.nRow*self.nCol, self.val)
self.imgList = [self.imgI, self.imgF, self.imgD]
self.mimgList = [self.mimgI, self.mimgF, self.mimgD]
self.vecList = [self.vecI, self.vecF, self.vecD]
def testErrorsFromVariance(self):
"""Test that we can estimate the errors from the incoming variances"""
weight, mean, variance = 0.1, 1.0, 10.0
ctrl = afwMath.StatisticsControl()
mi = afwImage.MaskedImageF(lsst.geom.Extent2I(10, 10))
npix = 10 * 10
mi.getImage().set(mean)
mi.getVariance().set(variance)
weights = afwImage.ImageF(mi.getDimensions())
weights.set(weight)
ctrl.setCalcErrorFromInputVariance(True)
weighted = afwMath.makeStatistics(
mi, weights,
afwMath.MEAN | afwMath.MEANCLIP | afwMath.SUM | afwMath.ERRORS,
ctrl)
self.assertAlmostEqual(
weighted.getValue(afwMath.SUM) / (npix * mean * weight), 1)
self.assertAlmostEqual(weighted.getValue(afwMath.MEAN), mean)
self.assertAlmostEqual(
weighted.getError(afwMath.MEAN)**2, variance / npix)
self.assertAlmostEqual(
weighted.getError(afwMath.MEANCLIP)**2, variance / npix)
def setBadRegions(self, exposure):
"""Set all BAD areas of the chip to the average of the rest of the exposure
@param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
"""
if self.config.badStatistic == "MEDIAN":
statistic = afwMath.MEDIAN
elif self.config.badStatistic == "MEANCLIP":
statistic = afwMath.MEANCLIP
else:
raise RuntimeError(
"Impossible method %s of bad region correction" %
self.config.badStatistic)
mi = exposure.getMaskedImage()
mask = mi.getMask()
BAD = mask.getPlaneBitMask("BAD")
INTRP = mask.getPlaneBitMask("INTRP")
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(BAD)
value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
maskArray = mask.getArray()
imageArray = mi.getImage().getArray()
badPixels = numpy.logical_and((maskArray & BAD) > 0,
(maskArray & INTRP) == 0)
imageArray[:] = numpy.where(badPixels, value, imageArray)
self.log.info("Set %d BAD pixels to %.2f" % (badPixels.sum(), value))
def testMasked(self):
# get the stats for the image with two values
self.mimgR.set(self.valR, 0x0, self.valR)
stats = afwMath.makeStatistics(
self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV)
mean = 0.5 * (self.valL + self.valR)
nL, nR = self.mimgL.getWidth() * self.mimgL.getHeight(
), self.mimgR.getWidth() * self.mimgR.getHeight()
stdev = ((nL * (self.valL - mean)**2 + nR * (self.valR - mean)**2) /
(nL + nR - 1))**0.5
self.assertEqual(stats.getValue(afwMath.NPOINT), self.n)
self.assertEqual(stats.getValue(afwMath.MEAN), mean)
self.assertEqual(stats.getValue(afwMath.STDEV), stdev)
# set the right side Mask and the StatisticsControl andMask to 0x1
# Stats should be just for the left side!
maskBit = 0x1
self.mimgR.getMask().set(maskBit)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(maskBit)
statsNaN = afwMath.makeStatistics(
self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV, sctrl)
mean = self.valL
stdev = 0.0
self.assertEqual(statsNaN.getValue(afwMath.NPOINT), nL)
self.assertEqual(statsNaN.getValue(afwMath.MEAN), mean)
self.assertEqual(statsNaN.getValue(afwMath.STDEV), stdev)
def writeThumbnail(self, dataRef, dataset, exposure):
"""Write out exposure to a snapshot file named outfile in the given size.
"""
filename = dataRef.get(dataset + "_filename")[0]
directory = os.path.dirname(filename)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
# Don't fail if directory exists due to race
if e.errno != errno.EEXIST:
raise e
binning = self.config.thumbnailBinning
binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning,
binning, afwMath.MEAN)
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(
["SAT", "BAD", "INTRP"]))
stats = afwMath.makeStatistics(
binnedImage, afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX,
statsCtrl)
low = stats.getValue(
afwMath.MEDIAN) - self.config.thumbnailStdev * stats.getValue(
afwMath.STDEVCLIP)
makeRGB(binnedImage,
binnedImage,
binnedImage,
minimum=low,
dataRange=self.config.thumbnailRange,
Q=self.config.thumbnailQ,
fileName=filename,
saturatedBorderWidth=self.config.thumbnailSatBorder,
saturatedPixelValue=stats.getValue(afwMath.MAX))
def testBadRows(self):
"""Test that a bad set of rows in an image doesn't cause a failure
"""
initialValue = 20
mi = afwImage.MaskedImageF(500, 200)
mi.set((initialValue, 0x0, 1.0))
im = mi.getImage()
im[:, 0:100] = np.nan
del im
msk = mi.getMask()
badBits = msk.getPlaneBitMask(['EDGE', 'DETECTED', 'DETECTED_NEGATIVE'])
msk[0:400, :] |= badBits
del msk
if display:
ds9.mtv(mi, frame=0)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(badBits)
nx, ny = 17, 17
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
bkgd = afwMath.makeBackground(mi, bctrl)
statsImage = afwMath.cast_BackgroundMI(bkgd).getStatsImage()
if display:
ds9.mtv(statsImage, frame=1)
# the test is that this doesn't fail if the bug (#2297) is fixed
bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE, afwMath.REDUCE_INTERP_ORDER)
self.assertEqual(np.mean(bkgdImage[0:100, 0:100].getArray()), initialValue)
if display:
ds9.mtv(bkgdImage, frame=2)
def __init__(self, *args, **kwargs):
pipeBase.Task.__init__(self, *args, **kwargs)
self._badPixelMask = Mask.getPlaneBitMask(self.config.badMaskPlanes)
self._statsControl = afwMath.StatisticsControl()
self._statsControl.setNumSigmaClip(self.config.numSigmaClip)
self._statsControl.setNumIter(self.config.numIter)
self._statsControl.setAndMask(self._badPixelMask)
def testBadImage(self):
"""Test that an entirely bad image doesn't cause an absolute failure"""
initialValue = 20
mi = afwImage.MaskedImageF(500, 200)
# Check that no good values don't crash (they return NaN), and that a single good value
# is enough to redeem the entire image
for pix00 in [np.nan, initialValue]:
mi.image[:] = np.nan
mi.image[0, 0] = pix00
sctrl = afwMath.StatisticsControl()
nx, ny = 17, 17
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
bkgd = afwMath.makeBackground(mi, bctrl)
for interpStyle in [
afwMath.Interpolate.CONSTANT, afwMath.Interpolate.LINEAR,
afwMath.Interpolate.NATURAL_SPLINE,
afwMath.Interpolate.AKIMA_SPLINE
]:
# the test is that this doesn't fail if the bug (#2297) is
# fixed
bkgdImage = bkgd.getImageF(interpStyle,
afwMath.REDUCE_INTERP_ORDER)
val = np.mean(bkgdImage[0:100, 0:100].array)
if np.isfinite(pix00):
self.assertEqual(val, pix00)
else:
self.assertTrue(np.isnan(val))
def main():
gaussFunction = afwMath.GaussianFunction2D(3, 2, 0.5)
gaussKernel = afwMath.AnalyticKernel(10, 10, gaussFunction)
inImage = afwImage.ImageF(afwGeom.Extent2I(100, 100))
inImage.set(1)
if disp:
ds9.mtv(inImage, frame = 0)
# works
outImage = afwImage.ImageF(afwGeom.Extent2I(100, 100))
afwMath.convolve(outImage, inImage, gaussKernel, False, True)
if disp:
ds9.mtv(outImage, frame = 1)
print("Should be a number: ", afwMath.makeStatistics(
outImage, afwMath.MEAN).getValue())
print("Should be a number: ", afwMath.makeStatistics(
outImage, afwMath.STDEV).getValue())
# not works ... now does work
outImage = afwImage.ImageF(afwGeom.Extent2I(100, 100))
afwMath.convolve(outImage, inImage, gaussKernel, False, False)
if disp:
ds9.mtv(outImage, frame = 2)
print("Should be a number: ", afwMath.makeStatistics(
outImage, afwMath.MEAN).getValue())
print("Should be a number: ", afwMath.makeStatistics(
outImage, afwMath.STDEV).getValue())
# This will print nan
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
print ("Should be a nan (nanSafe set to False): " +
str(afwMath.makeStatistics(outImage, afwMath.MEAN, sctrl).getValue()))
def thresholdImage(self, image, thresholdParity, maskName="DETECTED"):
"""!Threshold the convolved image, returning a FootprintSet.
Helper function for detect().
\param image The (optionally convolved) MaskedImage to threshold
\param thresholdParity Parity of threshold
\param maskName Name of mask to set
\return FootprintSet
"""
parity = False if thresholdParity == "negative" else True
threshold = afwDet.createThreshold(self.config.thresholdValue,
self.config.thresholdType, parity)
threshold.setIncludeMultiplier(self.config.includeThresholdMultiplier)
if self.config.thresholdType == 'stdev':
bad = image.getMask().getPlaneBitMask([
'BAD',
'SAT',
'EDGE',
'NO_DATA',
])
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(bad)
stats = afwMath.makeStatistics(image, afwMath.STDEVCLIP, sctrl)
thres = stats.getValue(
afwMath.STDEVCLIP) * self.config.thresholdValue
threshold = afwDet.createThreshold(thres, 'value', parity)
threshold.setIncludeMultiplier(
self.config.includeThresholdMultiplier)
fpSet = afwDet.FootprintSet(image, threshold, maskName,
self.config.minPixels)
return fpSet
def __init__(self,
imfile,
mask_files=(),
bias_frame=None,
applyMasks=True,
linearity_correction=None):
super(MaskedCCD, self).__init__()
self.imfile = imfile
self.md = imutils.Metadata(imfile)
self.amp_geom = makeAmplifierGeometry(imfile)
all_amps = imutils.allAmps(imfile)
for amp in all_amps:
image = afwImage.ImageF(imfile, imutils.dm_hdu(amp))
mask = afwImage_Mask(image.getDimensions())
self[amp] = afwImage.MaskedImageF(image, mask)
self._added_mask_types = []
for mask_file in mask_files:
self.add_masks(mask_file)
self.stat_ctrl = afwMath.StatisticsControl()
if mask_files:
self.setAllMasks()
if bias_frame is not None:
self.bias_frame = MaskedCCD(bias_frame)
else:
self.bias_frame = None
self._applyMasks = applyMasks
self._linearity_correction = linearity_correction
