def main():
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(100, 100))
mimValue = (2, 0x0, 1)
mimg.set(mimValue)
# call with the factory function ... should get stats on the image plane
fmt = "%-40s %-16s %3.1f\n"
print fmt % ("Using makeStatistics:", "(should be " + str(mimValue[0]) + ")",
afwMath.makeStatistics(mimg, afwMath.MEAN).getValue()),
# call the constructor directly ... once for image plane, then for variance
# - make sure we're not using weighted stats for this
sctrl = afwMath.StatisticsControl()
sctrl.setWeighted(False)
print fmt % ("Using Statistics on getImage():", "(should be " + str(mimValue[0]) + ")",
afwMath.StatisticsF(mimg.getImage(), mimg.getMask(), mimg.getVariance(),
afwMath.MEAN, sctrl).getValue()),
print fmt % ("Using Statistics on getVariance():", "(should be " + str(mimValue[2]) + ")",
afwMath.StatisticsF(mimg.getVariance(), mimg.getMask(), mimg.getVariance(),
afwMath.MEAN, sctrl).getValue()),
# call makeStatistics as a front-end for the constructor
print fmt % ("Using makeStatistics on getImage():", "(should be " + str(mimValue[0]) + ")",
afwMath.makeStatistics(mimg.getImage(), mimg.getMask(), afwMath.MEAN).getValue()),
print fmt % ("Using makeStatistics on getVariance():", "(should be " + str(mimValue[2]) + ")",
afwMath.makeStatistics(mimg.getVariance(), mimg.getMask(), afwMath.MEAN).getValue()),
def testRunDataRef(self, pedestal=0.0, stddevMax=1.0e-4):
"""Test the .runDataRef method for complete test converage.
Paramters
---------
pedestal : `float`, optional
Pedestal to add into fringe frame.
stddevMax : `float`, optional
Maximum allowable standard deviation.
"""
xFreq = np.pi/10.0
xOffset = 1.0
yFreq = np.pi/15.0
yOffset = 0.5
scale = 1.0
fringe = createFringe(self.size, self.size, xFreq, xOffset, yFreq, yOffset)
fMi = fringe.getMaskedImage()
fMi += pedestal
exp = createFringe(self.size, self.size, xFreq, xOffset, yFreq, yOffset)
eMi = exp.getMaskedImage()
eMi *= scale
task = FringeTask(name="fringe", config=self.config)
dataRef = FringeDataRef(fringe)
task.runDataRef(exp, dataRef)
mi = exp.getMaskedImage()
mi -= afwMath.makeStatistics(mi, afwMath.MEAN).getValue()
self.assertLess(afwMath.makeStatistics(mi, afwMath.STDEV).getValue(), stddevMax)
def checkFringe(self, task, exp, fringes, precision):
"""Run fringe subtraction and verify
@param task Task to run
@param exp Science exposure
@param dataRef Data reference that will provide the fringes
@param precision Precision for assertAlmostEqual
"""
if debug:
global frame
ds9.mtv(exp, frame=frame, title="Science exposure")
frame += 1
fringe = dataRef.get()
if not isinstance(fringe, list):
fringe = [fringe]
for i, f in enumerate(fringe):
ds9.mtv(f, frame=frame, title="Fringe frame %d" % (i+1))
frame += 1
task.run(exp, fringes)
mi = exp.getMaskedImage()
if debug:
ds9.mtv(exp, frame=frame, title="Subtracted")
frame += 1
mi -= afwMath.makeStatistics(mi, afwMath.MEAN).getValue()
self.assertLess(afwMath.makeStatistics(mi, afwMath.STDEV).getValue(), precision)
def testNaN(self):
# get the stats for the image with two values
stats = afwMath.makeStatistics(
self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV)
mean = 0.5*(self.valL + self.valR)
nL, nR = self.mimgL.getWidth()*self.mimgL.getHeight(), self.mimgR.getWidth() * \
self.mimgR.getHeight()
stdev = ((nL*(self.valL - mean)**2 + nR *
(self.valR - mean)**2)/(nL + nR - 1))**0.5
self.assertEqual(stats.getValue(afwMath.NPOINT), self.n)
self.assertEqual(stats.getValue(afwMath.MEAN), mean)
self.assertEqual(stats.getValue(afwMath.STDEV), stdev)
# set the right side to NaN and stats should be just for the left side
self.mimgR.set(np.nan, 0x0, self.valR)
statsNaN = afwMath.makeStatistics(
self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV)
mean = self.valL
stdev = 0.0
self.assertEqual(statsNaN.getValue(afwMath.NPOINT), nL)
self.assertEqual(statsNaN.getValue(afwMath.MEAN), mean)
self.assertEqual(statsNaN.getValue(afwMath.STDEV), stdev)
def testRunDataRef(self, pedestal=0.0, precision=1.0e-4):
"""Test the .runDataRef method for complete test converage
@param pedestal Pedestal to add into fringe frame
@param precision Precision for assertAlmostEqual
"""
xFreq = numpy.pi / 10.0
xOffset = 1.0
yFreq = numpy.pi / 15.0
yOffset = 0.5
scale = 1.0
fringe = createFringe(self.size, self.size, xFreq, xOffset, yFreq, yOffset)
fMi = fringe.getMaskedImage()
fMi += pedestal
exp = createFringe(self.size, self.size, xFreq, xOffset, yFreq, yOffset)
eMi = exp.getMaskedImage()
eMi *= scale
task = FringeTask(name="fringe", config=self.config)
dataRef = FringeDataRef(fringe)
task.runDataRef(exp, dataRef)
mi = exp.getMaskedImage()
mi -= afwMath.makeStatistics(mi, afwMath.MEAN).getValue()
self.assertLess(afwMath.makeStatistics(mi, afwMath.STDEV).getValue(), precision)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0):
"""Apply flat correction in place
@param[in,out] maskedImage afw.image.MaskedImage to correct
@param[in] flatMaskedImage flat field afw.image.MaskedImage
@param[in] scalingType how to compute flat scale; one of 'MEAN', 'MEDIAN' or 'USER'
@param[in] userScale scale to use if scalingType is 'USER', else ignored
"""
if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" % \
(maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
# Figure out scale from the data
# Ideally the flats are normalized by the calibration product pipelin, but this allows some flexibility
# in the case that the flat is created by some other mechanism.
if scalingType == 'MEAN':
flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(), afwMath.MEAN).getValue(afwMath.MEAN)
elif scalingType == 'MEDIAN':
flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(), afwMath.MEDIAN).getValue(afwMath.MEDIAN)
elif scalingType == 'USER':
flatScale = userScale
else:
raise pexExcept.Exception, '%s : %s not implemented' % ("flatCorrection", scalingType)
maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
def cr(infn, crfn, maskfn):
exposure = afwImage.ExposureF(infn) #'850994p-21.fits'
print 'exposure', exposure
print 'w,h', exposure.getWidth(), exposure.getHeight()
W,H = exposure.getWidth(), exposure.getHeight()
#var = exposure.getMaskedImage().getVariance()
#print 'Variance', var.get(0,0)
wcs = exposure.getWcs()
print 'wcs', wcs
pixscale = wcs.pixelScale().asArcseconds()
psf = getFakePsf(pixscale)
# CRs
mask = exposure.getMaskedImage().getMask()
crBit = mask.getMaskPlane("CR")
mask.clearMaskPlane(crBit)
mi = exposure.getMaskedImage()
bg = afwMath.makeStatistics(mi, afwMath.MEDIAN).getValue()
print 'bg', bg
varval = afwMath.makeStatistics(mi, afwMath.VARIANCE).getValue()
print 'variance', varval, 'std', math.sqrt(varval)
varval = afwMath.makeStatistics(mi, afwMath.VARIANCECLIP).getValue()
print 'clipped variance', varval, 'std', math.sqrt(varval)
var = exposure.getMaskedImage().getVariance()
var.set(varval)
var = exposure.getMaskedImage().getVariance()
print 'Variance:', var.get(0,0)
keepCRs = False
policy = pexPolicy.Policy()
# policy.add('minSigma', 6.)
# policy.add('min_DN', 150.)
# policy.add('cond3_fac', 2.5)
# policy.add('cond3_fac2', 0.6)
# policy.add('niteration', 3)
# policy.add('nCrPixelMax', 200000)
policy.add('minSigma', 10.)
policy.add('min_DN', 500.)
policy.add('cond3_fac', 2.5)
policy.add('cond3_fac2', 0.6)
policy.add('niteration', 1)
policy.add('nCrPixelMax', 100000)
#psfimg = psf.computeImage(afwGeom.Point2D(W/2., H/2.))
#psfimg.writeFits('psf.fits')
print 'Finding cosmics...'
crs = measAlg.findCosmicRays(mi, psf, bg, policy, keepCRs)
print 'got', len(crs), 'cosmic rays',
mask = mi.getMask()
crBit = mask.getPlaneBitMask("CR")
afwDet.setMaskFromFootprintList(mask, crs, crBit)
mask.writeFits(maskfn)
exposure.writeFits(crfn)
def __init__(self, img):
"""
img: an afwImage.ImageF
"""
self.mim = afwImage.MaskedImageF(img)
self.mean = afwMath.makeStatistics(img, afwMath.MEAN)
self.std = afwMath.makeStatistics(img, afwMath.STDEV)
def testComputeImage(self):
"""Test the computation of the PSF's image at a point"""
ccdXY = afwGeom.Point2D(0, 0)
kIm = self.psf.computeImage(ccdXY)
if False:
ds9.mtv(kIm)
self.assertTrue(kIm.getWidth() == self.ksize)
#
# Check that the image is as expected.
#
xcen, ycen = self.ksize/2, self.ksize/2
I0 = kIm.get(xcen, ycen)
self.assertAlmostEqual(kIm.get(xcen + 1, ycen + 1),
I0*self.psf.computeImage().get(xcen + 1, ycen + 1))
#
# Is image normalised to a peak value of unity?
#
self.assertAlmostEqual(afwMath.makeStatistics(kIm, afwMath.MAX).getValue(), 1.0)
#
# Now create a normalised version
#
kIm = self.psf.computeImage(ccdXY, False)
self.assertAlmostEqual(afwMath.makeStatistics(kIm, afwMath.SUM).getValue(), 1.0)
def main():
gaussFunction = afwMath.GaussianFunction2D(3, 2, 0.5)
gaussKernel = afwMath.AnalyticKernel(10, 10, gaussFunction)
inImage = afwImage.ImageF(afwGeom.Extent2I(100, 100))
inImage.set(1)
if disp:
ds9.mtv(inImage, frame = 0)
# works
outImage = afwImage.ImageF(afwGeom.Extent2I(100, 100))
afwMath.convolve(outImage, inImage, gaussKernel, False, True)
if disp:
ds9.mtv(outImage, frame = 1)
print "Should be a number: ", afwMath.makeStatistics(outImage, afwMath.MEAN).getValue()
print "Should be a number: ", afwMath.makeStatistics(outImage, afwMath.STDEV).getValue()
# not works ... now does work
outImage = afwImage.ImageF(afwGeom.Extent2I(100, 100))
afwMath.convolve(outImage, inImage, gaussKernel, False, False)
if disp:
ds9.mtv(outImage, frame = 2)
print "Should be a number: ", afwMath.makeStatistics(outImage, afwMath.MEAN).getValue()
print "Should be a number: ", afwMath.makeStatistics(outImage, afwMath.STDEV).getValue()
# This will print nan
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
print ("Should be a nan (nanSafe set to False): " +
str(afwMath.makeStatistics(outImage, afwMath.MEAN, sctrl).getValue()))
def testMasked(self):
# get the stats for the image with two values
self.mimgR.set(self.valR, 0x0, self.valR)
stats = afwMath.makeStatistics(
self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV)
mean = 0.5*(self.valL + self.valR)
nL, nR = self.mimgL.getWidth()*self.mimgL.getHeight(), self.mimgR.getWidth() * \
self.mimgR.getHeight()
stdev = ((nL*(self.valL - mean)**2 + nR *
(self.valR - mean)**2)/(nL + nR - 1))**0.5
self.assertEqual(stats.getValue(afwMath.NPOINT), self.n)
self.assertEqual(stats.getValue(afwMath.MEAN), mean)
self.assertEqual(stats.getValue(afwMath.STDEV), stdev)
# set the right side Mask and the StatisticsControl andMask to 0x1
# Stats should be just for the left side!
maskBit = 0x1
self.mimgR.getMask().set(maskBit)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(maskBit)
statsNaN = afwMath.makeStatistics(
self.mimg, afwMath.NPOINT | afwMath.MEAN | afwMath.STDEV, sctrl)
mean = self.valL
stdev = 0.0
self.assertEqual(statsNaN.getValue(afwMath.NPOINT), nL)
self.assertEqual(statsNaN.getValue(afwMath.MEAN), mean)
self.assertEqual(statsNaN.getValue(afwMath.STDEV), stdev)
def _testBadValue(self, badVal):
"""Test that we can handle an instance of `badVal` in the data correctly
Note that we only test ImageF here (as ImageI can't contain a NaN)
"""
mean = self.images[0][1]
x, y = 10, 10
for useImage in [True, False]:
if useImage:
image = afwImage.ImageF(100, 100)
image.set(mean)
image[x, y] = badVal
else:
image = afwImage.MaskedImageF(100, 100)
image.set(mean, 0x0, 1.0)
image[x, y] = (badVal, 0x0, 1.0)
self.assertEqual(afwMath.makeStatistics(image, afwMath.MAX).getValue(), mean)
self.assertEqual(afwMath.makeStatistics(image, afwMath.MEAN).getValue(), mean)
sctrl = afwMath.StatisticsControl()
sctrl.setNanSafe(False)
self.assertFalse(np.isfinite(afwMath.makeStatistics(image, afwMath.MAX, sctrl).getValue()))
self.assertFalse(np.isfinite(afwMath.makeStatistics(image, afwMath.MEAN, sctrl).getValue()))
def testStatsZebra(self):
"""Add 1 to every other row"""
for image, isInt, mean, median, std in self.images:
image2 = image.clone()
#
# Add 1 to every other row, so the variance is increased by 1/4
#
self.assertEqual(image2.getHeight() % 2, 0)
width = image2.getWidth()
for y in range(1, image2.getHeight(), 2):
sim = image2[lsst.geom.Box2I(lsst.geom.Point2I(0, y), lsst.geom.Extent2I(width, 1))]
sim += 1
if display:
afwDisplay.Display(frame=0).mtv(image, "Image 1")
afwDisplay.Display(frame=1).mtv(image2, "Image 2 (var inc by 1/4)")
stats = afwMath.makeStatistics(image2,
afwMath.NPOINT | afwMath.STDEV | afwMath.MEAN | afwMath.ERRORS)
meanRes = stats.getResult(afwMath.MEAN)
n = stats.getValue(afwMath.NPOINT)
sd = stats.getValue(afwMath.STDEV)
self.assertAlmostEqual(meanRes[0], mean + 0.5, delta=self.delta("mean", isInt))
self.assertAlmostEqual(sd, np.hypot(std, 1/math.sqrt(4.0)*math.sqrt(n/(n - 1))),
delta=0.00011)
self.assertAlmostEqual(meanRes[1], sd/math.sqrt(image2.getWidth()*image2.getHeight()), 10)
meanSquare = afwMath.makeStatistics(image2, afwMath.MEANSQUARE).getValue()
self.assertAlmostEqual(meanSquare, 0.5*(mean**2 + (mean + 1)**2) + std**2,
delta=0.00025 if isInt else 0.00006)
def testStatsZebra(self):
"""Add 1 to every other row"""
image2 = self.image.Factory(self.image, True)
#
# Add 1 to every other row, so the variance is 1/4
#
self.assertEqual(image2.getHeight()%2, 0)
width = image2.getWidth()
for y in range(1, image2.getHeight(), 2):
sim = image2.Factory(image2, afwGeom.Box2I(afwGeom.Point2I(0, y), afwGeom.Extent2I(width, 1)),
afwImage.LOCAL)
sim += 1
if display:
ds9.mtv(self.image, frame = 0)
ds9.mtv(image2, frame = 1)
stats = afwMath.makeStatistics(image2,
afwMath.NPOINT | afwMath.STDEV | afwMath.MEAN | afwMath.ERRORS)
mean = stats.getResult(afwMath.MEAN)
n = stats.getValue(afwMath.NPOINT)
sd = stats.getValue(afwMath.STDEV)
self.assertEqual(mean[0], image2.get(0, 0) + 0.5)
self.assertEqual(sd, 1/math.sqrt(4.0)*math.sqrt(n/(n - 1)))
self.assertAlmostEqual(mean[1], sd/math.sqrt(image2.getWidth()*image2.getHeight()), 10)
meanSquare = afwMath.makeStatistics(image2, afwMath.MEANSQUARE).getValue()
self.assertEqual(meanSquare, 0.5*(image2.get(0, 0)**2 + image2.get(0, 1)**2))
def testTicket1123(self):
"""
Ticket #1123 reported that the Statistics stack routine throws an exception
when all pixels in a stack are masked. Returning a NaN pixel in the stack is preferred
"""
ctrl = afwMath.StatisticsControl()
ctrl.setAndMask(~0x0)
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
mimg.set([self.val, 0x1, self.val])
# test the case with no valid pixels ... both mean and stdev should be nan
stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
mean = stat.getValue(afwMath.MEAN)
stdev = stat.getValue(afwMath.STDEV)
self.assertNotEqual(mean, mean) # NaN does not equal itself
self.assertNotEqual(stdev, stdev) # NaN does not equal itself
# test the case with one valid pixel ... mean is ok, but stdev should still be nan
mimg.getMask().set(1, 1, 0x0)
stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
mean = stat.getValue(afwMath.MEAN)
stdev = stat.getValue(afwMath.STDEV)
self.assertEqual(mean, self.val)
self.assertNotEqual(stdev, stdev) # NaN does not equal itself
# test the case with two valid pixels ... both mean and stdev are ok
mimg.getMask().set(1, 2, 0x0)
stat = afwMath.makeStatistics(mimg, afwMath.MEAN | afwMath.STDEV, ctrl)
mean = stat.getValue(afwMath.MEAN)
stdev = stat.getValue(afwMath.STDEV)
self.assertEqual(mean, self.val)
self.assertEqual(stdev, 0.0)
def _setFallbackValue(self, mi=None):
"""Set the edge fallbackValue for interpolation
\param[in] mi input maksedImage on which to calculate the statistics
Must be provided if fallbackValueType != "USER".
\return fallbackValue The value set/computed based on the fallbackValueType
and negativeFallbackAllowed config parameters
"""
if self.config.fallbackValueType != "USER":
assert mi, "No maskedImage provided"
if self.config.fallbackValueType == "MEAN":
fallbackValue = afwMath.makeStatistics(mi, afwMath.MEAN).getValue()
elif self.config.fallbackValueType == "MEDIAN":
fallbackValue = afwMath.makeStatistics(mi, afwMath.MEDIAN).getValue()
elif self.config.fallbackValueType == "MEANCLIP":
fallbackValue = afwMath.makeStatistics(mi, afwMath.MEANCLIP).getValue()
elif self.config.fallbackValueType == "USER":
fallbackValue = self.config.fallbackUserValue
else:
raise NotImplementedError("%s : %s not implemented" % ("fallbackValueType", self.config.fallbackValueType))
if not self.config.negativeFallbackAllowed and fallbackValue < 0.0:
self.log.warn(
"Negative interpolation edge fallback value computed but "
"negativeFallbackAllowed is False: setting fallbackValue to 0.0"
)
fallbackValue = max(fallbackValue, 0.0)
self.log.info("fallbackValueType %s has been set to %.4f" % (self.config.fallbackValueType, fallbackValue))
return fallbackValue
def addNoise(self, mi):
img = mi.getImage()
seed = int(afwMath.makeStatistics(mi.getVariance(), afwMath.MEDIAN).getValue())
rdm = afwMath.Random(afwMath.Random.MT19937, seed)
rdmImage = img.Factory(img.getDimensions())
afwMath.randomGaussianImage(rdmImage, rdm)
img += rdmImage
return afwMath.makeStatistics(rdmImage, afwMath.MEAN).getValue(afwMath.MEAN)
def testMedian(self):
"""Test the median code"""
for image, isInt, mean, median, std in self.images:
med = afwMath.makeStatistics(image, afwMath.MEDIAN).getValue()
self.assertAlmostEqual(med, median, delta=self.delta("median", isInt))
values = [1.0, 2.0, 3.0, 2.0]
self.assertEqual(afwMath.makeStatistics(values, afwMath.MEDIAN).getValue(), 2.0)
def testMedian(self):
"""Test the median code"""
stats = afwMath.makeStatistics(self.image, afwMath.MEDIAN)
self.assertEqual(stats.getValue(afwMath.MEDIAN), self.val)
values = [1.0, 2.0, 3.0, 2.0 ]
self.assertEqual(afwMath.makeStatistics(values, afwMath.MEDIAN).getValue(), 2.0)
def testVarianceClip(self):
"""Test the 3-sigma clipped standard deviation and variance"""
for image, isInt, mean, median, std in self.images:
delta = 0.0006 if isInt else 0.0014
stdevClip = afwMath.makeStatistics(image, afwMath.STDEVCLIP).getValue()
self.assertAlmostEqual(stdevClip, math.sqrt(self.clippedVariance3)*std, delta=delta)
varianceClip = afwMath.makeStatistics(image, afwMath.VARIANCECLIP).getValue()
self.assertAlmostEqual(varianceClip, self.clippedVariance3*std**2, delta=2*delta)
def _checkMaskedImage(self, mim, width, height, val1, val2, val3):
# Check that the input image has dimensions width & height and that the image, mask and
# variance have mean val1, val2 & val3 respectively.
self.assertEqual(mim.getWidth(), width)
self.assertEqual(mim.getHeight(), width)
self.assertEqual(afwMath.makeStatistics(mim.getImage(), afwMath.MEAN).getValue(), val1)
s = afwMath.makeStatistics(mim.getMask(), afwMath.SUM | afwMath.NPOINT)
self.assertEqual(float(s.getValue(afwMath.SUM)) / s.getValue(afwMath.NPOINT), val2)
self.assertEqual(afwMath.makeStatistics(mim.getVariance(), afwMath.MEAN).getValue(), val3)
def scaleVariance(self, exposure):
ctrl = afwMath.StatisticsControl()
ctrl.setAndMask(~0x0)
var = exposure.getMaskedImage().getVariance()
mask = exposure.getMaskedImage().getMask()
dstats = afwMath.makeStatistics(exposure.getMaskedImage(), afwMath.VARIANCECLIP, ctrl).getValue(afwMath.VARIANCECLIP)
vstats = afwMath.makeStatistics(var, mask, afwMath.MEANCLIP, ctrl).getValue(afwMath.MEANCLIP)
vrat = dstats / vstats
self.log.info("Renormalising variance by %f" % (vrat))
var *= vrat
def fitRadialParabola(mi, niter=3, tol=1e-5, nsigma=5, returnResidualImage=False):
"""Fit a radial parabola (centered at (0, 0) to the image im using a linear fit with an nsigma clip"""
width, height = mi.getDimensions()
BAD = afwImage.MaskU.getPlaneBitMask("BAD")
X, Y = np.meshgrid(np.arange(width), np.arange(height))
R = np.hypot(X + mi.getX0(), Y + mi.getY0())
R = R.flatten()
one = np.ones_like(R)
A = np.array([one, R, R**2]).transpose()
isMaskedImage = hasattr(mi, "getImage")
im = mi.getImage() if isMaskedImage else mi
ima = im.getArray().flatten()
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(afwImage.MaskU.getPlaneBitMask("BAD"))
c0, c1, c2 = afwMath.makeStatistics(mi, afwMath.MEANCLIP, sctrl).getValue(), 0, 0 # initial guess
c00 = c0
returnResidualImage = True
if returnResidualImage:
res = mi.clone() # residuals from plane fit
resim = res.getImage() if isMaskedImage else res
for i in range(niter):
fit = c0 + c1*R + c2*R**2
resim.getArray()[:] = im.getArray() - fit.reshape(height, width)
stats = afwMath.makeStatistics(res, afwMath.STDEVCLIP, sctrl)
stdev = stats.getValue(afwMath.STDEVCLIP)
good = np.abs(resim.getArray().flatten()) < nsigma*stdev
if isMaskedImage:
good = np.where(np.logical_and(good,
np.bitwise_and(res.getMask().getArray().flatten(), BAD) == 0))[0]
else:
good = np.where(np.logical_and(good, np.isfinite(ima)))[0]
b, residuals = np.linalg.lstsq(A[good], ima[good])[:2]
if np.all(residuals < tol):
break
c0, c1, c2 = b
if returnResidualImage: # need to update with latest values
fit = (c0 - c00) + c1*R + c2*R**2 # n.b. not c0
resim.getArray()[:] = im.getArray() - fit.reshape(height, width)
return b, res
else:
return b, None
def testNMasked(self):
"""Test that NMASKED works"""
maskVal = 0xBE
ctrl = afwMath.StatisticsControl()
ctrl.setAndMask(maskVal)
for image, isInt, mean, median, std in self.images:
mask = afwImage.Mask(image.getBBox())
mask.set(0)
self.assertEqual(afwMath.makeStatistics(image, mask, afwMath.NMASKED, ctrl).getValue(), 0)
mask[1, 1] = maskVal
self.assertEqual(afwMath.makeStatistics(image, mask, afwMath.NMASKED, ctrl).getValue(), 1)
def calcEffectiveGain(maskedImage):
"""Calculate effective gain
@param[in] maskedImage afw.image.MaskedImage to process
@return (median gain, mean gain) in e-/ADU
"""
im = afwImage.ImageF(maskedImage.getImage(), True)
var = maskedImage.getVariance()
im /= var
medgain = afwMath.makeStatistics(im, afwMath.MEDIAN).getValue()
meangain = afwMath.makeStatistics(im, afwMath.MEANCLIP).getValue()
return medgain, meangain
def testStatsStdevclip(self):
"""Test STDEVCLIP; cf. #611"""
image2 = self.image.Factory(self.image, True)
stats = afwMath.makeStatistics(image2, afwMath.STDEVCLIP | afwMath.NPOINT | afwMath.SUM)
self.assertEqual(stats.getValue(afwMath.STDEVCLIP), 0)
#
# Check we get the correct sum even when clipping
#
self.assertEqual(stats.getValue(afwMath.NPOINT)*
afwMath.makeStatistics(image2, afwMath.MEAN).getValue(),
stats.getValue(afwMath.SUM))
def testMeanClip(self):
"""Verify that the 3-sigma clipped mean doesn't not return NaN for a single value."""
stats = afwMath.makeStatistics(self.image, afwMath.MEANCLIP)
self.assertEqual(stats.getValue(afwMath.MEANCLIP), self.val)
# this bug was caused by the iterative nature of the MEANCLIP.
# With only one point, the sample variance returns NaN to avoid a divide by zero error
# Thus, on the second iteration, the clip width (based on _variance) is NaN and corrupts
# all further calculations.
img = afwImage.ImageF(afwGeom.Extent2I(1, 1))
img.set(0)
stats = afwMath.makeStatistics(img, afwMath.MEANCLIP)
self.assertEqual(stats.getValue(), 0)
def measureBackground(self, exposure):
statsControl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
self.config.qa.flatness.nIter)
maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
statsControl.setAndMask(maskVal)
maskedImage = exposure.getMaskedImage()
stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
skyLevel = stats.getValue(afwMath.MEDIAN)
skySigma = stats.getValue(afwMath.STDEVCLIP)
self.log.info("Flattened sky level: %f +/- %f" % (skyLevel, skySigma))
metadata = exposure.getMetadata()
metadata.set('SKYLEVEL', skyLevel)
metadata.set('SKYSIGMA', skySigma)
# calcluating flatlevel over the subgrids
stat = afwMath.MEANCLIP if self.config.qa.flatness.doClip else afwMath.MEAN
meshXHalf = int(self.config.qa.flatness.meshX/2.)
meshYHalf = int(self.config.qa.flatness.meshY/2.)
nX = int((exposure.getWidth() + meshXHalf) / self.config.qa.flatness.meshX)
nY = int((exposure.getHeight() + meshYHalf) / self.config.qa.flatness.meshY)
skyLevels = numpy.zeros((nX,nY))
for j in range(nY):
yc = meshYHalf + j * self.config.qa.flatness.meshY
for i in range(nX):
xc = meshXHalf + i * self.config.qa.flatness.meshX
xLLC = xc - meshXHalf
yLLC = yc - meshYHalf
xURC = xc + meshXHalf - 1
yURC = yc + meshYHalf - 1
bbox = afwGeom.Box2I(afwGeom.Point2I(xLLC, yLLC), afwGeom.Point2I(xURC, yURC))
miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
skyLevels[i,j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
good = numpy.where(numpy.isfinite(skyLevels))
skyMedian = numpy.median(skyLevels[good])
flatness = (skyLevels[good] - skyMedian) / skyMedian
flatness_rms = numpy.std(flatness)
flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
self.log.info("Measuring sky levels in %dx%d grids: %f" % (nX, nY, skyMedian))
self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f" % (nX, nY, flatness_pp, flatness_rms))
metadata.set('FLATNESS_PP', float(flatness_pp))
metadata.set('FLATNESS_RMS', float(flatness_rms))
metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
metadata.set('FLATNESS_MESHX', self.config.qa.flatness.meshX)
metadata.set('FLATNESS_MESHY', self.config.qa.flatness.meshY)
def fitPlane(mi, niter=3, tol=1e-5, nsigma=5, returnResidualImage=False):
"""Fit a plane to the image im using a linear fit with an nsigma clip"""
width, height = mi.getDimensions()
BAD = afwImage.MaskU.getPlaneBitMask("BAD")
X, Y = np.meshgrid(np.arange(width), np.arange(height))
X, Y = X.flatten(), Y.flatten()
one = np.ones_like(X)
A = np.array([one, X, Y]).transpose()
isMaskedImage = hasattr(mi, "getImage")
im = mi.getImage() if isMaskedImage else mi
ima = im.getArray().flatten()
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(afwImage.MaskU.getPlaneBitMask("BAD"))
z, dzdx, dzdy = afwMath.makeStatistics(mi, afwMath.MEANCLIP, sctrl).getValue(), 0, 0 # initial guess
returnResidualImage = True
if returnResidualImage:
res = mi.clone() # residuals from plane fit
resim = res.getImage() if isMaskedImage else res
for i in range(niter):
plane = z + dzdx*X + dzdy*Y
resim.getArray()[:] = im.getArray() - plane.reshape(height, width)
if isMaskedImage:
stats = afwMath.makeStatistics(res, afwMath.STDEVCLIP, sctrl)
stdev = stats.getValue(afwMath.STDEVCLIP)
good = np.where(np.logical_and(np.abs(res.getImage().getArray().flatten()) < nsigma*stdev,
np.bitwise_and(res.getMask().getArray().flatten(), BAD) == 0))[0]
b, residuals = np.linalg.lstsq(A[good], ima[good])[:2]
else:
b, residuals = np.linalg.lstsq(A, ima)[:2]
if np.all(residuals < tol):
break
z, dzdx, dzdy = b
if returnResidualImage: # need to update with latest values
plane = z + dzdx*X + dzdy*Y
resim.getArray()[:] = im.getArray() - plane.reshape(height, width)
return b, res
else:
return b, None
def scaleVariance(self, maskedImage):
"""Scale the variance in a maskedImage
Scales the variance plane to match the measured variance.
"""
ctrl = afwMath.StatisticsControl()
ctrl.setAndMask(~0x0)
var = maskedImage.getVariance()
mask = maskedImage.getMask()
dstats = afwMath.makeStatistics(maskedImage, afwMath.VARIANCECLIP, ctrl).getValue()
vstats = afwMath.makeStatistics(var, mask, afwMath.MEANCLIP, ctrl).getValue()
vrat = dstats / vstats
self.log.info("Renormalising variance by %f" % (vrat))
var *= vrat
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False):
"""Apply flat correction in place.
Parameters
----------
maskedImage : `lsst.afw.image.MaskedImage`
Image to process. The image is modified.
flatMaskedImage : `lsst.afw.image.MaskedImage`
Flat image of the same size as ``maskedImage``
scalingType : str
Flat scale computation method. Allowed values are 'MEAN',
'MEDIAN', or 'USER'.
userScale : scalar, optional
Scale to use if ``scalingType``='USER'.
invert : `Bool`, optional
If True, unflatten an already flattened image.
trimToFit : `Bool`, optional
If True, raw data is symmetrically trimmed to match
calibration size.
Raises
------
RuntimeError
Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
the same size or if ``scalingType`` is not an allowed value.
"""
if trimToFit:
maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)
if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
(maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
# Figure out scale from the data
# Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility
# in the case that the flat is created by some other mechanism.
if scalingType in ('MEAN', 'MEDIAN'):
scalingType = afwMath.stringToStatisticsProperty(scalingType)
flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue()
elif scalingType == 'USER':
flatScale = userScale
else:
raise RuntimeError('%s : %s not implemented' % ("flatCorrection", scalingType))
if not invert:
maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
else:
maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
def cosmicray(self, exposure, psf):
"""Cosmic ray masking
@param exposure Exposure to process
@param psf PSF
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayCR = lsstDebug.Info(__name__).displayCR
assert exposure, "No exposure provided"
assert psf, "No psf provided"
# Blow away old mask
try:
mask = exposure.getMaskedImage().getMask()
crBit = mask.getMaskPlane("CR")
mask.clearMaskPlane(crBit)
except:
pass
if display and displayCR:
ds9.incrDefaultFrame()
ds9.mtv(exposure, title="Pre-CR")
policy = self.config['cosmicray'].getPolicy()
mi = exposure.getMaskedImage()
bg = afwMath.makeStatistics(mi, afwMath.MEDIAN).getValue()
crs = measAlg.findCosmicRays(mi, psf, bg, policy, self._keepCRs)
num = 0
if crs is not None:
mask = mi.getMask()
crBit = mask.getPlaneBitMask("CR")
afwDet.setMaskFromFootprintList(mask, crs, crBit)
num = len(crs)
if display and displayCR:
ds9.incrDefaultFrame()
ds9.mtv(exposure, title="Post-CR")
ds9.cmdBuffer.pushSize()
for cr in crs:
displayUtils.drawBBox(cr.getBBox(), borderWidth=0.55)
ds9.cmdBuffer.popSize()
self.log.log(self.log.INFO, "Identified %d cosmic rays." % num)
return
def makeExposure(inputFile, weightFile, varianceFile, badPixelValue, variance):
exposure = afwImage.ExposureF(inputFile)
if np.isfinite(badPixelValue):
mi = exposure.getMaskedImage()
bad = mi.getImage().getArray() == badPixelValue
mi.getMask().getArray()[bad] |= mi.getMask().getPlaneBitMask("BAD")
del bad
del mi
if weightFile or varianceFile:
assert (not (weightFile and varianceFile)) # we checked this earlier
assert not np.isfinite(variance), \
"Please don't specify a variance and %s file" % ("weight" if weightFile else "variance")
mi = exposure.getMaskedImage()
if weightFile:
variance = afwImage.ImageF(weightFile)
varr = variance.getArray()
bad = (varr == 0)
varr[bad] = np.inf # avoid numpy warning
varr[:] = 1 / varr
else:
variance = afwImage.ImageF(varianceFile)
bad = np.logical_not(np.isfinite(mi.getImage().getArray()))
mi.getMask().getArray()[bad] |= mi.getMask().getPlaneBitMask("BAD")
del bad
mi.getVariance()[:] = variance
del mi
else:
if not np.isfinite(variance) or variance <= 0:
mi = exposure.getMaskedImage()
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(mi.getMask().getPlaneBitMask("BAD"))
variance = afwMath.makeStatistics(mi, afwMath.VARIANCECLIP,
sctrl).getValue()
del sctrl
del mi
exposure.getMaskedImage().getVariance()[:] = variance
return exposure
def interpolate(self, exposure, psf, defects):
"""Interpolate over defects
@param exposure Exposure to process
@param psf PSF for interpolation
@param defects Defect list
"""
assert exposure, "No exposure provided"
assert defects is not None, "No defects provided"
assert psf, "No psf provided"
mi = exposure.getMaskedImage()
fallbackValue = afwMath.makeStatistics(mi, afwMath.MEANCLIP).getValue()
measAlg.interpolateOverDefects(mi, psf, defects, fallbackValue)
self.log.log(self.log.INFO,
"Interpolated over %d defects." % len(defects))
return
def testMakeMatchStatisticsInPixels(self):
"""Test testMakeMatchStatisticsInPixels
"""
np.random.seed(164)
offList = [lsst.geom.Extent2D(val) for val in (np.random.random_sample([self.numMatches, 2])-0.5)*10]
for off, match in zip(offList, self.matchList):
centroid = match.second.get(self.sourceCentroidKey)
offCentroid = centroid + off
match.second.set(self.sourceCentroidKey, offCentroid)
itemList = (afwMath.MEDIAN, afwMath.MEAN, afwMath.STDEVCLIP)
itemMask = reduce(lambda a, b: a | b, itemList)
distStats = measAstrom.makeMatchStatisticsInPixels(self.wcs, self.matchList, itemMask)
distList = [math.hypot(*val) for val in offList]
directStats = afwMath.makeStatistics(distList, itemMask)
for item in itemList:
self.assertAlmostEqual(distStats.getValue(item), directStats.getValue(item))
def _inverted_image(self, offset=None):
"""
Return a masked image which is the trimmed and unbiased amp
image subtracted from an offset value. This enables regions
below a specified threshold value to be found using afwDetect.
If offset is None, then use the median of the unmasked pixels.
"""
my_image = self.ccd.unbiased_and_trimmed_image(self.amp).clone()
if offset is None:
median = afwMath.makeStatistics(my_image, afwMath.MEDIAN,
self.ccd.stat_ctrl).getValue()
else:
median = offset
imarr = my_image.getImage().getArray() # a view into the image data
imarr[:] = median - imarr[:]
return my_image, median
def setVariance(self, exposure):
mi = exposure.getMaskedImage()
image = mi.getImage().getArray()
variance = mi.getVariance().getArray()
if self.config.isBackgroundSubtracted:
bkgdVariance = afwMath.makeStatistics(mi.getImage(), afwMath.VARIANCECLIP).getValue()
self.log.info("Setting variance: background variance = %g ADU" % (bkgdVariance))
else:
self.log.info("Setting variance: noise=%g ADU" % (self.config.noise))
bkgdVariance = self.config.noise**2
variance[:] = bkgdVariance
if self.config.gain > 0.0:
self.log.info("Setting variance: gain=%g e/ADU" % (self.config.gain))
variance[:] += image/self.config.gain
def testComputeImage(self):
"""Test the computation of the PSF's image at a point."""
for psf in [self.psfDg, self.psfSg]:
ccdXY = lsst.geom.Point2D(0, 0)
kIm = psf.computeImage(ccdXY)
if False:
afwDisplay.Display(frame=1).mtv(kIm,
title=self._testMethodName +
": kIm")
self.assertEqual(kIm.getWidth(), self.ksize)
kIm = psf.computeImage(ccdXY)
self.assertAlmostEqual(
afwMath.makeStatistics(kIm, afwMath.SUM).getValue(), 1.0)
def makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[]):
"""Average the amplifier level kernels to create a detector level
kernel.
"""
inKernels = np.array([self.ampKernels[amp] for amp in
self.ampKernels if amp not in ampsToExclude])
averagingList = np.transpose(inKernels)
avgKernel = np.zeros_like(inKernels[0])
sctrl = afwMath.StatisticsControl()
sctrl.setNumSigmaClip(5.0)
for i in range(np.shape(avgKernel)[0]):
for j in range(np.shape(avgKernel)[1]):
avgKernel[i, j] = afwMath.makeStatistics(averagingList[i, j],
afwMath.MEANCLIP, sctrl).getValue()
self.detKernels[detectorName] = avgKernel
def testTicket1125(self):
"""Ticket 1125 reported that the clipped routines were aborting when called with no valid pixels. """
mimg = afwImage.MaskedImageF(afwGeom.Extent2I(10, 10))
mimg.set([self.val, 0x1, self.val])
ctrl = afwMath.StatisticsControl()
ctrl.setAndMask(~0x0)
# test the case with no valid pixels ... try MEANCLIP and STDEVCLIP
stat = afwMath.makeStatistics(mimg,
afwMath.MEANCLIP | afwMath.STDEVCLIP,
ctrl)
mean = stat.getValue(afwMath.MEANCLIP)
stdev = stat.getValue(afwMath.STDEVCLIP)
self.assertNotEqual(mean, mean) # NaN does not equal itself
self.assertNotEqual(stdev, stdev) # NaN does not equal itself
def testMask(self):
mask = afwImage.MaskU(afwGeom.Extent2I(10, 10))
mask.set(0x0)
mask.set(1, 1, 0x10)
mask.set(3, 1, 0x08)
mask.set(5, 4, 0x08)
mask.set(4, 5, 0x02)
stats = afwMath.makeStatistics(mask, afwMath.SUM | afwMath.NPOINT)
self.assertEqual(mask.getWidth()*mask.getHeight(), stats.getValue(afwMath.NPOINT))
self.assertEqual(0x1a, stats.getValue(afwMath.SUM))
def tst():
stats = afwMath.makeStatistics(mask, afwMath.MEAN)
self.assertRaises(lsst.pex.exceptions.InvalidParameterError, tst)
def testRunDataRef(self):
"""Test LsstSimIsrTask on amp-sized images in tests/data/
applyToSensorRef is not intended to take single amp-sized exposures, but will
run if the doAssembleCcd config parameter is False.
However, the exposure is not trimmed, and gain not reset.
"""
config = LsstSimIsrTask.ConfigClass()
config.doDark = False
config.doFringe = False
config.doAssembleCcd = False
config.doSnapCombine = False
lsstIsrTask = LsstSimIsrTask(config=config)
exposure = lsstIsrTask.runDataRef(self.ampRef).exposure
self.assertAlmostEqual(afwMath.makeStatistics(exposure.getMaskedImage(), afwMath.MEAN).getValue(),
2.855780, places=3)
def amplifierStats(self,
exposure,
keywordDict,
statControl,
failAll=False):
"""Measure amplifier level statistics from the exposure.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
The exposure to measure.
keywordDict : `dict` [`str`, `str`]
A dictionary of keys to use in the output results, with
values the string name associated with the
`lsst.afw.math.statistics.Property` to measure.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object with parameters defined by
the config.
failAll : `bool`, optional
If True, all tests will be set as failed.
Returns
-------
ampStats : `dict` [`str`, `dict` [`str`, scalar]]
A dictionary indexed by the amplifier name, containing
dictionaries of the statistics measured and their values.
"""
ampStats = {}
statisticToRun, statAccessor = self._configHelper(keywordDict)
# Measure stats on all amplifiers.
for ampIdx, amp in enumerate(exposure.getDetector()):
ampName = amp.getName()
theseStats = {}
ampExp = exposure.Factory(exposure, amp.getBBox())
stats = afwMath.makeStatistics(ampExp.getMaskedImage(),
statisticToRun, statControl)
for k, v in statAccessor.items():
theseStats[k] = stats.getValue(v)
if failAll:
theseStats['FORCE_FAILURE'] = failAll
ampStats[ampName] = theseStats
return ampStats
def setBadRegions(exposure, badStatistic="MEDIAN"):
"""Set all BAD areas of the chip to the average of the rest of the exposure
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure to mask. The exposure mask is modified.
badStatistic : `str`, optional
Statistic to use to generate the replacement value from the
image data. Allowed values are 'MEDIAN' or 'MEANCLIP'.
Returns
-------
badPixelCount : scalar
Number of bad pixels masked.
badPixelValue : scalar
Value substituted for bad pixels.
Raises
------
RuntimeError
Raised if `badStatistic` is not an allowed value.
"""
if badStatistic == "MEDIAN":
statistic = afwMath.MEDIAN
elif badStatistic == "MEANCLIP":
statistic = afwMath.MEANCLIP
else:
raise RuntimeError("Impossible method %s of bad region correction" %
badStatistic)
mi = exposure.getMaskedImage()
mask = mi.getMask()
BAD = mask.getPlaneBitMask("BAD")
INTRP = mask.getPlaneBitMask("INTRP")
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(BAD)
value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
maskArray = mask.getArray()
imageArray = mi.getImage().getArray()
badPixels = numpy.logical_and((maskArray & BAD) > 0,
(maskArray & INTRP) == 0)
imageArray[:] = numpy.where(badPixels, value, imageArray)
return badPixels.sum(), value
def addCosmicRays(image, nCR=100, emin=800, emax=1000, seed=None):
"""Add nCR fake cosmic rays to a frame, with pixel values between emin and emax (and some extra associated fainter pixels too)"""
#
if seed is None:
seed = int(afwMath.makeStatistics(image, afwMath.MAX).getValue())
if seed == 0:
seed = 1
width = image.getWidth()
height = image.getHeight()
rand = afwMath.Random(afwMath.Random.RANLUX, seed)
for i in range(nCR):
#
# Initial point in CR
#
x = rand.uniformInt(width)
y = rand.uniformInt(height)
amp = emin + rand.uniformInt(emax - emin)
#
# Extra contamination at about the initial amplitude
#
badPixels = []
while True:
badPixels.append([x, y, amp + 0.1*(emax - emin)*rand.uniform()])
if rand.uniform() > 0.5:
break
x += rand.uniformInt(3) - 1
y += rand.uniformInt(3) - 1
#
# Add a little extra CR flux to the pixels surrounding badPixels
#
for x, y, amp in badPixels:
while rand.uniform() < 0.5:
image.set(x, y, amp*rand.uniform())
x += rand.uniformInt(3) - 1
y += rand.uniformInt(3) - 1
#
# And set the initial badPixels themselves
#
for x, y, amp in badPixels:
if x >= 0 and x < width and y >= 0 and y < height:
image.set(x, y, amp)
def makePsfCandidates(self, exposure, starCat):
"""!Make a list of PSF candidates from a star catalog
@param[in] exposure the exposure containing the sources
@param[in] starCat catalog of stars (an lsst.afw.table.SourceCatalog),
e.g. as returned by the run or selectStars method
@return an lsst.pipe.base.Struct with fields:
- psfCandidates list of PSF candidates (lsst.meas.algorithms.PsfCandidate)
- goodStarCat catalog of stars that were successfully made into PSF candidates (a subset of starCat)
"""
goodStarCat = SourceCatalog(starCat.schema)
psfCandidateList = []
didSetSize = False
for star in starCat:
try:
psfCandidate = algorithmsLib.makePsfCandidate(star, exposure)
# The setXXX methods are class static, but it's convenient to call them on
# an instance as we don't know Exposure's pixel type
# (and hence psfCandidate's exact type)
if not didSetSize:
psfCandidate.setBorderWidth(self.config.borderWidth)
psfCandidate.setWidth(self.config.kernelSize +
2 * self.config.borderWidth)
psfCandidate.setHeight(self.config.kernelSize +
2 * self.config.borderWidth)
didSetSize = True
im = psfCandidate.getMaskedImage().getImage()
except Exception as err:
self.log.debug(
"Failed to make a psfCandidate from star %d: %s",
star.getId(), err)
continue
vmax = afwMath.makeStatistics(im, afwMath.MAX).getValue()
if not np.isfinite(vmax):
continue
psfCandidateList.append(psfCandidate)
goodStarCat.append(star)
return pipeBase.Struct(
psfCandidates=psfCandidateList,
goodStarCat=goodStarCat,
)
def addExposure(self, exposure, weightFactor=1.0):
"""Add an Exposure to the coadd
Parameters
----------
exposure: `afwImage.Exposure`
Exposure to add to coadd; this should be:
- background-subtracted or background-matched to the other images
being coadded
- psf-matched to the desired PSF model (optional)
- warped to match the coadd
- photometrically scaled to the desired flux magnitude
weightFactor : `float`
the extra weight factor for this exposure
Returns
--------
overlapBBox, weight : `afwGeom.Box2I`, `float`
the region of overlap between exposure and coadd in parent
coordinates. weight with which exposure was added to coadd;
weight = weightFactor / clipped mean variance
Subclasses may override to preprocess the exposure or change
the way it is added to the coadd.
"""
maskedImage = exposure.getMaskedImage()
# compute the weight
statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
afwMath.MEANCLIP, self._statsControl)
meanVar = statObj.getResult(afwMath.MEANCLIP)[0]
weight = weightFactor / float(meanVar)
if math.isnan(weight):
raise RuntimeError("Weight is NaN (weightFactor=%s; mean variance=%s)" % (weightFactor, meanVar))
# save filter info
filter = exposure.getFilter()
self._filterDict.setdefault(filter.getName(), filter)
self._log.info("Add exposure to coadd with weight=%0.3g", weight)
overlapBBox = coadd_utils.addToCoadd(self._coadd.getMaskedImage(),
self._weightMap, maskedImage,
self._badPixelMask, weight)
return overlapBBox, weight
def __init__(self, ccd, amp, bg_reg=(10, 10)):
self.ccd = ccd
self.amp = amp
self.ccdtemp = ccd.md.get('CCDTEMP')
self.fe55_yield = Fe55Yield(self.ccdtemp)
raw_image = ccd[amp]
try:
self.imarr = raw_image.getArray()
except AttributeError:
self.imarr = raw_image.getImage().getArray()
self.image = imutils.trim(raw_image, imaging=ccd.amp_geom.imaging)
self.image -= self._bg_image(*bg_reg)
flags = afwMath.MEANCLIP | afwMath.STDEVCLIP
stats = afwMath.makeStatistics(self.image, flags, self.ccd.stat_ctrl)
self.mean = stats.getValue(afwMath.MEANCLIP)
self.stdev = stats.getValue(afwMath.STDEVCLIP)
self.footprint_signal = self._footprint_signal_spans
def _generate_stats(self, amp):
#
# Extract the imaging region of the masked image for the
# specified amplifier.
#
image = imutils.trim(self.ccd[amp])
#
# Store numpy array of full segment for
# self._footprint_signal, since footprint spans use full
# segment image pixel coordinates.
#
self.arr = self.ccd[amp].getImage().getArray()
stats = afwMath.makeStatistics(image,
afwMath.STDEVCLIP | afwMath.MEDIAN,
self.ccd.stat_ctrl)
self.noise = stats.getValue(afwMath.STDEVCLIP)
self.median = stats.getValue(afwMath.MEDIAN)
def testMakeMatchStatisticsInRadians(self):
"""Test makeMatchStatisticsInRadians
"""
np.random.seed(164)
offLenList = [val*lsst.geom.radians for val in np.random.random_sample([self.numMatches])]
offDirList = [val*lsst.geom.radians for val in np.random.random_sample([self.numMatches])*math.pi*2]
for offLen, offDir, match in zip(offLenList, offDirList, self.matchList):
coord = match.first.get(self.refCoordKey)
offsetCoord = coord.offset(offDir, offLen)
match.first.set(self.refCoordKey, offsetCoord)
itemList = (afwMath.MEDIAN, afwMath.MEANCLIP, afwMath.IQRANGE)
itemMask = reduce(lambda a, b: a | b, itemList)
distStats = measAstrom.makeMatchStatisticsInRadians(self.wcs, self.matchList, itemMask)
distRadiansList = [val.asRadians() for val in offLenList]
directStats = afwMath.makeStatistics(distRadiansList, itemMask)
for item in itemList:
self.assertAlmostEqual(distStats.getValue(item), directStats.getValue(item))
def testFootprintToBBoxList(self):
"""Test footprintToBBoxList"""
region = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(12, 10))
foot = afwDetect.Footprint(0, region)
for y, x0, x1 in [
(3, 3, 5),
(3, 7, 7),
(4, 2, 3),
(4, 5, 7),
(5, 2, 3),
(5, 5, 8),
(6, 3, 5),
]:
foot.addSpan(y, x0, x1)
idImage = afwImage.ImageU(region.getDimensions())
idImage.set(0)
foot.insertIntoImage(idImage, 1)
if display:
ds9.mtv(idImage)
idImageFromBBox = idImage.Factory(idImage, True)
idImageFromBBox.set(0)
bboxes = afwDetect.footprintToBBoxList(foot)
for bbox in bboxes:
x0, y0, x1, y1 = bbox.getMinX(), bbox.getMinY(), bbox.getMaxX(
), bbox.getMaxY()
for y in range(y0, y1 + 1):
for x in range(x0, x1 + 1):
idImageFromBBox.set(x, y, 1)
if display:
x0 -= 0.5
y0 -= 0.5
x1 += 0.5
y1 += 0.5
ds9.line([(x0, y0), (x1, y0), (x1, y1), (x0, y1), (x0, y0)],
ctype=ds9.RED)
idImageFromBBox -= idImage # should be blank
stats = afwMath.makeStatistics(idImageFromBBox, afwMath.MAX)
self.assertEqual(stats.getValue(), 0)
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
"""Interpolate over defects specified in a defect list
@param[in,out] maskedImage masked image to process
@param[in] defectList defect list
@param[in] fwhm FWHM of double Gaussian smoothing kernel
@param[in] fallbackValue fallback value if an interpolated value cannot be determined;
if None then use clipped mean image value
"""
psf = createPsf(fwhm)
if fallbackValue is None:
fallbackValue = afwMath.makeStatistics(maskedImage.getImage(),
afwMath.MEANCLIP).getValue()
if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
maskedImage.getMask.addMaskPlane('INTRP')
measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue,
True)
def computeVarianceMean(exposure,
actuallyDoImage=False,
statToDo=afwMath.MEANCLIP):
statsControl = afwMath.StatisticsControl()
statsControl.setNumSigmaClip(3.)
statsControl.setNumIter(3)
ignoreMaskPlanes = ("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD",
"NO_DATA", "DETECTED_NEGATIVE")
statsControl.setAndMask(afwImage.MaskU.getPlaneBitMask(ignoreMaskPlanes))
imToDo = exposure.getMaskedImage().getVariance()
if actuallyDoImage:
imToDo = exposure.getMaskedImage().getImage()
statObj = afwMath.makeStatistics(imToDo,
exposure.getMaskedImage().getMask(),
statToDo, statsControl)
var = statObj.getValue(statToDo)
return var
def _i_scale(self, algorithm, minval, maxval, unit, *args, **kwargs):
maskedPixels = kwargs.get("maskedPixels", [])
if isinstance(maskedPixels, str):
maskedPixels = [maskedPixels]
bitmask = afwImage.Mask.getPlaneBitMask(maskedPixels)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(bitmask)
if minval == "minmax":
if self._image is None:
raise RuntimeError("You may only use minmax if an image is loaded into the display")
mi = afwImage.makeMaskedImage(self._image, self._mask)
stats = afwMath.makeStatistics(mi, afwMath.MIN | afwMath.MAX, sctrl)
minval = stats.getValue(afwMath.MIN)
maxval = stats.getValue(afwMath.MAX)
elif minval == "zscale":
if bitmask:
print("scale(..., 'zscale', maskedPixels=...) is not yet implemented")
if algorithm is None:
self._normalize = None
elif algorithm == "asinh":
if minval == "zscale":
if self._image is None:
raise RuntimeError("You may only use zscale if an image is loaded into the display")
self._normalize = AsinhZScaleNormalize(image=self._image, Q=kwargs.get("Q", 8.0))
else:
self._normalize = AsinhNormalize(minimum=minval,
dataRange=maxval - minval, Q=kwargs.get("Q", 8.0))
elif algorithm == "linear":
if minval == "zscale":
if self._image is None:
raise RuntimeError("You may only use zscale if an image is loaded into the display")
self._normalize = ZScaleNormalize(image=self._image,
nSamples=kwargs.get("nSamples", 1000),
contrast=kwargs.get("contrast", 0.25))
else:
self._normalize = LinearNormalize(minimum=minval, maximum=maxval)
else:
raise RuntimeError("Unsupported stretch algorithm \"%s\"" % algorithm)
def testBadAreaFailsSpline(self):
"""Check that a NaN in the stats image doesn't cause spline interpolation to fail (#2734)"""
image = afwImage.ImageF(15, 9)
for y in range(image.getHeight()):
for x in range(image.getWidth()):
# n.b. linear, which is what the interpolation will fall back
# to
image[x, y, afwImage.LOCAL] = 1 + 2 * y
# Set the right corner to NaN. This will mean that we have too few
# points for a spline interpolator
binSize = 3
image[-binSize:, -binSize:, afwImage.LOCAL] = np.nan
nx = image.getWidth() // binSize
ny = image.getHeight() // binSize
sctrl = afwMath.StatisticsControl()
bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
bkgd = afwMath.makeBackground(image, bctrl)
if debugMode:
afwDisplay.Display(frame=0).mtv(image,
title=self._testMethodName +
" image")
afwDisplay.Display(frame=1).mtv(bkgd.getStatsImage(),
title=self._testMethodName +
" bkgd StatsImage")
# Should throw if we don't permit REDUCE_INTERP_ORDER
self.assertRaises(lsst.pex.exceptions.OutOfRangeError, bkgd.getImageF,
afwMath.Interpolate.NATURAL_SPLINE)
# The interpolation should fall back to linear for the right part of the image
# where the NaNs don't permit spline interpolation (n.b. this happens
# to be exact)
bkgdImage = bkgd.getImageF(afwMath.Interpolate.NATURAL_SPLINE,
afwMath.REDUCE_INTERP_ORDER)
if debugMode:
afwDisplay.Display(frame=2).mtv(bkgdImage,
title=self._testMethodName +
" bkgdImage")
image -= bkgdImage
self.assertEqual(
afwMath.makeStatistics(image, afwMath.MEAN).getValue(), 0.0)
def testNormalize(self):
"""Test Footprint.normalize"""
w, h = 12, 10
region = afwGeom.Box2I(afwGeom.Point2I(0,0), afwGeom.Extent2I(w,h))
im = afwImage.ImageU(afwGeom.Extent2I(w, h))
im.set(0)
#
# Create a footprint; note that these Spans overlap
#
for spans, box in (([(3, 5, 6),
(4, 7, 7), ], afwGeom.Box2I(afwGeom.Point2I(5,3), afwGeom.Point2I(7,4))),
([(3, 3, 5), (3, 6, 9),
(4, 2, 3), (4, 5, 7), (4, 8, 8),
(5, 2, 3), (5, 5, 8), (5, 6, 7),
(6, 3, 5),
], afwGeom.Box2I(afwGeom.Point2I(2,3), afwGeom.Point2I(9,6)))
):
foot = afwDetect.Footprint(0, region)
for y, x0, x1 in spans:
foot.addSpan(y, x0, x1)
for x in range(x0, x1 + 1): # also insert into im
im.set(x, y, 1)
idImage = afwImage.ImageU(afwGeom.Extent2I(w, h))
idImage.set(0)
foot.insertIntoImage(idImage, 1)
if display: # overlaping pixels will be > 1
ds9.mtv(idImage)
#
# Normalise the Footprint, removing overlapping spans
#
foot.normalize()
idImage.set(0)
foot.insertIntoImage(idImage, 1)
if display:
ds9.mtv(idImage, frame=1)
idImage -= im
self.assertTrue(box == foot.getBBox())
self.assertEqual(afwMath.makeStatistics(idImage, afwMath.MAX).getValue(), 0)
def flatCorrection(self, exposure, flatExposure):
"""Apply flat correction in-place
This version allows tweaking the flat-field to match the observed
ratios of background flux in the amplifiers. This may be necessary
if the gains drift or the levels are slightly affected by non-linearity
or similar. Note that this tweak may not work if there is significant
structure in the image, and especially if the structure varies over
the amplifiers.
@param[in,out] exposure exposure to process
@param[in] flatExposure flatfield exposure of same size as exposure
"""
if self.config.doTweakFlat:
data = []
flatAmpList = []
bad = exposure.getMaskedImage().getMask().getPlaneBitMask(
["BAD", "SAT"])
stats = afwMath.StatisticsControl()
stats.setAndMask(bad)
for amp in exposure.getDetector():
box = amp.getDataSec(True)
dataAmp = afwImage.MaskedImageF(exposure.getMaskedImage(), box,
afwImage.LOCAL).clone()
flatAmp = afwImage.MaskedImageF(flatExposure.getMaskedImage(),
box, afwImage.LOCAL)
flatAmpList.append(flatAmp)
dataAmp /= flatAmp
data.append(
afwMath.makeStatistics(dataAmp, afwMath.MEDIAN,
stats).getValue())
data = numpy.array(data)
tweak = data / data.sum() * len(data)
self.log.warn(
"Tweaking flat-field to match observed amplifier ratios: %s" %
tweak)
for i, flat in enumerate(flatAmpList):
flat *= tweak[i]
lsstIsr.flatCorrection(exposure.getMaskedImage(),
flatExposure.getMaskedImage(),
scalingType="USER",
userScale=1.0)
def get_stats(image, nsigma=10):
"""
Parameters
----------
image: lsst.afw.image.Image
Image object for which to compute the clipped mean and clipped
stdev.
nsigma: int [10]
Value to use for sigma clipping.
Returns
-------
(float, float): Tuple of the (mean, stdev) of the pixel values.
"""
stat_ctrl = afwMath.StatisticsControl(numSigmaClip=nsigma)
flags = afwMath.MEANCLIP | afwMath.STDEVCLIP
stats = afwMath.makeStatistics(image, flags=flags, sctrl=stat_ctrl)
return stats.getValue(afwMath.MEANCLIP), stats.getValue(afwMath.STDEVCLIP)
def testWeightedVector(self):
"""Test std::vector, but with weights"""
sctrl = afwMath.StatisticsControl()
nval = len(self.vecList[0])
weight = 10
weights = [i * weight / float(nval - 1) for i in range(nval)]
for vec in self.vecList:
stats = afwMath.makeStatistics(
vec, weights,
afwMath.NPOINT | afwMath.STDEV | afwMath.MEAN | afwMath.SUM,
sctrl)
self.assertAlmostEqual(
0.5 * weight * sum(vec) / stats.getValue(afwMath.SUM), 1.0)
self.assertAlmostEqual(
sum(vec) / len(vec), stats.getValue(afwMath.MEAN))
def __init__(self, minimum=None, maximum=None, image=None):
if minimum is None or maximum is None:
assert image is not None, "You must provide an image if you don't set both minimum and maximum"
stats = afwMath.makeStatistics(image, afwMath.MIN | afwMath.MAX)
if minimum is None:
minimum = stats.getValue(afwMath.MIN)
if maximum is None:
maximum = stats.getValue(afwMath.MAX)
Mapping.__init__(self, minimum, image)
self.maximum = maximum
if maximum is None:
self._range = None
else:
assert maximum - minimum != 0, "minimum and maximum values must not be equal"
self._range = float(maximum - minimum)