def testMultiplyImages(self):
"""Test multiplication"""
# Multiply by a MaskedImage
self.mimage2 *= self.mimage
self.assertEqual(self.mimage2[0, 0, afwImage.LOCAL],
(self.imgVal2*self.imgVal1, self.EDGE,
self.varVal2*pow(self.imgVal1, 2) + self.varVal1*pow(self.imgVal2, 2)))
# Divide a MaskedImage<int> by an Image<int>; this divides the variance Image<float>
# by an Image<int> in C++
mimage_i = afwImage.MaskedImageI(self.mimage2.getDimensions())
mimage_i.set(900, 0x0, 1000.0)
image_i = afwImage.ImageI(mimage_i.getDimensions(), 2)
mimage_i *= image_i
self.assertEqual(mimage_i[0, 0, afwImage.LOCAL], (1800, 0x0, 4000.0))
# multiply by a scalar
self.mimage *= self.imgVal1
self.assertEqual(self.mimage[0, 0, afwImage.LOCAL],
(self.imgVal1*self.imgVal1, self.EDGE, self.varVal1*pow(self.imgVal1, 2)))
self.assertEqual(self.mimage.mask[1, 1, afwImage.LOCAL], self.EDGE)
self.assertEqual(self.mimage.mask[2, 2, afwImage.LOCAL], 0x0)
def setUp(self):
self.val = 10
self.nRow, self.nCol = 100, 200
self.sctrl = afwMath.StatisticsControl()
# Integers
self.mimgI = afwImage.MaskedImageI(
afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgI.set(self.val, 0x0, self.val)
self.imgI = afwImage.ImageI(afwGeom.Extent2I(self.nRow, self.nCol),
self.val)
# TODO: pybind11, this should probably be ndarray
self.vecI = [self.val for i in range(self.nRow * self.nCol)]
# floats
self.mimgF = afwImage.MaskedImageF(
afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgF.set(self.val, 0x0, self.val)
self.imgF = afwImage.ImageF(afwGeom.Extent2I(self.nRow, self.nCol),
self.val)
# TODO: pybind11, this should probably be ndarray
self.vecF = [float(self.val) for i in range(self.nRow * self.nCol)]
# doubles
self.mimgD = afwImage.MaskedImageD(
afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgD.set(self.val, 0x0, self.val)
self.imgD = afwImage.ImageD(afwGeom.Extent2I(self.nRow, self.nCol),
self.val)
# TODO: pybind11, this should probably be ndarray
self.vecD = [float(self.val) for i in range(self.nRow * self.nCol)]
self.imgList = [self.imgI, self.imgF, self.imgD]
self.mimgList = [self.mimgI, self.mimgF, self.mimgD]
self.vecList = [self.vecI, self.vecF, self.vecD]
def testAddImages(self):
"Test addition"
# add an image
self.mimage2 += self.mimage
self.assertEqual(self.mimage2[0, 0, afwImage.LOCAL], (self.imgVal1 + self.imgVal2, self.EDGE,
self.varVal1 + self.varVal2))
# Add an Image<int> to a MaskedImage<int>
mimage_i = afwImage.MaskedImageI(self.mimage2.getDimensions())
mimage_i.set(900, 0x0, 1000.0)
image_i = afwImage.ImageI(mimage_i.getDimensions(), 2)
mimage_i += image_i
self.assertEqual(mimage_i[0, 0, afwImage.LOCAL], (902, 0x0, 1000.0))
# add a scalar
self.mimage += self.imgVal1
self.assertEqual(self.mimage[0, 0, afwImage.LOCAL],
(2*self.imgVal1, self.EDGE, self.varVal1))
self.assertEqual(self.mimage.mask[1, 1, afwImage.LOCAL], self.EDGE)
self.assertEqual(self.mimage.mask[2, 2, afwImage.LOCAL], 0x0)
# add a function
self.mimage.set(self.imgVal1, 0x0, 0.0)
self.mimage += self.function
for i, j in [(2, 3)]:
self.assertEqual(self.mimage.image[i, j, afwImage.LOCAL],
self.imgVal1 + self.function(i, j))
def setUp(self):
self.val = 10
self.nRow, self.nCol = 100, 200
self.sctrl = afwMath.StatisticsControl()
# Integers
self.mimgI = afwImage.MaskedImageI(afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgI.set(self.val, 0x0, self.val)
self.imgI = afwImage.ImageI(afwGeom.Extent2I(self.nRow, self.nCol), self.val)
self.vecI = afwMath.vectorI(self.nRow*self.nCol, self.val)
# floats
self.mimgF = afwImage.MaskedImageF(afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgF.set(self.val, 0x0, self.val)
self.imgF = afwImage.ImageF(afwGeom.Extent2I(self.nRow, self.nCol), self.val)
self.vecF = afwMath.vectorF(self.nRow*self.nCol, self.val)
# doubles
self.mimgD = afwImage.MaskedImageD(afwGeom.Extent2I(self.nRow, self.nCol))
self.mimgD.set(self.val, 0x0, self.val)
self.imgD = afwImage.ImageD(afwGeom.Extent2I(self.nRow, self.nCol), self.val)
self.vecD = afwMath.vectorD(self.nRow*self.nCol, self.val)
self.imgList = [self.imgI, self.imgF, self.imgD]
self.mimgList = [self.mimgI, self.mimgF, self.mimgD]
self.vecList = [self.vecI, self.vecF, self.vecD]
def integerConvert(image):
"""Return an integer version of the input image.
Parameters
----------
image : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
Image to convert to integers.
Returns
-------
outI : `numpy.ndarray`, `lsst.afw.image.Image` or `MaskedImage`
The integer converted image.
Raises
------
RuntimeError
Raised if the input image could not be converted.
"""
if hasattr(image, "image"):
# Is a maskedImage:
imageI = image.image.convertI()
outI = afwImage.MaskedImageI(imageI, image.mask, image.variance)
elif hasattr(image, "convertI"):
# Is an Image:
outI = image.convertI()
elif hasattr(image, "astype"):
# Is a numpy array:
outI = image.astype(int)
else:
raise RuntimeError("Could not convert this to integers: %s %s %s",
image, type(image), dir(image))
return outI
def testDivideImages(self):
"""Test division"""
# Divide by a MaskedImage
mimage2_copy = self.mimage2.Factory(self.mimage2, True) # make a copy
mimage2_copy /= self.mimage
self.assertEqual(mimage2_copy.getImage().get(0, 0),
self.imgVal2 / self.imgVal1)
self.assertEqual(mimage2_copy.getMask().get(0, 0), self.EDGE)
self.assertAlmostEqual(mimage2_copy.getVariance().get(0, 0),
(self.varVal2 * pow(self.imgVal1, 2) +
self.varVal1 * pow(self.imgVal2, 2)) /
pow(self.imgVal1, 4), 10)
# Divide by an Image (of the same type as MaskedImage.getImage())
mimage = self.mimage2.Factory(self.mimage2, True)
mimage /= mimage.getImage()
self.assertEqual(mimage.get(0, 0),
(self.imgVal2 / self.imgVal2, 0x0, self.varVal2))
# Divide by an Image (of a different type from MaskedImage.getImage())
# this isn't supported from python (it's OK in C++)
if False:
mimage = self.mimage2.Factory(self.mimage2, True)
image = afwImage.ImageI(mimage.getDimensions(), 1)
mimage /= image
self.assertEqual(mimage.get(0, 0),
(self.imgVal2, 0x0, self.varVal2))
# Divide a MaskedImage<int> by an Image<int>; this divides the variance Image<float>
# by an Image<int> in C++
mimage_i = afwImage.MaskedImageI(self.mimage2.getDimensions())
mimage_i.set(900, 0x0, 1000.0)
image_i = afwImage.ImageI(mimage_i.getDimensions(), 2)
mimage_i /= image_i
self.assertEqual(mimage_i.get(0, 0), (450, 0x0, 250.0))
# divide by a scalar
self.mimage /= self.imgVal1
self.assertEqual(self.mimage.getImage().get(0, 0),
self.imgVal1 / self.imgVal1)
self.assertEqual(self.mimage.getMask().get(0, 0), self.EDGE)
self.assertAlmostEqual(self.mimage.getVariance().get(0, 0),
self.varVal1 / pow(self.imgVal1, 2), 9)
self.assertEqual(self.mimage.getMask().get(1, 1), self.EDGE)
self.assertEqual(self.mimage.getMask().get(2, 2), 0x0)
def testSubtractImages(self):
"Test subtraction"
# subtract an image
self.mimage2 -= self.mimage
self.assertEqual(self.mimage2[0, 0, afwImage.LOCAL],
(self.imgVal2 - self.imgVal1, self.EDGE, self.varVal2 + self.varVal1))
# Subtract an Image<int> from a MaskedImage<int>
mimage_i = afwImage.MaskedImageI(self.mimage2.getDimensions())
mimage_i.set(900, 0x0, 1000.0)
image_i = afwImage.ImageI(mimage_i.getDimensions(), 2)
mimage_i -= image_i
self.assertEqual(mimage_i[0, 0, afwImage.LOCAL], (898, 0x0, 1000.0))
# subtract a scalar
self.mimage -= self.imgVal1
self.assertEqual(self.mimage[0, 0, afwImage.LOCAL], (0.0, self.EDGE, self.varVal1))
def measureVectorOverscan(self, image):
"""Calculate the 1-d vector overscan from the input overscan image.
Parameters
----------
image : `lsst.afw.image.MaskedImage`
Image containing the overscan data.
Returns
-------
results : `lsst.pipe.base.Struct`
Overscan result with entries:
- ``overscanValue``: Overscan value to subtract (`float`)
- ``maskArray`` : `list` [ `bool` ]
List of rows that should be masked as ``SUSPECT`` when the
overscan solution is applied.
- ``isTransposed`` : `bool`
Indicates if the overscan data was transposed during
calcuation, noting along which axis the overscan should be
subtracted.
"""
calcImage = self.getImageArray(image)
# operate on numpy-arrays from here
calcImage, isTransposed = self.transpose(calcImage)
masked = self.maskOutliers(calcImage)
startTime = time.perf_counter()
if self.config.fitType == 'MEDIAN_PER_ROW':
mi = afwImage.MaskedImageI(image.getBBox())
masked = masked.astype(int)
if isTransposed:
masked = masked.transpose()
mi.image.array[:, :] = masked.data[:, :]
if bool(masked.mask.shape):
mi.mask.array[:, :] = masked.mask[:, :]
overscanVector = fitOverscanImage(mi, self.config.maskPlanes,
isTransposed)
maskArray = self.maskExtrapolated(overscanVector)
else:
collapsed = self.collapseArray(masked)
num = len(collapsed)
indices = 2.0 * np.arange(num) / float(num) - 1.0
poly = np.polynomial
fitter, evaler = {
'POLY': (poly.polynomial.polyfit, poly.polynomial.polyval),
'CHEB': (poly.chebyshev.chebfit, poly.chebyshev.chebval),
'LEG': (poly.legendre.legfit, poly.legendre.legval),
'NATURAL_SPLINE': (self.splineFit, self.splineEval),
'CUBIC_SPLINE': (self.splineFit, self.splineEval),
'AKIMA_SPLINE': (self.splineFit, self.splineEval)
}[self.config.fitType]
# These are the polynomial coefficients, or an
# interpolation object.
coeffs = fitter(indices, collapsed, self.config.order)
if isinstance(coeffs, float):
self.log.warn(
"Using fallback value %f due to fitter failure. Amplifier will be masked.",
coeffs)
overscanVector = np.full_like(indices, coeffs)
maskArray = np.full_like(collapsed, True, dtype=bool)
else:
# Otherwise we can just use things as normal.
overscanVector = evaler(indices, coeffs)
maskArray = self.maskExtrapolated(collapsed)
endTime = time.perf_counter()
self.log.info(
f"Overscan measurement took {endTime - startTime}s for {self.config.fitType}"
)
return pipeBase.Struct(overscanValue=np.array(overscanVector),
maskArray=maskArray,
isTransposed=isTransposed)
def overscanCorrection(ampMaskedImage,
overscanImage,
fitType='MEDIAN',
order=1,
collapseRej=3.0,
statControl=None,
overscanIsInt=True):
"""Apply overscan correction in place.
Parameters
----------
ampMaskedImage : `lsst.afw.image.MaskedImage`
Image of amplifier to correct; modified.
overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
Image of overscan; modified.
fitType : `str`
Type of fit for overscan correction. May be one of:
- ``MEAN``: use mean of overscan.
- ``MEANCLIP``: use clipped mean of overscan.
- ``MEDIAN``: use median of overscan.
- ``POLY``: fit with ordinary polynomial.
- ``CHEB``: fit with Chebyshev polynomial.
- ``LEG``: fit with Legendre polynomial.
- ``NATURAL_SPLINE``: fit with natural spline.
- ``CUBIC_SPLINE``: fit with cubic spline.
- ``AKIMA_SPLINE``: fit with Akima spline.
order : `int`
Polynomial order or number of spline knots; ignored unless
``fitType`` indicates a polynomial or spline.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object. In particular, we pay attention to numSigmaClip
overscanIsInt : `bool`
Treat the overscan region as consisting of integers, even if it's been
converted to float. E.g. handle ties properly.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``imageFit``: Value(s) removed from image (scalar or
`lsst.afw.image.Image`)
- ``overscanFit``: Value(s) removed from overscan (scalar or
`lsst.afw.image.Image`)
- ``overscanImage``: Overscan corrected overscan region
(`lsst.afw.image.Image`)
Raises
------
pexExcept.Exception
Raised if ``fitType`` is not an allowed value.
Notes
-----
The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
subtracted. Note that the ``overscanImage`` should not be a subimage of
the ``ampMaskedImage``, to avoid being subtracted twice.
Debug plots are available for the SPLINE fitTypes by setting the
`debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
plots show the scatter plot of the overscan data (collapsed along
the perpendicular dimension) as a function of position on the CCD
(normalized between +/-1).
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
numSigmaClip = statControl.getNumSigmaClip()
if fitType in ('MEAN', 'MEANCLIP'):
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImage, fitType,
statControl).getValue()
overscanFit = offImage
elif fitType in ('MEDIAN', ):
if overscanIsInt:
# we need an image with integer pixels to handle ties properly
if hasattr(overscanImage, "image"):
imageI = overscanImage.image.convertI()
overscanImageI = afwImage.MaskedImageI(imageI,
overscanImage.mask,
overscanImage.variance)
else:
overscanImageI = overscanImage.convertI()
else:
overscanImageI = overscanImage
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImageI, fitType,
statControl).getValue()
overscanFit = offImage
if overscanIsInt:
del overscanImageI
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE',
'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(
overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74 * (percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(
diff > numSigmaClip * stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(
biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0 * numpy.arange(num) / float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {
"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(
len(collapsed) * [numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices,
bins=numBins,
weights=1 -
collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
with numpy.errstate(invalid="ignore"): # suppress NAN warnings
values = numpy.histogram(
indices,
bins=numBins,
weights=collapsed.data * ~collapsedMask)[0] / numPerBin
binCenters = numpy.histogram(
indices, bins=numBins,
weights=indices * ~collapsedMask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask],
collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
plot.xlabel("centered/scaled position along overscan region")
plot.ylabel("pixel value/fit value")
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in (
"",
"c",
):
break
if ans in ("p", ):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
plot.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
overscanFit = afwImage.ImageF(overscanImage.getDimensions())
overscanArray = overscanFit.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray(
).transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' %
("overscanCorrection", fitType))
ampImage -= offImage
overscanImage -= overscanFit
return Struct(imageFit=offImage,
overscanFit=overscanFit,
overscanImage=overscanImage)
