def testInvalidInputs(self):
"""Test that invalid inputs cause an abort"""
self.assertRaises(pexExcept.InvalidParameterError,
lambda : afwMath.makeInterpolate([], [],
afwMath.Interpolate.CONSTANT))
afwMath.makeInterpolate([0], [1], afwMath.Interpolate.CONSTANT)
self.assertRaises(pexExcept.OutOfRangeError,
lambda : afwMath.makeInterpolate([0], [1],
afwMath.Interpolate.LINEAR))
def testInvalidInputs(self):
"""Test that invalid inputs cause an abort"""
self.assertRaises(pexExcept.InvalidParameterError,
lambda : afwMath.makeInterpolate([], [],
afwMath.Interpolate.CONSTANT))
interp = afwMath.makeInterpolate([0], [1], afwMath.Interpolate.CONSTANT)
self.assertRaises(pexExcept.OutOfRangeError,
lambda : afwMath.makeInterpolate([0], [1],
afwMath.Interpolate.LINEAR))
def testInvalidInputs(self):
"""Test that invalid inputs cause an abort"""
utilsTests.assertRaisesLsstCpp(
self,
pexExcept.InvalidParameterException,
lambda: afwMath.makeInterpolate([], [], afwMath.Interpolate.CONSTANT),
)
interp = afwMath.makeInterpolate([0], [1], afwMath.Interpolate.CONSTANT)
utilsTests.assertRaisesLsstCpp(
self, pexExcept.MemoryException, lambda: afwMath.makeInterpolate([0], [1], afwMath.Interpolate.LINEAR)
)
def testAkimaSplineParabola(self):
"""test the Spline interpolator"""
# specify interp type with the enum style interface
yinterpS = afwMath.makeInterpolate(self.x, self.y2, afwMath.Interpolate.AKIMA_SPLINE)
youtS = yinterpS.interpolate(self.xtest)
self.assertEqual(youtS, self.y2test)
def getInterpImage(self, bbox):
"""Return an image interpolated in R.A direction covering supplied bounding box
@param[in] bbox: integer bounding box for image (afwGeom.Box2I)
"""
npoints = len(self._xList)
#sort by X coordinate
if npoints < 1:
raise RuntimeError("Cannot create scaling image. Found no fluxMag0s to interpolate")
x, z = zip(*sorted(zip(self._xList, self._scaleList)))
xvec = afwMath.vectorD(x)
zvec = afwMath.vectorD(z)
height = bbox.getHeight()
width = bbox.getWidth()
x0, y0 = bbox.getMin()
interp = afwMath.makeInterpolate(xvec, zvec, self.interpStyle)
interpValArr = numpy.zeros(width, dtype=numpy.float32)
for i, xInd in enumerate(range(x0, x0 + width)):
xPos = afwImage.indexToPosition(xInd)
interpValArr[i] = interp.interpolate(xPos)
# assume the maskedImage being scaled is MaskedImageF (which is usually true); see ticket #3070
interpGrid = numpy.meshgrid(interpValArr, range(0, height))[0].astype(numpy.float32)
image = afwImage.makeImageFromArray(interpGrid)
image.setXY0(x0, y0)
return image
def getInterpImage(self, bbox):
"""Return an image interpolated in R.A direction covering supplied bounding box
@param[in] bbox: integer bounding box for image (afwGeom.Box2I)
"""
npoints = len(self._xList)
# sort by X coordinate
if npoints < 1:
raise RuntimeError(
"Cannot create scaling image. Found no fluxMag0s to interpolate"
)
x, z = list(zip(*sorted(zip(self._xList, self._scaleList))))
xvec = np.array(x, dtype=float)
zvec = np.array(z, dtype=float)
height = bbox.getHeight()
width = bbox.getWidth()
x0, y0 = bbox.getMin()
interp = afwMath.makeInterpolate(xvec, zvec, self.interpStyle)
interpValArr = np.zeros(width, dtype=np.float32)
for i, xInd in enumerate(range(x0, x0 + width)):
xPos = afwImage.indexToPosition(xInd)
interpValArr[i] = interp.interpolate(xPos)
# assume the maskedImage being scaled is MaskedImageF (which is usually true); see ticket #3070
interpGrid = np.meshgrid(interpValArr,
range(0, height))[0].astype(np.float32)
image = afwImage.makeImageFromArray(interpGrid)
image.setXY0(x0, y0)
return image
def __call__(self, image, **kwargs):
"""Correct for non-linearity.
Parameters
----------
image : `lsst.afw.image.Image`
Image to be corrected
kwargs : `dict`
Dictionary of parameter keywords:
``"coeffs"``
Coefficient vector (`list` or `numpy.array`).
``"log"``
Logger to handle messages (`lsst.log.Log`).
Returns
-------
output : `tuple` [`bool`, `int`]
If true, a correction was applied successfully. The
integer indicates the number of pixels that were
uncorrectable by being out of range.
"""
splineCoeff = kwargs['coeffs']
centers, values = np.split(splineCoeff, 2)
interp = afwMath.makeInterpolate(centers.tolist(), values.tolist(),
afwMath.stringToInterpStyle("AKIMA_SPLINE"))
ampArr = image.getArray()
delta = interp.interpolate(ampArr.flatten())
ampArr -= np.array(delta).reshape(ampArr.shape)
return True, 0
def testConstant(self):
"""test the constant interpolator"""
# [xy]vec: point samples
# [xy]vec_c: centered values
xvec = np.array([ 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0])
xvec_c = np.array([-0.5, 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5])
yvec = np.array([ 1.0, 2.4, 5.0, 8.4, 13.0, 18.4, 25.0, 32.6, 41.0, 50.6])
yvec_c = np.array([ 1.0, 1.7, 3.7, 6.7, 10.7, 15.7, 21.7, 28.8, 36.8, 45.8, 50.6])
interp = afwMath.makeInterpolate(xvec, yvec, afwMath.Interpolate.CONSTANT)
for x, y in zip(xvec_c, yvec_c):
self.assertAlmostEqual(interp.interpolate(x + 0.1), y)
self.assertAlmostEqual(interp.interpolate(x), y)
self.assertEqual(interp.interpolate(xvec[0] - 10), yvec[0])
n = len(yvec)
self.assertEqual(interp.interpolate(xvec[n - 1] + 10), yvec[n - 1])
for x, y in reversed(zip(xvec_c, yvec_c)): # test caching as we go backwards
self.assertAlmostEqual(interp.interpolate(x + 0.1), y)
self.assertAlmostEqual(interp.interpolate(x), y)
i = 2
for x in np.arange(xvec_c[i], xvec_c[i + 1], 10):
self.assertEqual(interp.interpolate(x), yvec_c[i])
def testConstant(self):
"""test the constant interpolator"""
# [xy]vec: point samples
# [xy]vec_c: centered values
xvec = np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0])
xvec_c = np.array(
[-0.5, 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5])
yvec = np.array(
[1.0, 2.4, 5.0, 8.4, 13.0, 18.4, 25.0, 32.6, 41.0, 50.6])
yvec_c = np.array(
[1.0, 1.7, 3.7, 6.7, 10.7, 15.7, 21.7, 28.8, 36.8, 45.8, 50.6])
interp = afwMath.makeInterpolate(xvec, yvec,
afwMath.Interpolate.CONSTANT)
for x, y in zip(xvec_c, yvec_c):
self.assertAlmostEqual(interp.interpolate(x + 0.1), y)
self.assertAlmostEqual(interp.interpolate(x), y)
self.assertEqual(interp.interpolate(xvec[0] - 10), yvec[0])
n = len(yvec)
self.assertEqual(interp.interpolate(xvec[n - 1] + 10), yvec[n - 1])
for x, y in reversed(list(zip(
xvec_c, yvec_c))): # test caching as we go backwards
self.assertAlmostEqual(interp.interpolate(x + 0.1), y)
self.assertAlmostEqual(interp.interpolate(x), y)
i = 2
for x in np.arange(xvec_c[i], xvec_c[i + 1], 10):
self.assertEqual(interp.interpolate(x), yvec_c[i])
def testInvalidInputs(self):
"""Test that invalid inputs cause an abort"""
utilsTests.assertRaisesLsstCpp(
self, pexExcept.InvalidParameterException,
lambda : afwMath.makeInterpolate(np.array([], dtype=float), np.array([], dtype=float),
afwMath.Interpolate.CONSTANT)
)
interp = afwMath.makeInterpolate(np.array([0], dtype=float), np.array([1], dtype=float),
afwMath.Interpolate.CONSTANT)
utilsTests.assertRaisesLsstCpp(
self, pexExcept.OutOfRangeException,
lambda : afwMath.makeInterpolate(np.array([0], dtype=float), np.array([1], dtype=float),
afwMath.Interpolate.LINEAR)
)
def testLinearRamp(self):
# === test the Linear Interpolator ============================
# default is akima spline
yinterpL = afwMath.makeInterpolate(self.x, self.y1)
youtL = yinterpL.interpolate(self.xtest)
self.assertEqual(youtL, self.y1test)
def testNaturalSplineRamp(self):
# === test the Spline interpolator =======================
# specify interp type with the string interface
yinterpS = afwMath.makeInterpolate(self.x, self.y1, afwMath.Interpolate.NATURAL_SPLINE)
youtS = yinterpS.interpolate(self.xtest)
self.assertEqual(youtS, self.y1test)
def testNaturalSplineRamp(self):
# === test the Spline interpolator =======================
# specify interp type with the string interface
yinterpS = afwMath.makeInterpolate(self.x, self.y1, afwMath.Interpolate.NATURAL_SPLINE)
youtS = yinterpS.interpolate(self.xtest)
self.assertEqual(youtS, self.y1test)
def testAkimaSplineParabola(self):
"""test the Spline interpolator"""
# specify interp type with the enum style interface
yinterpS = afwMath.makeInterpolate(self.x, self.y2,
afwMath.Interpolate.AKIMA_SPLINE)
youtS = yinterpS.interpolate(self.xtest)
self.assertEqual(youtS, self.y2test)
def testLinearRamp(self):
# === test the Linear Interpolator ============================
# default is akima spline
yinterpL = afwMath.makeInterpolate(self.x, self.y1)
youtL = yinterpL.interpolate(self.xtest)
self.assertEqual(youtL, self.y1test)
def splineFit(self, indices, collapsed, numBins):
"""Wrapper function to match spline fit API to polynomial fit API.
Parameters
----------
indices : `numpy.ndarray`
Locations to evaluate the spline.
collapsed : `numpy.ndarray`
Collapsed overscan values corresponding to the spline
evaluation points.
numBins : `int`
Number of bins to use in constructing the spline.
Returns
-------
interp : `lsst.afw.math.Interpolate`
Interpolation object for later evaluation.
"""
if not np.ma.is_masked(collapsed):
collapsed.mask = np.array(len(collapsed) * [np.ma.nomask])
numPerBin, binEdges = np.histogram(indices,
bins=numBins,
weights=1 -
collapsed.mask.astype(int))
with np.errstate(invalid="ignore"):
values = np.histogram(
indices,
bins=numBins,
weights=collapsed.data * ~collapsed.mask)[0] / numPerBin
binCenters = np.histogram(
indices, bins=numBins,
weights=indices * ~collapsed.mask)[0] / numPerBin
if len(binCenters[numPerBin > 0]) < 5:
self.log.warn(
"Cannot do spline fitting for overscan: %s valid points.",
len(binCenters[numPerBin > 0]))
# Return a scalar value if we have one, otherwise
# return zero. This amplifier is hopefully already
# masked.
if len(values[numPerBin > 0]) != 0:
return float(values[numPerBin > 0][0])
else:
return 0.0
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(self.config.fitType))
return interp
def testInvalidInputs(self):
"""Test that invalid inputs cause an abort"""
with self.assertRaises(pexExcept.OutOfRangeError):
afwMath.makeInterpolate(np.array([], dtype=float), np.array([], dtype=float),
afwMath.Interpolate.CONSTANT)
afwMath.makeInterpolate(np.array([0], dtype=float), np.array([1], dtype=float),
afwMath.Interpolate.CONSTANT)
with self.assertRaises(pexExcept.OutOfRangeError):
afwMath.makeInterpolate(np.array([0], dtype=float), np.array([1], dtype=float),
afwMath.Interpolate.LINEAR)
def interpolate1D(method, xSample, ySample, xInterp):
"""Interpolate in one dimension
Interpolates the curve provided by `xSample` and `ySample` at
the positions of `xInterp`. Automatically backs off the
interpolation method to achieve successful interpolation.
Parameters
----------
method : `lsst.afw.math.Interpolate.Style`
Interpolation method to use.
xSample : `numpy.ndarray`
Vector of ordinates.
ySample : `numpy.ndarray`
Vector of coordinates.
xInterp : `numpy.ndarray`
Vector of ordinates to which to interpolate.
Returns
-------
yInterp : `numpy.ndarray`
Vector of interpolated coordinates.
"""
if len(xSample) == 0:
return numpy.ones_like(xInterp) * numpy.nan
try:
return afwMath.makeInterpolate(xSample.astype(float),
ySample.astype(float),
method).interpolate(
xInterp.astype(float))
except Exception:
if method == afwMath.Interpolate.CONSTANT:
# We've already tried the most basic interpolation and it failed
return numpy.ones_like(xInterp) * numpy.nan
newMethod = afwMath.lookupMaxInterpStyle(len(xSample))
if newMethod == method:
newMethod = afwMath.Interpolate.CONSTANT
return interpolate1D(newMethod, xSample, ySample, xInterp)
def splineFit(self, indices, collapsed, numBins):
"""Wrapper function to match spline fit API to polynomial fit API.
Parameters
----------
indices : `numpy.ndarray`
Locations to evaluate the spline.
collapsed : `numpy.ndarray`
Collapsed overscan values corresponding to the spline
evaluation points.
numBins : `int`
Number of bins to use in constructing the spline.
Returns
-------
interp : `lsst.afw.math.Interpolate`
Interpolation object for later evaluation.
"""
if not np.ma.is_masked(collapsed):
collapsed.mask = np.array(len(collapsed) * [np.ma.nomask])
numPerBin, binEdges = np.histogram(indices,
bins=numBins,
weights=1 -
collapsed.mask.astype(int))
with np.errstate(invalid="ignore"):
values = np.histogram(
indices,
bins=numBins,
weights=collapsed.data * ~collapsed.mask)[0] / numPerBin
binCenters = np.histogram(
indices, bins=numBins,
weights=indices * ~collapsed.mask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(self.config.fitType))
return interp
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
statControl=None):
"""Apply overscan correction in place
@param[in,out] ampMaskedImage masked image to correct
@param[in] overscanImage overscan data as an afw.image.Image or afw.image.MaskedImage.
If a masked image is passed in the mask plane will be used
to constrain the fit of the bias level.
@param[in] fitType type of fit for overscan correction; one of:
- 'MEAN'
- 'MEDIAN'
- 'POLY' (ordinary polynomial)
- 'CHEB' (Chebyshev polynomial)
- 'LEG' (Legendre polynomial)
- 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE' (splines)
@param[in] order polynomial order or spline knots (ignored unless fitType
indicates a polynomial or spline)
@param[in] collapseRej Rejection threshold (sigma) for collapsing dimension of overscan
@param[in] statControl Statistics control object
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
if fitType == 'MEAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEAN, statControl).getValue(afwMath.MEAN)
elif fitType == 'MEDIAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEDIAN, statControl).getValue(afwMath.MEDIAN)
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:,numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(diff > collapseRej*stdevBiasArr[:,numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0*numpy.arange(num)/float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
weights=1-collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
values = numpy.histogram(indices, bins=numBins, weights=collapsed)[0]/numPerBin
binCenters = numpy.histogram(indices, bins=numBins, weights=indices)[0]/numPerBin
interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices, collapsed, 'k+')
axes.plot(indices, fitBiasArr, 'r-')
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = raw_input(prompt).lower()
if ans in ("", "c",):
break
if ans in ("p",):
import pdb; pdb.set_trace()
elif ans in ("h", ):
print "h[elp] c[ontinue] p[db]"
figure.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
if shortInd == 1:
offArray[:,:] = fitBiasArr[:,numpy.newaxis]
else:
offArray[:,:] = fitBiasArr[numpy.newaxis,:]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in xrange(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low,:] |= suspect
for high in xrange(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:,:] |= suspect
else:
raise pexExcept.Exception, '%s : %s an invalid overscan type' % \
("overscanCorrection", fitType)
ampImage -= offImage
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
statControl=None, overscanIsInt=True):
"""Apply overscan correction in place.
Parameters
----------
ampMaskedImage : `lsst.afw.image.MaskedImage`
Image of amplifier to correct; modified.
overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
Image of overscan; modified.
fitType : `str`
Type of fit for overscan correction. May be one of:
- ``MEAN``: use mean of overscan.
- ``MEANCLIP``: use clipped mean of overscan.
- ``MEDIAN``: use median of overscan.
- ``POLY``: fit with ordinary polynomial.
- ``CHEB``: fit with Chebyshev polynomial.
- ``LEG``: fit with Legendre polynomial.
- ``NATURAL_SPLINE``: fit with natural spline.
- ``CUBIC_SPLINE``: fit with cubic spline.
- ``AKIMA_SPLINE``: fit with Akima spline.
order : `int`
Polynomial order or number of spline knots; ignored unless
``fitType`` indicates a polynomial or spline.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object. In particular, we pay attention to numSigmaClip
overscanIsInt : `bool`
Treat the overscan region as consisting of integers, even if it's been
converted to float. E.g. handle ties properly.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``imageFit``: Value(s) removed from image (scalar or
`lsst.afw.image.Image`)
- ``overscanFit``: Value(s) removed from overscan (scalar or
`lsst.afw.image.Image`)
- ``overscanImage``: Overscan corrected overscan region
(`lsst.afw.image.Image`)
Raises
------
pexExcept.Exception
Raised if ``fitType`` is not an allowed value.
Notes
-----
The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
subtracted. Note that the ``overscanImage`` should not be a subimage of
the ``ampMaskedImage``, to avoid being subtracted twice.
Debug plots are available for the SPLINE fitTypes by setting the
`debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
plots show the scatter plot of the overscan data (collapsed along
the perpendicular dimension) as a function of position on the CCD
(normalized between +/-1).
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
numSigmaClip = statControl.getNumSigmaClip()
if fitType in ('MEAN', 'MEANCLIP'):
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue()
overscanFit = offImage
elif fitType in ('MEDIAN',):
if overscanIsInt:
# we need an image with integer pixels to handle ties properly
if hasattr(overscanImage, "image"):
imageI = overscanImage.image.convertI()
overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance)
else:
overscanImageI = overscanImage.convertI()
else:
overscanImageI = overscanImage
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImageI, fitType, statControl).getValue()
overscanFit = offImage
if overscanIsInt:
del overscanImageI
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0*numpy.arange(num)/float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
weights=1-collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
with numpy.errstate(invalid="ignore"): # suppress NAN warnings
values = numpy.histogram(indices, bins=numBins,
weights=collapsed.data*~collapsedMask)[0]/numPerBin
binCenters = numpy.histogram(indices, bins=numBins,
weights=indices*~collapsedMask)[0]/numPerBin
interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
plot.xlabel("centered/scaled position along overscan region")
plot.ylabel("pixel value/fit value")
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in ("", "c",):
break
if ans in ("p",):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
plot.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
overscanFit = afwImage.ImageF(overscanImage.getDimensions())
overscanArray = overscanFit.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType))
ampImage -= offImage
overscanImage -= overscanFit
return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
def getDistanceFromFocus(dIcSrc,
dCcd,
dCcdDims,
zemaxFilename,
config,
plotFilename=None):
# Focus error is measured by using rms^2 of stars on focus CCDs.
# If there is a focus error d, rms^2 can be written as
# rms^2 = rms_atm^2 + rms_opt_0^2 + alpha*d^2,
# where rms_atm is from atmosphere and rms_opt if from optics with out any focus error.
# On the focus CCDs which have +/-delta offset, the equation becomes
# rms_+^2 = rms_atm^2 + rms_opt_0^2 + alpha(d+delta)^2
# rms_-^2 = rms_atm^2 + rms_opt_0^2 + alpha(d-delta)^2
# Thus, the difference of these rms^2 gives the focus error as
# d = (rms_+^2 - rms_-^2)/(4 alpha delta)
# alpha is determined by ZEMAX simulations. It turned out that alpha is a function of distance from the center of FOV r.
# Also the best focus varies as a function of r. Thus the focus error can be rewritten as
# d(r) = (rms_+(r)^2 - rms_-(r)^2)/(4 alpha(r) delta) + d0(r)
# I take a pair of CCDs on the corner, divide the focus CCDs into radian bins, calculate focus error d for each radial bin with alpha and d0 values at this radius, and then take median of these focus errors for all the radian bins and CCD pairs.
# rms^2 is measured by shape.simple. Although I intend to include minimum measurement bias, there exists still some bias. This is corrected by getCorrectedFocusError() at the end, which is a polynomial function derived by calibration data (well-behaved focus sweeps).
# set up radial bins
lRadialBinEdges = config.radialBinEdges
lRadialBinCenters = config.radialBinCenters
lRadialBinsLowerEdges = lRadialBinEdges[0:-1]
lRadialBinsUpperEdges = lRadialBinEdges[1:]
# make selection on data and get rms^2 for each bin, CCD by CCD
dlRmssq = dict(
) # rmssq list for radial bin, which is dictionary for each ccd
for ccdId in dIcSrc.keys():
# use only objects classified as PSF candidate
icSrc = dIcSrc[ccdId][dIcSrc[ccdId].get("hscPipeline_focus_candidate")]
# prepare for getting distance from center for each object
ccd = dCcd[ccdId]
x1, y1 = dCcdDims[ccdId]
# Get focal plane position in pixels
# Note that we constructed the zemax values alpha(r), d0(r), and this r is in pixel.
transform = ccd.getTransformMap().get(
ccd.makeCameraSys(afwCameraGeom.FOCAL_PLANE))
uLlc, vLlc = transform.forwardTransform(afwGeom.PointD(0., 0.))
uLrc, vLrc = transform.forwardTransform(afwGeom.PointD(x1, 0.))
uUlc, vUlc = transform.forwardTransform(afwGeom.PointD(0., y1))
uUrc, vUrc = transform.forwardTransform(afwGeom.PointD(x1, y1))
lDistanceFromCenter = list()
lRmssq = list()
for s in icSrc:
# reject blended objects
if len(s.getFootprint().getPeaks()) != 1:
continue
# calculate distance from center for each objects
x = s.getX()
y = s.getY()
uL = (uLrc - uLlc) / x1 * x + uLlc
uU = (uUrc - uUlc) / x1 * x + uUlc
u = (uU - uL) / y1 * y + uL
vL = (vLrc - vLlc) / x1 * x + vLlc
vU = (vUrc - vUlc) / x1 * x + vUlc
v = (vU - vL) / y1 * y + vL
lDistanceFromCenter.append(np.sqrt(u**2 + v**2))
# calculate rms^2
ixx = s.get(config.shape + "_xx")
iyy = s.get(config.shape + "_yy")
lRmssq.append((ixx + iyy) *
config.pixelScale**2) # convert from pixel^2 to mm^2
# calculate median rms^2 for each radial bin
lDistanceFromCenter = np.array(lDistanceFromCenter)
lRmssq = np.array(lRmssq)
lRmssqMedian = list()
for radialBinLowerEdge, radialBinUpperEdge in zip(
lRadialBinsLowerEdges, lRadialBinsUpperEdges):
sel = np.logical_and(lDistanceFromCenter > radialBinLowerEdge,
lDistanceFromCenter < radialBinUpperEdge)
lRmssqMedian.append(np.median(lRmssq[sel]))
dlRmssq[ccdId] = np.ma.masked_array(lRmssqMedian,
mask=np.isnan(lRmssqMedian))
# get ZEMAX values
d = np.loadtxt(zemaxFilename)
interpStyle = afwMath.stringToInterpStyle("NATURAL_SPLINE")
sAlpha = afwMath.makeInterpolate(d[:, 0], d[:, 1], interpStyle).interpolate
sD0 = afwMath.makeInterpolate(d[:, 0], d[:, 2], interpStyle).interpolate
# calculate rms^2 for each CCD pair
lCcdPairs = zip(config.belowList, config.aboveList)
llFocurErrors = list()
for ccdPair in lCcdPairs:
lFocusErrors = list()
if (ccdPair[0] not in dlRmssq or ccdPair[1] not in dlRmssq
or dlRmssq[ccdPair[0]] is None or dlRmssq[ccdPair[1]] is None):
continue
for i, radialBinCenter in enumerate(lRadialBinCenters):
rmssqAbove = dlRmssq[ccdPair[1]][i]
rmssqBelow = dlRmssq[ccdPair[0]][i]
rmssqDiff = rmssqAbove - rmssqBelow
delta = getFocusCcdOffset(ccdPair[1], config)
alpha = sAlpha(radialBinCenter)
focusError = rmssqDiff / 4. / alpha / delta + sD0(radialBinCenter)
lFocusErrors.append(focusError)
llFocurErrors.append(np.array(lFocusErrors))
llFocurErrors = np.ma.masked_array(llFocurErrors,
mask=np.isnan(llFocurErrors))
reconstructedFocusError = np.ma.median(llFocurErrors)
n = np.sum(np.invert(llFocurErrors.mask))
reconstructedFocusErrorStd = np.ma.std(llFocurErrors) * np.sqrt(
np.pi / 2.) / np.sqrt(n)
if config.doPlot == True:
if not plotFilename:
raise ValueError("no filename for focus plot")
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
lMarker = ["o", "x", "d", "^", "<", ">"]
lColor = ["blue", "green", "red", "cyan", "magenta", "yellow"]
for i, ccdPair in enumerate(lCcdPairs):
delta_plot = np.ma.masked_array([
getFocusCcdOffset(ccdPair[0], config),
getFocusCcdOffset(ccdPair[1], config)
])
rmssq_plot = np.ma.masked_array(
[dlRmssq[ccdPair[0]], dlRmssq[ccdPair[1]]])
for j in range(len(lRadialBinCenters)):
plt.plot(delta_plot,
rmssq_plot[:, j],
"%s--" % lMarker[i],
color=lColor[j])
plt.savefig(plotFilename)
correctedFocusError, correctedFocusErrorStd = getCorrectedFocusError(
reconstructedFocusError, reconstructedFocusErrorStd, config.corrCoeff)
return (correctedFocusError[0], correctedFocusErrorStd[0],
reconstructedFocusError[0], reconstructedFocusErrorStd, n)
def overscanCorrection(ampMaskedImage,
overscanImage,
fitType='MEDIAN',
order=1,
collapseRej=3.0,
statControl=None):
"""Apply overscan correction in place
@param[in,out] ampMaskedImage masked image to correct
@param[in] overscanImage overscan data as an afw.image.Image or afw.image.MaskedImage.
If a masked image is passed in the mask plane will be used
to constrain the fit of the bias level.
@param[in] fitType type of fit for overscan correction; one of:
- 'MEAN'
- 'MEDIAN'
- 'POLY' (ordinary polynomial)
- 'CHEB' (Chebyshev polynomial)
- 'LEG' (Legendre polynomial)
- 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE' (splines)
@param[in] order polynomial order or spline knots (ignored unless fitType
indicates a polynomial or spline)
@param[in] collapseRej Rejection threshold (sigma) for collapsing dimension of overscan
@param[in] statControl Statistics control object
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
if fitType == 'MEAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEAN,
statControl).getValue(afwMath.MEAN)
elif fitType == 'MEDIAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEDIAN,
statControl).getValue(afwMath.MEDIAN)
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE',
'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(
overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74 * (percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(
diff > collapseRej * stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(
biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0 * numpy.arange(num) / float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {
"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(
len(collapsed) * [numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices,
bins=numBins,
weights=1 -
collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
values = numpy.histogram(
indices, bins=numBins,
weights=collapsed.data * ~collapsedMask)[0] / numPerBin
binCenters = numpy.histogram(
indices, bins=numBins,
weights=indices * ~collapsedMask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask],
collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in (
"",
"c",
):
break
if ans in ("p", ):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
figure.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray(
).transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' % \
("overscanCorrection", fitType))
ampImage -= offImage
def overscanCorrection(ampMaskedImage,
overscanImage,
fitType='MEDIAN',
order=1,
collapseRej=3.0,
statControl=None,
overscanIsInt=True):
"""Apply overscan correction in place.
Parameters
----------
ampMaskedImage : `lsst.afw.image.MaskedImage`
Image of amplifier to correct; modified.
overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
Image of overscan; modified.
fitType : `str`
Type of fit for overscan correction. May be one of:
- ``MEAN``: use mean of overscan.
- ``MEANCLIP``: use clipped mean of overscan.
- ``MEDIAN``: use median of overscan.
- ``POLY``: fit with ordinary polynomial.
- ``CHEB``: fit with Chebyshev polynomial.
- ``LEG``: fit with Legendre polynomial.
- ``NATURAL_SPLINE``: fit with natural spline.
- ``CUBIC_SPLINE``: fit with cubic spline.
- ``AKIMA_SPLINE``: fit with Akima spline.
order : `int`
Polynomial order or number of spline knots; ignored unless
``fitType`` indicates a polynomial or spline.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object. In particular, we pay attention to numSigmaClip
overscanIsInt : `bool`
Treat the overscan region as consisting of integers, even if it's been
converted to float. E.g. handle ties properly.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``imageFit``: Value(s) removed from image (scalar or
`lsst.afw.image.Image`)
- ``overscanFit``: Value(s) removed from overscan (scalar or
`lsst.afw.image.Image`)
- ``overscanImage``: Overscan corrected overscan region
(`lsst.afw.image.Image`)
Raises
------
pexExcept.Exception
Raised if ``fitType`` is not an allowed value.
Notes
-----
The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
subtracted. Note that the ``overscanImage`` should not be a subimage of
the ``ampMaskedImage``, to avoid being subtracted twice.
Debug plots are available for the SPLINE fitTypes by setting the
`debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
plots show the scatter plot of the overscan data (collapsed along
the perpendicular dimension) as a function of position on the CCD
(normalized between +/-1).
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
numSigmaClip = statControl.getNumSigmaClip()
if fitType in ('MEAN', 'MEANCLIP'):
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImage, fitType,
statControl).getValue()
overscanFit = offImage
elif fitType in ('MEDIAN', ):
if overscanIsInt:
# we need an image with integer pixels to handle ties properly
if hasattr(overscanImage, "image"):
imageI = overscanImage.image.convertI()
overscanImageI = afwImage.MaskedImageI(imageI,
overscanImage.mask,
overscanImage.variance)
else:
overscanImageI = overscanImage.convertI()
else:
overscanImageI = overscanImage
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImageI, fitType,
statControl).getValue()
overscanFit = offImage
if overscanIsInt:
del overscanImageI
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE',
'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(
overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74 * (percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(
diff > numSigmaClip * stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(
biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0 * numpy.arange(num) / float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {
"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(
len(collapsed) * [numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices,
bins=numBins,
weights=1 -
collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
with numpy.errstate(invalid="ignore"): # suppress NAN warnings
values = numpy.histogram(
indices,
bins=numBins,
weights=collapsed.data * ~collapsedMask)[0] / numPerBin
binCenters = numpy.histogram(
indices, bins=numBins,
weights=indices * ~collapsedMask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask],
collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
plot.xlabel("centered/scaled position along overscan region")
plot.ylabel("pixel value/fit value")
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in (
"",
"c",
):
break
if ans in ("p", ):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
plot.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
overscanFit = afwImage.ImageF(overscanImage.getDimensions())
overscanArray = overscanFit.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray(
).transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' %
("overscanCorrection", fitType))
ampImage -= offImage
overscanImage -= overscanFit
return Struct(imageFit=offImage,
overscanFit=overscanFit,
overscanImage=overscanImage)
def getDistanceFromFocus(dIcSrc, dCcd, dCcdDims, zemaxFilename, config, plotFilename=None):
# Focus error is measured by using rms^2 of stars on focus CCDs.
# If there is a focus error d, rms^2 can be written as
# rms^2 = rms_atm^2 + rms_opt_0^2 + alpha*d^2,
# where rms_atm is from atmosphere and rms_opt if from optics with out any focus error.
# On the focus CCDs which have +/-delta offset, the equation becomes
# rms_+^2 = rms_atm^2 + rms_opt_0^2 + alpha(d+delta)^2
# rms_-^2 = rms_atm^2 + rms_opt_0^2 + alpha(d-delta)^2
# Thus, the difference of these rms^2 gives the focus error as
# d = (rms_+^2 - rms_-^2)/(4 alpha delta)
# alpha is determined by ZEMAX simulations. It turned out that alpha is a function of distance from the center of FOV r.
# Also the best focus varies as a function of r. Thus the focus error can be rewritten as
# d(r) = (rms_+(r)^2 - rms_-(r)^2)/(4 alpha(r) delta) + d0(r)
# I take a pair of CCDs on the corner, divide the focus CCDs into radian bins, calculate focus error d for each radial bin with alpha and d0 values at this radius, and then take median of these focus errors for all the radian bins and CCD pairs.
# rms^2 is measured by shape.simple. Although I intend to include minimum measurement bias, there exists still some bias. This is corrected by getCorrectedFocusError() at the end, which is a polynomial function derived by calibration data (well-behaved focus sweeps).
# set up radial bins
lRadialBinEdges = config.radialBinEdges
lRadialBinCenters = config.radialBinCenters
lRadialBinsLowerEdges = lRadialBinEdges[0:-1]
lRadialBinsUpperEdges = lRadialBinEdges[1:]
# make selection on data and get rms^2 for each bin, CCD by CCD
dlRmssq = dict() # rmssq list for radial bin, which is dictionary for each ccd
for ccdId in dIcSrc.keys():
# use only objects classified as PSF candidate
icSrc = dIcSrc[ccdId][dIcSrc[ccdId].get("hscPipeline_focus_candidate")]
# prepare for getting distance from center for each object
ccd = dCcd[ccdId]
x1, y1 = dCcdDims[ccdId]
# Get focal plane position in pixels
# Note that we constructed the zemax values alpha(r), d0(r), and this r is in pixel.
transform = ccd.getTransformMap().get(ccd.makeCameraSys(afwCameraGeom.FOCAL_PLANE))
uLlc, vLlc = transform.forwardTransform(afwGeom.PointD(0., 0.))
uLrc, vLrc = transform.forwardTransform(afwGeom.PointD(x1, 0.))
uUlc, vUlc = transform.forwardTransform(afwGeom.PointD(0., y1))
uUrc, vUrc = transform.forwardTransform(afwGeom.PointD(x1, y1))
lDistanceFromCenter = list()
lRmssq = list()
for s in icSrc:
# reject blended objects
if len(s.getFootprint().getPeaks()) != 1:
continue
# calculate distance from center for each objects
x = s.getX()
y = s.getY()
uL = (uLrc-uLlc)/x1*x+uLlc
uU = (uUrc-uUlc)/x1*x+uUlc
u = (uU-uL)/y1*y+uL
vL = (vLrc-vLlc)/x1*x+vLlc
vU = (vUrc-vUlc)/x1*x+vUlc
v = (vU-vL)/y1*y+vL
lDistanceFromCenter.append(np.sqrt(u**2 + v**2))
# calculate rms^2
ixx = s.get(config.shape + "_xx")
iyy = s.get(config.shape + "_yy")
lRmssq.append((ixx + iyy)*config.pixelScale**2) # convert from pixel^2 to mm^2
# calculate median rms^2 for each radial bin
lDistanceFromCenter = np.array(lDistanceFromCenter)
lRmssq = np.array(lRmssq)
lRmssqMedian = list()
for radialBinLowerEdge, radialBinUpperEdge in zip(lRadialBinsLowerEdges, lRadialBinsUpperEdges):
sel = np.logical_and(lDistanceFromCenter > radialBinLowerEdge, lDistanceFromCenter < radialBinUpperEdge)
lRmssqMedian.append(np.median(lRmssq[sel]))
dlRmssq[ccdId] = np.ma.masked_array(lRmssqMedian, mask = np.isnan(lRmssqMedian))
# get ZEMAX values
d = np.loadtxt(zemaxFilename)
interpStyle = afwMath.stringToInterpStyle("NATURAL_SPLINE")
sAlpha = afwMath.makeInterpolate(d[:,0], d[:,1], interpStyle).interpolate
sD0 = afwMath.makeInterpolate(d[:,0], d[:,2], interpStyle).interpolate
# calculate rms^2 for each CCD pair
lCcdPairs = zip(config.belowList, config.aboveList)
llFocurErrors = list()
for ccdPair in lCcdPairs:
lFocusErrors = list()
if (ccdPair[0] not in dlRmssq or ccdPair[1] not in dlRmssq or
dlRmssq[ccdPair[0]] is None or dlRmssq[ccdPair[1]] is None):
continue
for i, radialBinCenter in enumerate(lRadialBinCenters):
rmssqAbove = dlRmssq[ccdPair[1]][i]
rmssqBelow = dlRmssq[ccdPair[0]][i]
rmssqDiff = rmssqAbove - rmssqBelow
delta = getFocusCcdOffset(ccdPair[1], config)
alpha = sAlpha(radialBinCenter)
focusError = rmssqDiff/4./alpha/delta + sD0(radialBinCenter)
lFocusErrors.append(focusError)
llFocurErrors.append(np.array(lFocusErrors))
llFocurErrors = np.ma.masked_array(llFocurErrors, mask = np.isnan(llFocurErrors))
reconstructedFocusError = np.ma.median(llFocurErrors)
n = np.sum(np.invert(llFocurErrors.mask))
reconstructedFocusErrorStd= np.ma.std(llFocurErrors)*np.sqrt(np.pi/2.)/np.sqrt(n)
if config.doPlot == True:
if not plotFilename:
raise ValueError("no filename for focus plot")
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
lMarker = ["o", "x", "d", "^", "<", ">"]
lColor = ["blue", "green", "red", "cyan", "magenta", "yellow"]
for i, ccdPair in enumerate(lCcdPairs):
delta_plot = np.ma.masked_array([getFocusCcdOffset(ccdPair[0], config),
getFocusCcdOffset(ccdPair[1], config)])
rmssq_plot = np.ma.masked_array([dlRmssq[ccdPair[0]], dlRmssq[ccdPair[1]]])
for j in range(len(lRadialBinCenters)):
plt.plot(delta_plot, rmssq_plot[:, j], "%s--" % lMarker[i], color = lColor[j])
plt.savefig(plotFilename)
correctedFocusError, correctedFocusErrorStd = getCorrectedFocusError(
reconstructedFocusError, reconstructedFocusErrorStd, config.corrCoeff)
return (correctedFocusError[0], correctedFocusErrorStd[0],
reconstructedFocusError[0], reconstructedFocusErrorStd, n)
def referenceImage(image, detector, linearityType, inputData, table=None):
"""Generate a reference linearization.
Parameters
----------
image: `lsst.afw.image.Image`
Image to linearize.
detector: `lsst.afw.cameraGeom.Detector`
Detector this image is from.
linearityType: `str`
Type of linearity to apply.
inputData: `numpy.array`
An array of values for the linearity correction.
table: `numpy.array`, optional
An optional lookup table to use.
Returns
-------
outImage: `lsst.afw.image.Image`
The output linearized image.
numOutOfRange: `int`
The number of values that could not be linearized.
Raises
------
RuntimeError :
Raised if an invalid linearityType is supplied.
"""
numOutOfRange = 0
for ampIdx, amp in enumerate(detector.getAmplifiers()):
ampIdx = (ampIdx // 3, ampIdx % 3)
bbox = amp.getBBox()
imageView = image.Factory(image, bbox)
if linearityType == 'Squared':
sqCoeff = inputData[ampIdx]
array = imageView.getArray()
array[:] = array + sqCoeff * array**2
elif linearityType == 'LookupTable':
rowInd, colIndOffset = inputData[ampIdx]
rowInd = int(rowInd)
tableRow = table[rowInd, :]
numOutOfRange += applyLookupTable(imageView, tableRow,
colIndOffset)
elif linearityType == 'Polynomial':
coeffs = inputData[ampIdx]
array = imageView.getArray()
summation = np.zeros_like(array)
for index, coeff in enumerate(coeffs):
summation += coeff * np.power(array, (index + 2))
array += summation
elif linearityType == 'Spline':
centers, values = np.split(inputData, 2) # This uses the full data
interp = afwMath.makeInterpolate(
centers.tolist(), values.tolist(),
afwMath.stringToInterpStyle('AKIMA_SPLINE'))
array = imageView.getArray()
delta = interp.interpolate(array.flatten())
array -= np.array(delta).reshape(array.shape)
else:
raise RuntimeError(f"Unknown linearity: {linearityType}")
return image, numOutOfRange
def run(self, inputPtc, camera, inputDims):
"""Fit non-linearity to PTC data, returning the correct Linearizer
object.
Parameters
----------
inputPtc : `lsst.cp.pipe.PtcDataset`
Pre-measured PTC dataset.
camera : `lsst.afw.cameraGeom.Camera`
Camera geometry.
inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
DataIds to use to populate the output calibration.
Returns
-------
results : `lsst.pipe.base.Struct`
The results struct containing:
``outputLinearizer`` : `lsst.ip.isr.Linearizer`
Final linearizer calibration.
``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
Provenance data for the new calibration.
Notes
-----
This task currently fits only polynomial-defined corrections,
where the correction coefficients are defined such that:
corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i)
These `c_i` are defined in terms of the direct polynomial fit:
meanVector ~ P(x=timeVector) = sum_j k_j x^j
such that c_(j-2) = -k_j/(k_1^j) in units of DN^(1-j) (c.f.,
Eq. 37 of 2003.05978). The `config.polynomialOrder` or
`config.splineKnots` define the maximum order of x^j to fit.
As k_0 and k_1 are degenerate with bias level and gain, they
are not included in the non-linearity correction.
"""
detector = camera[inputDims['detector']]
if self.config.linearityType == 'LookupTable':
table = np.zeros((len(detector), self.config.maxLookupTableAdu),
dtype=np.float32)
tableIndex = 0
else:
table = None
tableIndex = None # This will fail if we increment it.
if self.config.linearityType == 'Spline':
fitOrder = self.config.splineKnots
else:
fitOrder = self.config.polynomialOrder
# Initialize the linearizer.
linearizer = Linearizer(detector=detector, table=table, log=self.log)
for i, amp in enumerate(detector):
ampName = amp.getName()
if (len(inputPtc.expIdMask[ampName]) == 0):
self.log.warn(
f"Mask not found for {ampName} in non-linearity fit. Using all points."
)
mask = np.repeat(True, len(inputPtc.expIdMask[ampName]))
else:
mask = inputPtc.expIdMask[ampName]
inputAbscissa = np.array(inputPtc.rawExpTimes[ampName])[mask]
inputOrdinate = np.array(inputPtc.rawMeans[ampName])[mask]
# Determine proxy-to-linear-flux transformation
fluxMask = inputOrdinate < self.config.maxLinearAdu
lowMask = inputOrdinate > self.config.minLinearAdu
fluxMask = fluxMask & lowMask
linearAbscissa = inputAbscissa[fluxMask]
linearOrdinate = inputOrdinate[fluxMask]
linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 100.0],
linearAbscissa,
linearOrdinate,
funcPolynomial)
# Convert this proxy-to-flux fit into an expected linear flux
linearOrdinate = linearFit[0] + linearFit[1] * inputAbscissa
# Exclude low end outliers
threshold = self.config.nSigmaClipLinear * np.sqrt(linearOrdinate)
fluxMask = np.abs(inputOrdinate - linearOrdinate) < threshold
linearOrdinate = linearOrdinate[fluxMask]
fitOrdinate = inputOrdinate[fluxMask]
self.debugFit('linearFit', inputAbscissa, inputOrdinate,
linearOrdinate, fluxMask, ampName)
# Do fits
if self.config.linearityType in [
'Polynomial', 'Squared', 'LookupTable'
]:
polyFit = np.zeros(fitOrder + 1)
polyFit[1] = 1.0
polyFit, polyFitErr, chiSq, weights = irlsFit(
polyFit, linearOrdinate, fitOrdinate, funcPolynomial)
# Truncate the polynomial fit
k1 = polyFit[1]
linearityFit = [
-coeff / (k1**order) for order, coeff in enumerate(polyFit)
]
significant = np.where(
np.abs(linearityFit) > 1e-10, True, False)
self.log.info(f"Significant polynomial fits: {significant}")
modelOrdinate = funcPolynomial(polyFit, linearAbscissa)
self.debugFit('polyFit', linearAbscissa, fitOrdinate,
modelOrdinate, None, ampName)
if self.config.linearityType == 'Squared':
linearityFit = [linearityFit[2]]
elif self.config.linearityType == 'LookupTable':
# Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
tMax = (self.config.maxLookupTableAdu -
polyFit[0]) / polyFit[1]
timeRange = np.linspace(0, tMax,
self.config.maxLookupTableAdu)
signalIdeal = polyFit[0] + polyFit[1] * timeRange
signalUncorrected = funcPolynomial(polyFit, timeRange)
lookupTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has correction
linearizer.tableData[tableIndex, :] = lookupTableRow
linearityFit = [tableIndex, 0]
tableIndex += 1
elif self.config.linearityType in ['Spline']:
# See discussion in `lsst.ip.isr.linearize.py` before modifying.
numPerBin, binEdges = np.histogram(linearOrdinate,
bins=fitOrder)
with np.errstate(invalid="ignore"):
# Algorithm note: With the counts of points per
# bin above, the next histogram calculates the
# values to put in each bin by weighting each
# point by the correction value.
values = np.histogram(
linearOrdinate,
bins=fitOrder,
weights=(inputOrdinate[fluxMask] -
linearOrdinate))[0] / numPerBin
# After this is done, the binCenters are
# calculated by weighting by the value we're
# binning over. This ensures that widely
# spaced/poorly sampled data aren't assigned to
# the midpoint of the bin (as could be done using
# the binEdges above), but to the weighted mean of
# the inputs. Note that both histograms are
# scaled by the count per bin to normalize what
# the histogram returns (a sum of the points
# inside) into an average.
binCenters = np.histogram(
linearOrdinate, bins=fitOrder,
weights=linearOrdinate)[0] / numPerBin
values = values[numPerBin > 0]
binCenters = binCenters[numPerBin > 0]
self.debugFit('splineFit', binCenters, np.abs(values), values,
None, ampName)
interp = afwMath.makeInterpolate(
binCenters.tolist(), values.tolist(),
afwMath.stringToInterpStyle("AKIMA_SPLINE"))
modelOrdinate = linearOrdinate + interp.interpolate(
linearOrdinate)
self.debugFit('splineFit', linearOrdinate, fitOrdinate,
modelOrdinate, None, ampName)
# If we exclude a lot of points, we may end up with
# less than fitOrder points. Pad out the low-flux end
# to ensure equal lengths.
if len(binCenters) != fitOrder:
padN = fitOrder - len(binCenters)
binCenters = np.pad(binCenters, (padN, 0),
'linear_ramp',
end_values=(binCenters.min() - 1.0, ))
# This stores the correction, which is zero at low values.
values = np.pad(values, (padN, 0))
# Pack the spline into a single array.
linearityFit = np.concatenate(
(binCenters.tolist(), values.tolist())).tolist()
polyFit = [0.0]
polyFitErr = [0.0]
chiSq = np.nan
else:
polyFit = [0.0]
polyFitErr = [0.0]
chiSq = np.nan
linearityFit = [0.0]
linearizer.linearityType[ampName] = self.config.linearityType
linearizer.linearityCoeffs[ampName] = np.array(linearityFit)
linearizer.linearityBBox[ampName] = amp.getBBox()
linearizer.fitParams[ampName] = np.array(polyFit)
linearizer.fitParamsErr[ampName] = np.array(polyFitErr)
linearizer.fitChiSq[ampName] = chiSq
image = afwImage.ImageF(len(inputOrdinate), 1)
image.getArray()[:, :] = inputOrdinate
linearizeFunction = linearizer.getLinearityTypeByName(
linearizer.linearityType[ampName])
linearizeFunction()(image, **{
'coeffs': linearizer.linearityCoeffs[ampName],
'table': linearizer.tableData,
'log': linearizer.log
})
linearizeModel = image.getArray()[0, :]
self.debugFit('solution', inputOrdinate[fluxMask], linearOrdinate,
linearizeModel[fluxMask], None, ampName)
linearizer.hasLinearity = True
linearizer.validate()
linearizer.updateMetadata(camera=camera,
detector=detector,
filterName='NONE')
linearizer.updateMetadata(setDate=True, setCalibId=True)
provenance = IsrProvenance(calibType='linearizer')
return pipeBase.Struct(
outputLinearizer=linearizer,
outputProvenance=provenance,
)
