def testStringToInterpStyle(self):
for name, desiredStyle in afwMath.Interpolate.Style.__members__.items():
if name in ("UNKNOWN", "NUM_STYLES"):
with self.assertRaises(pexExcept.InvalidParameterError):
afwMath.stringToInterpStyle(name)
else:
style = afwMath.stringToInterpStyle(name)
self.assertEqual(style, desiredStyle)
for badName in ("BOGUS", ""):
with self.assertRaises(pexExcept.InvalidParameterError):
afwMath.stringToInterpStyle(badName)
def exposureToBackground(bgExp):
"""Convert an exposure to background model
Calibs need to be persisted as an Exposure, so we need to convert
the persisted Exposure to a background model.
Parameters
----------
bgExp : `lsst.afw.image.Exposure`
Background model in Exposure format.
Returns
-------
bg : `lsst.afw.math.BackgroundList`
Background model
"""
header = bgExp.getMetadata()
xMin = header.getScalar("BOX.MINX")
yMin = header.getScalar("BOX.MINY")
xMax = header.getScalar("BOX.MAXX")
yMax = header.getScalar("BOX.MAXY")
algorithm = header.getScalar("ALGORITHM")
bbox = afwGeom.Box2I(afwGeom.Point2I(xMin, yMin), afwGeom.Point2I(xMax, yMax))
return afwMath.BackgroundList(
(afwMath.BackgroundMI(bbox, bgExp.getMaskedImage()),
afwMath.stringToInterpStyle(algorithm),
afwMath.stringToUndersampleStyle("REDUCE_INTERP_ORDER"),
afwMath.ApproximateControl.UNKNOWN,
0, 0, False))
def interpolateBadPixels(array, isBad, interpolationStyle):
"""Interpolate bad pixels in an image array
The bad pixels are modified in the array.
Parameters
----------
array : `numpy.ndarray`
Image array with bad pixels.
isBad : `numpy.ndarray` of type `bool`
Boolean array indicating which pixels are bad.
interpolationStyle : `str`
Style for interpolation (see `lsst.afw.math.Background`);
supported values are CONSTANT, LINEAR, NATURAL_SPLINE,
AKIMA_SPLINE.
"""
if numpy.all(isBad):
raise RuntimeError("No good pixels in image array")
height, width = array.shape
xIndices = numpy.arange(width, dtype=float)
yIndices = numpy.arange(height, dtype=float)
method = afwMath.stringToInterpStyle(interpolationStyle)
isGood = ~isBad
for y in range(height):
if numpy.any(isBad[y, :]) and numpy.any(isGood[y, :]):
array[y][isBad[y]] = interpolate1D(method, xIndices[isGood[y]], array[y][isGood[y]],
xIndices[isBad[y]])
isBad = numpy.isnan(array)
isGood = ~isBad
for x in range(width):
if numpy.any(isBad[:, x]) and numpy.any(isGood[:, x]):
array[:, x][isBad[:, x]] = interpolate1D(method, yIndices[isGood[:, x]],
array[:, x][isGood[:, x]], yIndices[isBad[:, x]])
def __call__(self, image, **kwargs):
"""Correct for non-linearity.
Parameters
----------
image : `lsst.afw.image.Image`
Image to be corrected
kwargs : `dict`
Dictionary of parameter keywords:
``"coeffs"``
Coefficient vector (`list` or `numpy.array`).
``"log"``
Logger to handle messages (`lsst.log.Log`).
Returns
-------
output : `tuple` [`bool`, `int`]
If true, a correction was applied successfully. The
integer indicates the number of pixels that were
uncorrectable by being out of range.
"""
splineCoeff = kwargs['coeffs']
centers, values = np.split(splineCoeff, 2)
interp = afwMath.makeInterpolate(centers.tolist(), values.tolist(),
afwMath.stringToInterpStyle("AKIMA_SPLINE"))
ampArr = image.getArray()
delta = interp.interpolate(ampArr.flatten())
ampArr -= np.array(delta).reshape(ampArr.shape)
return True, 0
def splineFit(self, indices, collapsed, numBins):
"""Wrapper function to match spline fit API to polynomial fit API.
Parameters
----------
indices : `numpy.ndarray`
Locations to evaluate the spline.
collapsed : `numpy.ndarray`
Collapsed overscan values corresponding to the spline
evaluation points.
numBins : `int`
Number of bins to use in constructing the spline.
Returns
-------
interp : `lsst.afw.math.Interpolate`
Interpolation object for later evaluation.
"""
if not np.ma.is_masked(collapsed):
collapsed.mask = np.array(len(collapsed) * [np.ma.nomask])
numPerBin, binEdges = np.histogram(indices,
bins=numBins,
weights=1 -
collapsed.mask.astype(int))
with np.errstate(invalid="ignore"):
values = np.histogram(
indices,
bins=numBins,
weights=collapsed.data * ~collapsed.mask)[0] / numPerBin
binCenters = np.histogram(
indices, bins=numBins,
weights=indices * ~collapsed.mask)[0] / numPerBin
if len(binCenters[numPerBin > 0]) < 5:
self.log.warn(
"Cannot do spline fitting for overscan: %s valid points.",
len(binCenters[numPerBin > 0]))
# Return a scalar value if we have one, otherwise
# return zero. This amplifier is hopefully already
# masked.
if len(values[numPerBin > 0]) != 0:
return float(values[numPerBin > 0][0])
else:
return 0.0
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(self.config.fitType))
return interp
def toCcdBackground(self, detector, bbox):
"""Produce a background model for a CCD
The superpixel background model is warped back to the
CCD frame, for application to the individual CCD.
Parameters
----------
detector : `lsst.afw.cameraGeom.Detector`
CCD for which to produce background model.
bbox : `lsst.geom.Box2I`
Bounding box of CCD exposure.
Returns
-------
bg : `lsst.afw.math.BackgroundList`
Background model for CCD.
"""
transform = detector.getTransformMap().getTransform(
detector.makeCameraSys(afwCameraGeom.PIXELS),
detector.makeCameraSys(afwCameraGeom.FOCAL_PLANE))
binTransform = (
geom.AffineTransform.makeScaling(self.config.binning) *
geom.AffineTransform.makeTranslation(geom.Extent2D(0.5, 0.5)))
# Binned image on CCD --> unbinned image on CCD --> focal plane --> binned focal plane
toSample = afwGeom.makeTransform(binTransform).then(transform).then(
self.transform)
focalPlane = self.getStatsImage()
fpNorm = afwImage.ImageF(focalPlane.getBBox())
fpNorm.set(1.0)
image = afwImage.ImageF(bbox.getDimensions() // self.config.binning)
norm = afwImage.ImageF(image.getBBox())
ctrl = afwMath.WarpingControl("bilinear")
afwMath.warpImage(image, focalPlane, toSample.inverted(), ctrl)
afwMath.warpImage(norm, fpNorm, toSample.inverted(), ctrl)
image /= norm
mask = afwImage.Mask(image.getBBox())
isBad = numpy.isnan(image.getArray())
mask.getArray()[isBad] = mask.getPlaneBitMask("BAD")
image.getArray()[isBad] = image.getArray()[~isBad].mean()
return afwMath.BackgroundList(
(afwMath.BackgroundMI(bbox, afwImage.makeMaskedImage(image, mask)),
afwMath.stringToInterpStyle(self.config.interpolation),
afwMath.stringToUndersampleStyle("REDUCE_INTERP_ORDER"),
afwMath.ApproximateControl.UNKNOWN, 0, 0, False))
def measureBackground(self, image):
"""Measure a background model for image
This doesn't use a full-featured background model (e.g., no Chebyshev
approximation) because we just want the binning behaviour. This will
allow us to average the bins later (`averageBackgrounds`).
The `BackgroundMI` is wrapped in a `BackgroundList` so it can be
pickled and persisted.
Parameters
----------
image : `lsst.afw.image.MaskedImage`
Image for which to measure background.
Returns
-------
bgModel : `lsst.afw.math.BackgroundList`
Background model.
"""
stats = afwMath.StatisticsControl()
stats.setAndMask(image.getMask().getPlaneBitMask(self.config.background.mask))
stats.setNanSafe(True)
ctrl = afwMath.BackgroundControl(
self.config.background.algorithm,
max(int(image.getWidth()/self.config.background.xBinSize + 0.5), 1),
max(int(image.getHeight()/self.config.background.yBinSize + 0.5), 1),
"REDUCE_INTERP_ORDER",
stats,
self.config.background.statistic
)
bg = afwMath.makeBackground(image, ctrl)
return afwMath.BackgroundList((
bg,
afwMath.stringToInterpStyle(self.config.background.algorithm),
afwMath.stringToUndersampleStyle("REDUCE_INTERP_ORDER"),
afwMath.ApproximateControl.UNKNOWN,
0, 0, False
))
def __init__(self, interpStyle, xList, yList, scaleList):
"""Construct an SdssImageScaler
@warning: scaleErrList is presently not used
@param[in] interpStyle: interpolation style (see lsst.afw.math.Interpolate for options)
@param[in] xList: list of X pixel positions
@param[in] yList: list of Y pixel positions
@param[in] scaleList: list of multiplicative scales at (x,y)
@raise RuntimeError if the lists have different lengths
"""
if len(xList) != len(yList) or len(xList) != len(scaleList):
raise RuntimeError(
"len(xList)=%s len(yList)=%s, len(scaleList)=%s but all lists must have the same length"
% (len(xList), len(yList), len(scaleList)))
self.interpStyle = afwMath.stringToInterpStyle(interpStyle)
self._xList = xList
self._yList = yList
self._scaleList = scaleList
def splineFit(self, indices, collapsed, numBins):
"""Wrapper function to match spline fit API to polynomial fit API.
Parameters
----------
indices : `numpy.ndarray`
Locations to evaluate the spline.
collapsed : `numpy.ndarray`
Collapsed overscan values corresponding to the spline
evaluation points.
numBins : `int`
Number of bins to use in constructing the spline.
Returns
-------
interp : `lsst.afw.math.Interpolate`
Interpolation object for later evaluation.
"""
if not np.ma.is_masked(collapsed):
collapsed.mask = np.array(len(collapsed) * [np.ma.nomask])
numPerBin, binEdges = np.histogram(indices,
bins=numBins,
weights=1 -
collapsed.mask.astype(int))
with np.errstate(invalid="ignore"):
values = np.histogram(
indices,
bins=numBins,
weights=collapsed.data * ~collapsed.mask)[0] / numPerBin
binCenters = np.histogram(
indices, bins=numBins,
weights=indices * ~collapsed.mask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(self.config.fitType))
return interp
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
statControl=None, overscanIsInt=True):
"""Apply overscan correction in place.
Parameters
----------
ampMaskedImage : `lsst.afw.image.MaskedImage`
Image of amplifier to correct; modified.
overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
Image of overscan; modified.
fitType : `str`
Type of fit for overscan correction. May be one of:
- ``MEAN``: use mean of overscan.
- ``MEANCLIP``: use clipped mean of overscan.
- ``MEDIAN``: use median of overscan.
- ``POLY``: fit with ordinary polynomial.
- ``CHEB``: fit with Chebyshev polynomial.
- ``LEG``: fit with Legendre polynomial.
- ``NATURAL_SPLINE``: fit with natural spline.
- ``CUBIC_SPLINE``: fit with cubic spline.
- ``AKIMA_SPLINE``: fit with Akima spline.
order : `int`
Polynomial order or number of spline knots; ignored unless
``fitType`` indicates a polynomial or spline.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object. In particular, we pay attention to numSigmaClip
overscanIsInt : `bool`
Treat the overscan region as consisting of integers, even if it's been
converted to float. E.g. handle ties properly.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``imageFit``: Value(s) removed from image (scalar or
`lsst.afw.image.Image`)
- ``overscanFit``: Value(s) removed from overscan (scalar or
`lsst.afw.image.Image`)
- ``overscanImage``: Overscan corrected overscan region
(`lsst.afw.image.Image`)
Raises
------
pexExcept.Exception
Raised if ``fitType`` is not an allowed value.
Notes
-----
The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
subtracted. Note that the ``overscanImage`` should not be a subimage of
the ``ampMaskedImage``, to avoid being subtracted twice.
Debug plots are available for the SPLINE fitTypes by setting the
`debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
plots show the scatter plot of the overscan data (collapsed along
the perpendicular dimension) as a function of position on the CCD
(normalized between +/-1).
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
numSigmaClip = statControl.getNumSigmaClip()
if fitType in ('MEAN', 'MEANCLIP'):
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue()
overscanFit = offImage
elif fitType in ('MEDIAN',):
if overscanIsInt:
# we need an image with integer pixels to handle ties properly
if hasattr(overscanImage, "image"):
imageI = overscanImage.image.convertI()
overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance)
else:
overscanImageI = overscanImage.convertI()
else:
overscanImageI = overscanImage
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImageI, fitType, statControl).getValue()
overscanFit = offImage
if overscanIsInt:
del overscanImageI
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0*numpy.arange(num)/float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
weights=1-collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
with numpy.errstate(invalid="ignore"): # suppress NAN warnings
values = numpy.histogram(indices, bins=numBins,
weights=collapsed.data*~collapsedMask)[0]/numPerBin
binCenters = numpy.histogram(indices, bins=numBins,
weights=indices*~collapsedMask)[0]/numPerBin
interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
plot.xlabel("centered/scaled position along overscan region")
plot.ylabel("pixel value/fit value")
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in ("", "c",):
break
if ans in ("p",):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
plot.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
overscanFit = afwImage.ImageF(overscanImage.getDimensions())
overscanArray = overscanFit.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType))
ampImage -= offImage
overscanImage -= overscanFit
return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
def getDistanceFromFocus(dIcSrc,
dCcd,
dCcdDims,
zemaxFilename,
config,
plotFilename=None):
# Focus error is measured by using rms^2 of stars on focus CCDs.
# If there is a focus error d, rms^2 can be written as
# rms^2 = rms_atm^2 + rms_opt_0^2 + alpha*d^2,
# where rms_atm is from atmosphere and rms_opt if from optics with out any focus error.
# On the focus CCDs which have +/-delta offset, the equation becomes
# rms_+^2 = rms_atm^2 + rms_opt_0^2 + alpha(d+delta)^2
# rms_-^2 = rms_atm^2 + rms_opt_0^2 + alpha(d-delta)^2
# Thus, the difference of these rms^2 gives the focus error as
# d = (rms_+^2 - rms_-^2)/(4 alpha delta)
# alpha is determined by ZEMAX simulations. It turned out that alpha is a function of distance from the center of FOV r.
# Also the best focus varies as a function of r. Thus the focus error can be rewritten as
# d(r) = (rms_+(r)^2 - rms_-(r)^2)/(4 alpha(r) delta) + d0(r)
# I take a pair of CCDs on the corner, divide the focus CCDs into radian bins, calculate focus error d for each radial bin with alpha and d0 values at this radius, and then take median of these focus errors for all the radian bins and CCD pairs.
# rms^2 is measured by shape.simple. Although I intend to include minimum measurement bias, there exists still some bias. This is corrected by getCorrectedFocusError() at the end, which is a polynomial function derived by calibration data (well-behaved focus sweeps).
# set up radial bins
lRadialBinEdges = config.radialBinEdges
lRadialBinCenters = config.radialBinCenters
lRadialBinsLowerEdges = lRadialBinEdges[0:-1]
lRadialBinsUpperEdges = lRadialBinEdges[1:]
# make selection on data and get rms^2 for each bin, CCD by CCD
dlRmssq = dict(
) # rmssq list for radial bin, which is dictionary for each ccd
for ccdId in dIcSrc.keys():
# use only objects classified as PSF candidate
icSrc = dIcSrc[ccdId][dIcSrc[ccdId].get("hscPipeline_focus_candidate")]
# prepare for getting distance from center for each object
ccd = dCcd[ccdId]
x1, y1 = dCcdDims[ccdId]
# Get focal plane position in pixels
# Note that we constructed the zemax values alpha(r), d0(r), and this r is in pixel.
transform = ccd.getTransformMap().get(
ccd.makeCameraSys(afwCameraGeom.FOCAL_PLANE))
uLlc, vLlc = transform.forwardTransform(afwGeom.PointD(0., 0.))
uLrc, vLrc = transform.forwardTransform(afwGeom.PointD(x1, 0.))
uUlc, vUlc = transform.forwardTransform(afwGeom.PointD(0., y1))
uUrc, vUrc = transform.forwardTransform(afwGeom.PointD(x1, y1))
lDistanceFromCenter = list()
lRmssq = list()
for s in icSrc:
# reject blended objects
if len(s.getFootprint().getPeaks()) != 1:
continue
# calculate distance from center for each objects
x = s.getX()
y = s.getY()
uL = (uLrc - uLlc) / x1 * x + uLlc
uU = (uUrc - uUlc) / x1 * x + uUlc
u = (uU - uL) / y1 * y + uL
vL = (vLrc - vLlc) / x1 * x + vLlc
vU = (vUrc - vUlc) / x1 * x + vUlc
v = (vU - vL) / y1 * y + vL
lDistanceFromCenter.append(np.sqrt(u**2 + v**2))
# calculate rms^2
ixx = s.get(config.shape + "_xx")
iyy = s.get(config.shape + "_yy")
lRmssq.append((ixx + iyy) *
config.pixelScale**2) # convert from pixel^2 to mm^2
# calculate median rms^2 for each radial bin
lDistanceFromCenter = np.array(lDistanceFromCenter)
lRmssq = np.array(lRmssq)
lRmssqMedian = list()
for radialBinLowerEdge, radialBinUpperEdge in zip(
lRadialBinsLowerEdges, lRadialBinsUpperEdges):
sel = np.logical_and(lDistanceFromCenter > radialBinLowerEdge,
lDistanceFromCenter < radialBinUpperEdge)
lRmssqMedian.append(np.median(lRmssq[sel]))
dlRmssq[ccdId] = np.ma.masked_array(lRmssqMedian,
mask=np.isnan(lRmssqMedian))
# get ZEMAX values
d = np.loadtxt(zemaxFilename)
interpStyle = afwMath.stringToInterpStyle("NATURAL_SPLINE")
sAlpha = afwMath.makeInterpolate(d[:, 0], d[:, 1], interpStyle).interpolate
sD0 = afwMath.makeInterpolate(d[:, 0], d[:, 2], interpStyle).interpolate
# calculate rms^2 for each CCD pair
lCcdPairs = zip(config.belowList, config.aboveList)
llFocurErrors = list()
for ccdPair in lCcdPairs:
lFocusErrors = list()
if (ccdPair[0] not in dlRmssq or ccdPair[1] not in dlRmssq
or dlRmssq[ccdPair[0]] is None or dlRmssq[ccdPair[1]] is None):
continue
for i, radialBinCenter in enumerate(lRadialBinCenters):
rmssqAbove = dlRmssq[ccdPair[1]][i]
rmssqBelow = dlRmssq[ccdPair[0]][i]
rmssqDiff = rmssqAbove - rmssqBelow
delta = getFocusCcdOffset(ccdPair[1], config)
alpha = sAlpha(radialBinCenter)
focusError = rmssqDiff / 4. / alpha / delta + sD0(radialBinCenter)
lFocusErrors.append(focusError)
llFocurErrors.append(np.array(lFocusErrors))
llFocurErrors = np.ma.masked_array(llFocurErrors,
mask=np.isnan(llFocurErrors))
reconstructedFocusError = np.ma.median(llFocurErrors)
n = np.sum(np.invert(llFocurErrors.mask))
reconstructedFocusErrorStd = np.ma.std(llFocurErrors) * np.sqrt(
np.pi / 2.) / np.sqrt(n)
if config.doPlot == True:
if not plotFilename:
raise ValueError("no filename for focus plot")
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
lMarker = ["o", "x", "d", "^", "<", ">"]
lColor = ["blue", "green", "red", "cyan", "magenta", "yellow"]
for i, ccdPair in enumerate(lCcdPairs):
delta_plot = np.ma.masked_array([
getFocusCcdOffset(ccdPair[0], config),
getFocusCcdOffset(ccdPair[1], config)
])
rmssq_plot = np.ma.masked_array(
[dlRmssq[ccdPair[0]], dlRmssq[ccdPair[1]]])
for j in range(len(lRadialBinCenters)):
plt.plot(delta_plot,
rmssq_plot[:, j],
"%s--" % lMarker[i],
color=lColor[j])
plt.savefig(plotFilename)
correctedFocusError, correctedFocusErrorStd = getCorrectedFocusError(
reconstructedFocusError, reconstructedFocusErrorStd, config.corrCoeff)
return (correctedFocusError[0], correctedFocusErrorStd[0],
reconstructedFocusError[0], reconstructedFocusErrorStd, n)
def referenceImage(image, detector, linearityType, inputData, table=None):
"""Generate a reference linearization.
Parameters
----------
image: `lsst.afw.image.Image`
Image to linearize.
detector: `lsst.afw.cameraGeom.Detector`
Detector this image is from.
linearityType: `str`
Type of linearity to apply.
inputData: `numpy.array`
An array of values for the linearity correction.
table: `numpy.array`, optional
An optional lookup table to use.
Returns
-------
outImage: `lsst.afw.image.Image`
The output linearized image.
numOutOfRange: `int`
The number of values that could not be linearized.
Raises
------
RuntimeError :
Raised if an invalid linearityType is supplied.
"""
numOutOfRange = 0
for ampIdx, amp in enumerate(detector.getAmplifiers()):
ampIdx = (ampIdx // 3, ampIdx % 3)
bbox = amp.getBBox()
imageView = image.Factory(image, bbox)
if linearityType == 'Squared':
sqCoeff = inputData[ampIdx]
array = imageView.getArray()
array[:] = array + sqCoeff * array**2
elif linearityType == 'LookupTable':
rowInd, colIndOffset = inputData[ampIdx]
rowInd = int(rowInd)
tableRow = table[rowInd, :]
numOutOfRange += applyLookupTable(imageView, tableRow,
colIndOffset)
elif linearityType == 'Polynomial':
coeffs = inputData[ampIdx]
array = imageView.getArray()
summation = np.zeros_like(array)
for index, coeff in enumerate(coeffs):
summation += coeff * np.power(array, (index + 2))
array += summation
elif linearityType == 'Spline':
centers, values = np.split(inputData, 2) # This uses the full data
interp = afwMath.makeInterpolate(
centers.tolist(), values.tolist(),
afwMath.stringToInterpStyle('AKIMA_SPLINE'))
array = imageView.getArray()
delta = interp.interpolate(array.flatten())
array -= np.array(delta).reshape(array.shape)
else:
raise RuntimeError(f"Unknown linearity: {linearityType}")
return image, numOutOfRange
def overscanCorrection(ampMaskedImage,
overscanImage,
fitType='MEDIAN',
order=1,
collapseRej=3.0,
statControl=None,
overscanIsInt=True):
"""Apply overscan correction in place.
Parameters
----------
ampMaskedImage : `lsst.afw.image.MaskedImage`
Image of amplifier to correct; modified.
overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
Image of overscan; modified.
fitType : `str`
Type of fit for overscan correction. May be one of:
- ``MEAN``: use mean of overscan.
- ``MEANCLIP``: use clipped mean of overscan.
- ``MEDIAN``: use median of overscan.
- ``POLY``: fit with ordinary polynomial.
- ``CHEB``: fit with Chebyshev polynomial.
- ``LEG``: fit with Legendre polynomial.
- ``NATURAL_SPLINE``: fit with natural spline.
- ``CUBIC_SPLINE``: fit with cubic spline.
- ``AKIMA_SPLINE``: fit with Akima spline.
order : `int`
Polynomial order or number of spline knots; ignored unless
``fitType`` indicates a polynomial or spline.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object. In particular, we pay attention to numSigmaClip
overscanIsInt : `bool`
Treat the overscan region as consisting of integers, even if it's been
converted to float. E.g. handle ties properly.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``imageFit``: Value(s) removed from image (scalar or
`lsst.afw.image.Image`)
- ``overscanFit``: Value(s) removed from overscan (scalar or
`lsst.afw.image.Image`)
- ``overscanImage``: Overscan corrected overscan region
(`lsst.afw.image.Image`)
Raises
------
pexExcept.Exception
Raised if ``fitType`` is not an allowed value.
Notes
-----
The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
subtracted. Note that the ``overscanImage`` should not be a subimage of
the ``ampMaskedImage``, to avoid being subtracted twice.
Debug plots are available for the SPLINE fitTypes by setting the
`debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
plots show the scatter plot of the overscan data (collapsed along
the perpendicular dimension) as a function of position on the CCD
(normalized between +/-1).
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
numSigmaClip = statControl.getNumSigmaClip()
if fitType in ('MEAN', 'MEANCLIP'):
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImage, fitType,
statControl).getValue()
overscanFit = offImage
elif fitType in ('MEDIAN', ):
if overscanIsInt:
# we need an image with integer pixels to handle ties properly
if hasattr(overscanImage, "image"):
imageI = overscanImage.image.convertI()
overscanImageI = afwImage.MaskedImageI(imageI,
overscanImage.mask,
overscanImage.variance)
else:
overscanImageI = overscanImage.convertI()
else:
overscanImageI = overscanImage
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImageI, fitType,
statControl).getValue()
overscanFit = offImage
if overscanIsInt:
del overscanImageI
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE',
'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(
overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74 * (percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(
diff > numSigmaClip * stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(
biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0 * numpy.arange(num) / float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {
"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(
len(collapsed) * [numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices,
bins=numBins,
weights=1 -
collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
with numpy.errstate(invalid="ignore"): # suppress NAN warnings
values = numpy.histogram(
indices,
bins=numBins,
weights=collapsed.data * ~collapsedMask)[0] / numPerBin
binCenters = numpy.histogram(
indices, bins=numBins,
weights=indices * ~collapsedMask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask],
collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
plot.xlabel("centered/scaled position along overscan region")
plot.ylabel("pixel value/fit value")
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in (
"",
"c",
):
break
if ans in ("p", ):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
plot.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
overscanFit = afwImage.ImageF(overscanImage.getDimensions())
overscanArray = overscanFit.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray(
).transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' %
("overscanCorrection", fitType))
ampImage -= offImage
overscanImage -= overscanFit
return Struct(imageFit=offImage,
overscanFit=overscanFit,
overscanImage=overscanImage)
def run(self, inputPtc, camera, inputDims):
"""Fit non-linearity to PTC data, returning the correct Linearizer
object.
Parameters
----------
inputPtc : `lsst.cp.pipe.PtcDataset`
Pre-measured PTC dataset.
camera : `lsst.afw.cameraGeom.Camera`
Camera geometry.
inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
DataIds to use to populate the output calibration.
Returns
-------
results : `lsst.pipe.base.Struct`
The results struct containing:
``outputLinearizer`` : `lsst.ip.isr.Linearizer`
Final linearizer calibration.
``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
Provenance data for the new calibration.
Notes
-----
This task currently fits only polynomial-defined corrections,
where the correction coefficients are defined such that:
corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i)
These `c_i` are defined in terms of the direct polynomial fit:
meanVector ~ P(x=timeVector) = sum_j k_j x^j
such that c_(j-2) = -k_j/(k_1^j) in units of DN^(1-j) (c.f.,
Eq. 37 of 2003.05978). The `config.polynomialOrder` or
`config.splineKnots` define the maximum order of x^j to fit.
As k_0 and k_1 are degenerate with bias level and gain, they
are not included in the non-linearity correction.
"""
detector = camera[inputDims['detector']]
if self.config.linearityType == 'LookupTable':
table = np.zeros((len(detector), self.config.maxLookupTableAdu),
dtype=np.float32)
tableIndex = 0
else:
table = None
tableIndex = None # This will fail if we increment it.
if self.config.linearityType == 'Spline':
fitOrder = self.config.splineKnots
else:
fitOrder = self.config.polynomialOrder
# Initialize the linearizer.
linearizer = Linearizer(detector=detector, table=table, log=self.log)
for i, amp in enumerate(detector):
ampName = amp.getName()
if (len(inputPtc.expIdMask[ampName]) == 0):
self.log.warn(
f"Mask not found for {ampName} in non-linearity fit. Using all points."
)
mask = np.repeat(True, len(inputPtc.expIdMask[ampName]))
else:
mask = inputPtc.expIdMask[ampName]
inputAbscissa = np.array(inputPtc.rawExpTimes[ampName])[mask]
inputOrdinate = np.array(inputPtc.rawMeans[ampName])[mask]
# Determine proxy-to-linear-flux transformation
fluxMask = inputOrdinate < self.config.maxLinearAdu
lowMask = inputOrdinate > self.config.minLinearAdu
fluxMask = fluxMask & lowMask
linearAbscissa = inputAbscissa[fluxMask]
linearOrdinate = inputOrdinate[fluxMask]
linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 100.0],
linearAbscissa,
linearOrdinate,
funcPolynomial)
# Convert this proxy-to-flux fit into an expected linear flux
linearOrdinate = linearFit[0] + linearFit[1] * inputAbscissa
# Exclude low end outliers
threshold = self.config.nSigmaClipLinear * np.sqrt(linearOrdinate)
fluxMask = np.abs(inputOrdinate - linearOrdinate) < threshold
linearOrdinate = linearOrdinate[fluxMask]
fitOrdinate = inputOrdinate[fluxMask]
self.debugFit('linearFit', inputAbscissa, inputOrdinate,
linearOrdinate, fluxMask, ampName)
# Do fits
if self.config.linearityType in [
'Polynomial', 'Squared', 'LookupTable'
]:
polyFit = np.zeros(fitOrder + 1)
polyFit[1] = 1.0
polyFit, polyFitErr, chiSq, weights = irlsFit(
polyFit, linearOrdinate, fitOrdinate, funcPolynomial)
# Truncate the polynomial fit
k1 = polyFit[1]
linearityFit = [
-coeff / (k1**order) for order, coeff in enumerate(polyFit)
]
significant = np.where(
np.abs(linearityFit) > 1e-10, True, False)
self.log.info(f"Significant polynomial fits: {significant}")
modelOrdinate = funcPolynomial(polyFit, linearAbscissa)
self.debugFit('polyFit', linearAbscissa, fitOrdinate,
modelOrdinate, None, ampName)
if self.config.linearityType == 'Squared':
linearityFit = [linearityFit[2]]
elif self.config.linearityType == 'LookupTable':
# Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
tMax = (self.config.maxLookupTableAdu -
polyFit[0]) / polyFit[1]
timeRange = np.linspace(0, tMax,
self.config.maxLookupTableAdu)
signalIdeal = polyFit[0] + polyFit[1] * timeRange
signalUncorrected = funcPolynomial(polyFit, timeRange)
lookupTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has correction
linearizer.tableData[tableIndex, :] = lookupTableRow
linearityFit = [tableIndex, 0]
tableIndex += 1
elif self.config.linearityType in ['Spline']:
# See discussion in `lsst.ip.isr.linearize.py` before modifying.
numPerBin, binEdges = np.histogram(linearOrdinate,
bins=fitOrder)
with np.errstate(invalid="ignore"):
# Algorithm note: With the counts of points per
# bin above, the next histogram calculates the
# values to put in each bin by weighting each
# point by the correction value.
values = np.histogram(
linearOrdinate,
bins=fitOrder,
weights=(inputOrdinate[fluxMask] -
linearOrdinate))[0] / numPerBin
# After this is done, the binCenters are
# calculated by weighting by the value we're
# binning over. This ensures that widely
# spaced/poorly sampled data aren't assigned to
# the midpoint of the bin (as could be done using
# the binEdges above), but to the weighted mean of
# the inputs. Note that both histograms are
# scaled by the count per bin to normalize what
# the histogram returns (a sum of the points
# inside) into an average.
binCenters = np.histogram(
linearOrdinate, bins=fitOrder,
weights=linearOrdinate)[0] / numPerBin
values = values[numPerBin > 0]
binCenters = binCenters[numPerBin > 0]
self.debugFit('splineFit', binCenters, np.abs(values), values,
None, ampName)
interp = afwMath.makeInterpolate(
binCenters.tolist(), values.tolist(),
afwMath.stringToInterpStyle("AKIMA_SPLINE"))
modelOrdinate = linearOrdinate + interp.interpolate(
linearOrdinate)
self.debugFit('splineFit', linearOrdinate, fitOrdinate,
modelOrdinate, None, ampName)
# If we exclude a lot of points, we may end up with
# less than fitOrder points. Pad out the low-flux end
# to ensure equal lengths.
if len(binCenters) != fitOrder:
padN = fitOrder - len(binCenters)
binCenters = np.pad(binCenters, (padN, 0),
'linear_ramp',
end_values=(binCenters.min() - 1.0, ))
# This stores the correction, which is zero at low values.
values = np.pad(values, (padN, 0))
# Pack the spline into a single array.
linearityFit = np.concatenate(
(binCenters.tolist(), values.tolist())).tolist()
polyFit = [0.0]
polyFitErr = [0.0]
chiSq = np.nan
else:
polyFit = [0.0]
polyFitErr = [0.0]
chiSq = np.nan
linearityFit = [0.0]
linearizer.linearityType[ampName] = self.config.linearityType
linearizer.linearityCoeffs[ampName] = np.array(linearityFit)
linearizer.linearityBBox[ampName] = amp.getBBox()
linearizer.fitParams[ampName] = np.array(polyFit)
linearizer.fitParamsErr[ampName] = np.array(polyFitErr)
linearizer.fitChiSq[ampName] = chiSq
image = afwImage.ImageF(len(inputOrdinate), 1)
image.getArray()[:, :] = inputOrdinate
linearizeFunction = linearizer.getLinearityTypeByName(
linearizer.linearityType[ampName])
linearizeFunction()(image, **{
'coeffs': linearizer.linearityCoeffs[ampName],
'table': linearizer.tableData,
'log': linearizer.log
})
linearizeModel = image.getArray()[0, :]
self.debugFit('solution', inputOrdinate[fluxMask], linearOrdinate,
linearizeModel[fluxMask], None, ampName)
linearizer.hasLinearity = True
linearizer.validate()
linearizer.updateMetadata(camera=camera,
detector=detector,
filterName='NONE')
linearizer.updateMetadata(setDate=True, setCalibId=True)
provenance = IsrProvenance(calibType='linearizer')
return pipeBase.Struct(
outputLinearizer=linearizer,
outputProvenance=provenance,
)
def getDistanceFromFocus(dIcSrc, dCcd, dCcdDims, zemaxFilename, config, plotFilename=None):
# Focus error is measured by using rms^2 of stars on focus CCDs.
# If there is a focus error d, rms^2 can be written as
# rms^2 = rms_atm^2 + rms_opt_0^2 + alpha*d^2,
# where rms_atm is from atmosphere and rms_opt if from optics with out any focus error.
# On the focus CCDs which have +/-delta offset, the equation becomes
# rms_+^2 = rms_atm^2 + rms_opt_0^2 + alpha(d+delta)^2
# rms_-^2 = rms_atm^2 + rms_opt_0^2 + alpha(d-delta)^2
# Thus, the difference of these rms^2 gives the focus error as
# d = (rms_+^2 - rms_-^2)/(4 alpha delta)
# alpha is determined by ZEMAX simulations. It turned out that alpha is a function of distance from the center of FOV r.
# Also the best focus varies as a function of r. Thus the focus error can be rewritten as
# d(r) = (rms_+(r)^2 - rms_-(r)^2)/(4 alpha(r) delta) + d0(r)
# I take a pair of CCDs on the corner, divide the focus CCDs into radian bins, calculate focus error d for each radial bin with alpha and d0 values at this radius, and then take median of these focus errors for all the radian bins and CCD pairs.
# rms^2 is measured by shape.simple. Although I intend to include minimum measurement bias, there exists still some bias. This is corrected by getCorrectedFocusError() at the end, which is a polynomial function derived by calibration data (well-behaved focus sweeps).
# set up radial bins
lRadialBinEdges = config.radialBinEdges
lRadialBinCenters = config.radialBinCenters
lRadialBinsLowerEdges = lRadialBinEdges[0:-1]
lRadialBinsUpperEdges = lRadialBinEdges[1:]
# make selection on data and get rms^2 for each bin, CCD by CCD
dlRmssq = dict() # rmssq list for radial bin, which is dictionary for each ccd
for ccdId in dIcSrc.keys():
# use only objects classified as PSF candidate
icSrc = dIcSrc[ccdId][dIcSrc[ccdId].get("hscPipeline_focus_candidate")]
# prepare for getting distance from center for each object
ccd = dCcd[ccdId]
x1, y1 = dCcdDims[ccdId]
# Get focal plane position in pixels
# Note that we constructed the zemax values alpha(r), d0(r), and this r is in pixel.
transform = ccd.getTransformMap().get(ccd.makeCameraSys(afwCameraGeom.FOCAL_PLANE))
uLlc, vLlc = transform.forwardTransform(afwGeom.PointD(0., 0.))
uLrc, vLrc = transform.forwardTransform(afwGeom.PointD(x1, 0.))
uUlc, vUlc = transform.forwardTransform(afwGeom.PointD(0., y1))
uUrc, vUrc = transform.forwardTransform(afwGeom.PointD(x1, y1))
lDistanceFromCenter = list()
lRmssq = list()
for s in icSrc:
# reject blended objects
if len(s.getFootprint().getPeaks()) != 1:
continue
# calculate distance from center for each objects
x = s.getX()
y = s.getY()
uL = (uLrc-uLlc)/x1*x+uLlc
uU = (uUrc-uUlc)/x1*x+uUlc
u = (uU-uL)/y1*y+uL
vL = (vLrc-vLlc)/x1*x+vLlc
vU = (vUrc-vUlc)/x1*x+vUlc
v = (vU-vL)/y1*y+vL
lDistanceFromCenter.append(np.sqrt(u**2 + v**2))
# calculate rms^2
ixx = s.get(config.shape + "_xx")
iyy = s.get(config.shape + "_yy")
lRmssq.append((ixx + iyy)*config.pixelScale**2) # convert from pixel^2 to mm^2
# calculate median rms^2 for each radial bin
lDistanceFromCenter = np.array(lDistanceFromCenter)
lRmssq = np.array(lRmssq)
lRmssqMedian = list()
for radialBinLowerEdge, radialBinUpperEdge in zip(lRadialBinsLowerEdges, lRadialBinsUpperEdges):
sel = np.logical_and(lDistanceFromCenter > radialBinLowerEdge, lDistanceFromCenter < radialBinUpperEdge)
lRmssqMedian.append(np.median(lRmssq[sel]))
dlRmssq[ccdId] = np.ma.masked_array(lRmssqMedian, mask = np.isnan(lRmssqMedian))
# get ZEMAX values
d = np.loadtxt(zemaxFilename)
interpStyle = afwMath.stringToInterpStyle("NATURAL_SPLINE")
sAlpha = afwMath.makeInterpolate(d[:,0], d[:,1], interpStyle).interpolate
sD0 = afwMath.makeInterpolate(d[:,0], d[:,2], interpStyle).interpolate
# calculate rms^2 for each CCD pair
lCcdPairs = zip(config.belowList, config.aboveList)
llFocurErrors = list()
for ccdPair in lCcdPairs:
lFocusErrors = list()
if (ccdPair[0] not in dlRmssq or ccdPair[1] not in dlRmssq or
dlRmssq[ccdPair[0]] is None or dlRmssq[ccdPair[1]] is None):
continue
for i, radialBinCenter in enumerate(lRadialBinCenters):
rmssqAbove = dlRmssq[ccdPair[1]][i]
rmssqBelow = dlRmssq[ccdPair[0]][i]
rmssqDiff = rmssqAbove - rmssqBelow
delta = getFocusCcdOffset(ccdPair[1], config)
alpha = sAlpha(radialBinCenter)
focusError = rmssqDiff/4./alpha/delta + sD0(radialBinCenter)
lFocusErrors.append(focusError)
llFocurErrors.append(np.array(lFocusErrors))
llFocurErrors = np.ma.masked_array(llFocurErrors, mask = np.isnan(llFocurErrors))
reconstructedFocusError = np.ma.median(llFocurErrors)
n = np.sum(np.invert(llFocurErrors.mask))
reconstructedFocusErrorStd= np.ma.std(llFocurErrors)*np.sqrt(np.pi/2.)/np.sqrt(n)
if config.doPlot == True:
if not plotFilename:
raise ValueError("no filename for focus plot")
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
lMarker = ["o", "x", "d", "^", "<", ">"]
lColor = ["blue", "green", "red", "cyan", "magenta", "yellow"]
for i, ccdPair in enumerate(lCcdPairs):
delta_plot = np.ma.masked_array([getFocusCcdOffset(ccdPair[0], config),
getFocusCcdOffset(ccdPair[1], config)])
rmssq_plot = np.ma.masked_array([dlRmssq[ccdPair[0]], dlRmssq[ccdPair[1]]])
for j in range(len(lRadialBinCenters)):
plt.plot(delta_plot, rmssq_plot[:, j], "%s--" % lMarker[i], color = lColor[j])
plt.savefig(plotFilename)
correctedFocusError, correctedFocusErrorStd = getCorrectedFocusError(
reconstructedFocusError, reconstructedFocusErrorStd, config.corrCoeff)
return (correctedFocusError[0], correctedFocusErrorStd[0],
reconstructedFocusError[0], reconstructedFocusErrorStd, n)
def overscanCorrection(ampMaskedImage,
overscanImage,
fitType='MEDIAN',
order=1,
collapseRej=3.0,
statControl=None):
"""Apply overscan correction in place
@param[in,out] ampMaskedImage masked image to correct
@param[in] overscanImage overscan data as an afw.image.Image or afw.image.MaskedImage.
If a masked image is passed in the mask plane will be used
to constrain the fit of the bias level.
@param[in] fitType type of fit for overscan correction; one of:
- 'MEAN'
- 'MEDIAN'
- 'POLY' (ordinary polynomial)
- 'CHEB' (Chebyshev polynomial)
- 'LEG' (Legendre polynomial)
- 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE' (splines)
@param[in] order polynomial order or spline knots (ignored unless fitType
indicates a polynomial or spline)
@param[in] collapseRej Rejection threshold (sigma) for collapsing dimension of overscan
@param[in] statControl Statistics control object
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
if fitType == 'MEAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEAN,
statControl).getValue(afwMath.MEAN)
elif fitType == 'MEDIAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEDIAN,
statControl).getValue(afwMath.MEDIAN)
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE',
'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(
overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74 * (percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(
diff > collapseRej * stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(
biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0 * numpy.arange(num) / float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {
"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(
len(collapsed) * [numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices,
bins=numBins,
weights=1 -
collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
values = numpy.histogram(
indices, bins=numBins,
weights=collapsed.data * ~collapsedMask)[0] / numPerBin
binCenters = numpy.histogram(
indices, bins=numBins,
weights=indices * ~collapsedMask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask],
collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in (
"",
"c",
):
break
if ans in ("p", ):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
figure.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray(
).transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' % \
("overscanCorrection", fitType))
ampImage -= offImage
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
statControl=None):
"""Apply overscan correction in place
@param[in,out] ampMaskedImage masked image to correct
@param[in] overscanImage overscan data as an afw.image.Image or afw.image.MaskedImage.
If a masked image is passed in the mask plane will be used
to constrain the fit of the bias level.
@param[in] fitType type of fit for overscan correction; one of:
- 'MEAN'
- 'MEDIAN'
- 'POLY' (ordinary polynomial)
- 'CHEB' (Chebyshev polynomial)
- 'LEG' (Legendre polynomial)
- 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE' (splines)
@param[in] order polynomial order or spline knots (ignored unless fitType
indicates a polynomial or spline)
@param[in] collapseRej Rejection threshold (sigma) for collapsing dimension of overscan
@param[in] statControl Statistics control object
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
if fitType == 'MEAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEAN, statControl).getValue(afwMath.MEAN)
elif fitType == 'MEDIAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEDIAN, statControl).getValue(afwMath.MEDIAN)
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:,numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(diff > collapseRej*stdevBiasArr[:,numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0*numpy.arange(num)/float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
weights=1-collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
values = numpy.histogram(indices, bins=numBins, weights=collapsed)[0]/numPerBin
binCenters = numpy.histogram(indices, bins=numBins, weights=indices)[0]/numPerBin
interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices, collapsed, 'k+')
axes.plot(indices, fitBiasArr, 'r-')
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = raw_input(prompt).lower()
if ans in ("", "c",):
break
if ans in ("p",):
import pdb; pdb.set_trace()
elif ans in ("h", ):
print "h[elp] c[ontinue] p[db]"
figure.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
if shortInd == 1:
offArray[:,:] = fitBiasArr[:,numpy.newaxis]
else:
offArray[:,:] = fitBiasArr[numpy.newaxis,:]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in xrange(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low,:] |= suspect
for high in xrange(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:,:] |= suspect
else:
raise pexExcept.Exception, '%s : %s an invalid overscan type' % \
("overscanCorrection", fitType)
ampImage -= offImage
