def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0):
"""Apply flat correction in place
@param[in,out] maskedImage afw.image.MaskedImage to correct
@param[in] flatMaskedImage flat field afw.image.MaskedImage
@param[in] scalingType how to compute flat scale; one of 'MEAN', 'MEDIAN' or 'USER'
@param[in] userScale scale to use if scalingType is 'USER', else ignored
"""
if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(
afwImage.LOCAL):
raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
(maskedImage.getBBox(afwImage.LOCAL),
flatMaskedImage.getBBox(afwImage.LOCAL)))
# Figure out scale from the data
# Ideally the flats are normalized by the calibration product pipelin, but this allows some flexibility
# in the case that the flat is created by some other mechanism.
if scalingType == 'MEAN':
flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(),
afwMath.MEAN).getValue(afwMath.MEAN)
elif scalingType == 'MEDIAN':
flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(),
afwMath.MEDIAN).getValue(
afwMath.MEDIAN)
elif scalingType == 'USER':
flatScale = userScale
else:
raise pexExcept.Exception('%s : %s not implemented' %
("flatCorrection", scalingType))
maskedImage.scaledDivides(1.0 / flatScale, flatMaskedImage)
def _inputUsingButler(stage, policy, clipboard, log):
inputPolicy = policy.getPolicy('parameters.inputItems')
itemNames = inputPolicy.policyNames(True)
for item in itemNames:
itemPolicy = inputPolicy.getPolicy(item)
datasetType = itemPolicy.getString('datasetType')
datasetIdPolicy = itemPolicy.getPolicy('datasetId')
required = True
if itemPolicy.exists('required'):
required = itemPolicy.getBool('required')
timeout = 60.0
if itemPolicy.exists('timeout'):
timeout = itemPolicy.get('timeout')
if datasetIdPolicy.exists('fromInputDatasets') and \
datasetIdPolicy.getBool('fromInputDatasets'):
inputDatasets = clipboard.get(
policy.getString('inputKeys.inputDatasets'))
itemList = []
for ds in inputDatasets:
if ds.type == datasetType:
_waitForDataset(stage.butler, datasetType, ds.ids, log,
required, timeout)
log.log(
Log.INFO, "will load %s from %s with keys %s" %
(item, datasetType, str(ds.ids)))
obj = stage.butler.get(datasetType, dataId=ds.ids)
itemList.append(obj)
if len(itemList) == 0:
raise IOError, "No input datasets of type %s for item %s" % \
(datasetType, item)
elif len(itemList) == 1:
clipboard.put(item, itemList[0])
else:
clipboard.put(item, itemList)
elif datasetIdPolicy.exists('fromJobIdentity'):
jobIdentity = clipboard.get(
policy.getString('inputKeys.jobIdentity'))
dataId = {}
for key in datasetIdPolicy.getStringArray('fromJobIdentity'):
dataId[key] = jobIdentity[key]
if datasetIdPolicy.exists('set'):
setPolicy = datasetIdPolicy.getPolicy('set')
for param in setPolicy.paramNames():
dataId[param] = setPolicy.get(param)
_waitForDataset(stage.butler, datasetType, dataId, log, required,
timeout)
log.log(
Log.INFO, "will load %s from %s with keys %s" %
(item, datasetType, str(dataId)))
obj = stage.butler.get(datasetType, dataId=dataId)
clipboard.put(item, obj)
else:
raise pexExcept.Exception(
"datasetId missing both fromInputDatasets and fromJobIdentity")
def _waitFor(func,
log,
timeoutMsg="Unavailable dataset",
timeout=60.0,
initial=0.5,
backoff=1.2):
sleep = initial
totalSleep = 0
while func() is False: # Allow for None return
log.log(Log.INFO, "waiting for dataset")
if timeout > 0:
time.sleep(sleep)
totalSleep += sleep
sleep *= backoff
if totalSleep >= timeout:
if timeoutMsg is not None:
raise pexExcept.Exception(timeoutMsg)
else:
return
def overscanCorrection(ampMaskedImage,
overscanImage,
fitType='MEDIAN',
order=1,
collapseRej=3.0,
statControl=None):
"""Apply overscan correction in place
@param[in,out] ampMaskedImage masked image to correct
@param[in] overscanImage overscan data as an afw.image.Image or afw.image.MaskedImage.
If a masked image is passed in the mask plane will be used
to constrain the fit of the bias level.
@param[in] fitType type of fit for overscan correction; one of:
- 'MEAN'
- 'MEDIAN'
- 'POLY' (ordinary polynomial)
- 'CHEB' (Chebyshev polynomial)
- 'LEG' (Legendre polynomial)
- 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE' (splines)
@param[in] order polynomial order or spline knots (ignored unless fitType
indicates a polynomial or spline)
@param[in] collapseRej Rejection threshold (sigma) for collapsing dimension of overscan
@param[in] statControl Statistics control object
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
if fitType == 'MEAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEAN,
statControl).getValue(afwMath.MEAN)
elif fitType == 'MEDIAN':
offImage = afwMath.makeStatistics(overscanImage, afwMath.MEDIAN,
statControl).getValue(afwMath.MEDIAN)
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE',
'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(
overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74 * (percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(
diff > collapseRej * stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(
biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0 * numpy.arange(num) / float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {
"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(
len(collapsed) * [numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices,
bins=numBins,
weights=1 -
collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
values = numpy.histogram(
indices, bins=numBins,
weights=collapsed.data * ~collapsedMask)[0] / numPerBin
binCenters = numpy.histogram(
indices, bins=numBins,
weights=indices * ~collapsedMask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask],
collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in (
"",
"c",
):
break
if ans in ("p", ):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
figure.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray(
).transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' % \
("overscanCorrection", fitType))
ampImage -= offImage
def overscanCorrection(ampMaskedImage,
overscanImage,
fitType='MEDIAN',
order=1,
collapseRej=3.0,
statControl=None,
overscanIsInt=True):
"""Apply overscan correction in place.
Parameters
----------
ampMaskedImage : `lsst.afw.image.MaskedImage`
Image of amplifier to correct; modified.
overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
Image of overscan; modified.
fitType : `str`
Type of fit for overscan correction. May be one of:
- ``MEAN``: use mean of overscan.
- ``MEANCLIP``: use clipped mean of overscan.
- ``MEDIAN``: use median of overscan.
- ``POLY``: fit with ordinary polynomial.
- ``CHEB``: fit with Chebyshev polynomial.
- ``LEG``: fit with Legendre polynomial.
- ``NATURAL_SPLINE``: fit with natural spline.
- ``CUBIC_SPLINE``: fit with cubic spline.
- ``AKIMA_SPLINE``: fit with Akima spline.
order : `int`
Polynomial order or number of spline knots; ignored unless
``fitType`` indicates a polynomial or spline.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object. In particular, we pay attention to numSigmaClip
overscanIsInt : `bool`
Treat the overscan region as consisting of integers, even if it's been
converted to float. E.g. handle ties properly.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``imageFit``: Value(s) removed from image (scalar or
`lsst.afw.image.Image`)
- ``overscanFit``: Value(s) removed from overscan (scalar or
`lsst.afw.image.Image`)
- ``overscanImage``: Overscan corrected overscan region
(`lsst.afw.image.Image`)
Raises
------
pexExcept.Exception
Raised if ``fitType`` is not an allowed value.
Notes
-----
The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
subtracted. Note that the ``overscanImage`` should not be a subimage of
the ``ampMaskedImage``, to avoid being subtracted twice.
Debug plots are available for the SPLINE fitTypes by setting the
`debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
plots show the scatter plot of the overscan data (collapsed along
the perpendicular dimension) as a function of position on the CCD
(normalized between +/-1).
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
numSigmaClip = statControl.getNumSigmaClip()
if fitType in ('MEAN', 'MEANCLIP'):
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImage, fitType,
statControl).getValue()
overscanFit = offImage
elif fitType in ('MEDIAN', ):
if overscanIsInt:
# we need an image with integer pixels to handle ties properly
if hasattr(overscanImage, "image"):
imageI = overscanImage.image.convertI()
overscanImageI = afwImage.MaskedImageI(imageI,
overscanImage.mask,
overscanImage.variance)
else:
overscanImageI = overscanImage.convertI()
else:
overscanImageI = overscanImage
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImageI, fitType,
statControl).getValue()
overscanFit = offImage
if overscanIsInt:
del overscanImageI
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE',
'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(
overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74 * (percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(
diff > numSigmaClip * stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(
biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0 * numpy.arange(num) / float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {
"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(
len(collapsed) * [numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices,
bins=numBins,
weights=1 -
collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
with numpy.errstate(invalid="ignore"): # suppress NAN warnings
values = numpy.histogram(
indices,
bins=numBins,
weights=collapsed.data * ~collapsedMask)[0] / numPerBin
binCenters = numpy.histogram(
indices, bins=numBins,
weights=indices * ~collapsedMask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask],
collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
plot.xlabel("centered/scaled position along overscan region")
plot.ylabel("pixel value/fit value")
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in (
"",
"c",
):
break
if ans in ("p", ):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
plot.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
overscanFit = afwImage.ImageF(overscanImage.getDimensions())
overscanArray = overscanFit.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray(
).transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' %
("overscanCorrection", fitType))
ampImage -= offImage
overscanImage -= overscanFit
return Struct(imageFit=offImage,
overscanFit=overscanFit,
overscanImage=overscanImage)