def measureConstantOverscan(self, image):
"""Measure a constant overscan value.
Parameters
----------
image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
Image data to measure the overscan from.
Returns
-------
results : `lsst.pipe.base.Struct`
Overscan result with entries:
- ``overscanValue``: Overscan value to subtract (`float`)
- ``maskArray``: Placeholder for a mask array (`list`)
- ``isTransposed``: Orientation of the overscan (`bool`)
"""
if self.config.fitType == 'MEDIAN':
calcImage = self.integerConvert(image)
else:
calcImage = image
fitType = afwMath.stringToStatisticsProperty(self.config.fitType)
overscanValue = afwMath.makeStatistics(calcImage, fitType,
self.statControl).getValue()
return pipeBase.Struct(overscanValue=overscanValue,
maskArray=None,
isTransposed=False)
def collapseArrayMedian(self, maskedArray):
"""Collapse overscan array (and mask) to a 1-D vector of using the
correct integer median of row-values.
Parameters
----------
maskedArray : `numpy.ma.masked_array`
Masked array of input overscan data.
Returns
-------
collapsed : `numpy.ma.masked_array`
Single dimensional overscan data, combined with the afwMath median.
"""
integerMI = self.integerConvert(maskedArray)
collapsed = []
fitType = afwMath.stringToStatisticsProperty('MEDIAN')
for row in integerMI:
newRow = row.compressed()
if len(newRow) > 0:
rowMedian = afwMath.makeStatistics(
newRow, fitType, self.statControl).getValue()
else:
rowMedian = np.nan
collapsed.append(rowMedian)
return np.array(collapsed)
def _configHelper(keywordDict):
"""Helper to convert keyword dictionary to stat value.
Convert the string names in the keywordDict to the afwMath values.
The statisticToRun is then the bitwise-or of that set.
Parameters
----------
keywordDict : `dict` [`str`, `str`]
A dictionary of keys to use in the output results, with
values the string name associated with the
`lsst.afw.math.statistics.Property` to measure.
Returns
-------
statisticToRun : `int`
The merged `lsst.afw.math` statistics property.
statAccessor : `dict` [`str`, `int`]
Dictionary containing statistics property indexed by name.
"""
statisticToRun = 0
statAccessor = {}
for k, v in keywordDict.items():
statValue = afwMath.stringToStatisticsProperty(v)
statisticToRun |= statValue
statAccessor[k] = statValue
return statisticToRun, statAccessor
def run(self, inputExposure, refObjLoader=None, dataId=None, skyCorr=None):
"""Identify bright stars within an exposure using a reference catalog,
extract stamps around each, then preprocess them. The preprocessing
steps are: shifting, warping and potentially rotating them to the same
pixel grid; computing their annular flux and normalizing them.
Parameters
----------
inputExposure : `afwImage.exposure.exposure.ExposureF`
The image from which bright star stamps should be extracted.
refObjLoader : `LoadIndexedReferenceObjectsTask`, optional
Loader to find objects within a reference catalog.
dataId : `dict` or `lsst.daf.butler.DataCoordinate`
The dataId of the exposure (and detector) bright stars should be
extracted from.
skyCorr : `lsst.afw.math.backgroundList.BackgroundList` or ``None``,
optional
Full focal plane sky correction, obtained by running
`lsst.pipe.drivers.skyCorrection.SkyCorrectionTask`.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with component:
- ``brightStarStamps``: ``bSS.BrightStarStamps``
"""
if self.config.doApplySkyCorr:
self.log.info("Applying sky correction to exposure %s (exposure will be modified in-place).",
dataId)
self.applySkyCorr(inputExposure, skyCorr)
self.log.info("Extracting bright stars from exposure %s", dataId)
# Extract stamps around bright stars
extractedStamps = self.extractStamps(inputExposure, refObjLoader=refObjLoader)
# Warp (and shift, and potentially rotate) them
self.log.info("Applying warp and/or shift to %i star stamps from exposure %s",
len(extractedStamps.starIms), dataId)
warpedStars = self.warpStamps(extractedStamps.starIms, extractedStamps.pixCenters)
brightStarList = [bSS.BrightStarStamp(stamp_im=warp,
gaiaGMag=extractedStamps.GMags[j],
gaiaId=extractedStamps.gaiaIds[j])
for j, warp in enumerate(warpedStars)]
# Compute annularFlux and normalize
self.log.info("Computing annular flux and normalizing %i bright stars from exposure %s",
len(warpedStars), dataId)
# annularFlux statistic set-up, excluding mask planes
statsControl = afwMath.StatisticsControl()
statsControl.setNumSigmaClip(self.config.numSigmaClip)
statsControl.setNumIter(self.config.numIter)
innerRadius, outerRadius = self.config.annularFluxRadii
statsFlag = afwMath.stringToStatisticsProperty(self.config.annularFluxStatistic)
brightStarStamps = bSS.BrightStarStamps.initAndNormalize(brightStarList,
innerRadius=innerRadius,
outerRadius=outerRadius,
imCenter=self.modelCenter,
statsControl=statsControl,
statsFlag=statsFlag,
badMaskPlanes=self.config.badMaskPlanes)
return pipeBase.Struct(brightStarStamps=brightStarStamps)
def flatCorrection(maskedImage,
flatMaskedImage,
scalingType,
userScale=1.0,
invert=False,
trimToFit=False):
"""Apply flat correction in place.
Parameters
----------
maskedImage : `lsst.afw.image.MaskedImage`
Image to process. The image is modified.
flatMaskedImage : `lsst.afw.image.MaskedImage`
Flat image of the same size as ``maskedImage``
scalingType : str
Flat scale computation method. Allowed values are 'MEAN',
'MEDIAN', or 'USER'.
userScale : scalar, optional
Scale to use if ``scalingType``='USER'.
invert : `Bool`, optional
If True, unflatten an already flattened image.
trimToFit : `Bool`, optional
If True, raw data is symmetrically trimmed to match
calibration size.
Raises
------
RuntimeError
Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
the same size or if ``scalingType`` is not an allowed value.
"""
if trimToFit:
maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)
if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(
afwImage.LOCAL):
raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
(maskedImage.getBBox(afwImage.LOCAL),
flatMaskedImage.getBBox(afwImage.LOCAL)))
# Figure out scale from the data
# Ideally the flats are normalized by the calibration product pipeline,
# but this allows some flexibility in the case that the flat is created by
# some other mechanism.
if scalingType in ('MEAN', 'MEDIAN'):
scalingType = afwMath.stringToStatisticsProperty(scalingType)
flatScale = afwMath.makeStatistics(flatMaskedImage.image,
scalingType).getValue()
elif scalingType == 'USER':
flatScale = userScale
else:
raise RuntimeError('%s : %s not implemented' %
("flatCorrection", scalingType))
if not invert:
maskedImage.scaledDivides(1.0 / flatScale, flatMaskedImage)
else:
maskedImage.scaledMultiplies(1.0 / flatScale, flatMaskedImage)
def measureScale(self, image, skyBackground):
"""Measure scale of background model in image
We treat the sky frame much as we would a fringe frame
(except the length scale of the variations is different):
we measure samples on the input image and the sky frame,
which we will use to determine the scaling factor in the
'solveScales` method.
Parameters
----------
image : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
Science image for which to measure scale.
skyBackground : `lsst.afw.math.BackgroundList`
Sky background model.
Returns
-------
imageSamples : `numpy.ndarray`
Sample measurements on image.
skySamples : `numpy.ndarray`
Sample measurements on sky frame.
"""
if isinstance(image, afwImage.Exposure):
image = image.getMaskedImage()
# Ensure more samples than pixels
xNumSamples = min(self.config.xNumSamples, image.getWidth())
yNumSamples = min(self.config.yNumSamples, image.getHeight())
xLimits = numpy.linspace(0,
image.getWidth(),
xNumSamples + 1,
dtype=int)
yLimits = numpy.linspace(0,
image.getHeight(),
yNumSamples + 1,
dtype=int)
sky = skyBackground.getImage()
maskVal = image.getMask().getPlaneBitMask(self.config.stats.mask)
ctrl = afwMath.StatisticsControl(self.config.stats.clip,
self.config.stats.nIter, maskVal)
statistic = afwMath.stringToStatisticsProperty(
self.config.stats.statistic)
imageSamples = []
skySamples = []
for xIndex, yIndex in itertools.product(range(xNumSamples),
range(yNumSamples)):
# -1 on the stop because Box2I is inclusive of the end point and we don't want to overlap boxes
xStart, xStop = xLimits[xIndex], xLimits[xIndex + 1] - 1
yStart, yStop = yLimits[yIndex], yLimits[yIndex + 1] - 1
box = geom.Box2I(geom.Point2I(xStart, yStart),
geom.Point2I(xStop, yStop))
subImage = image.Factory(image, box)
subSky = sky.Factory(sky, box)
imageSamples.append(
afwMath.makeStatistics(subImage, statistic, ctrl).getValue())
skySamples.append(
afwMath.makeStatistics(subSky, statistic, ctrl).getValue())
return imageSamples, skySamples
def test_online_coadd_input_variance_false(self):
"""Test online coaddition with calcErrorFromInputVariance=False."""
exposures, weights = self.make_test_images_to_coadd()
coadd_exposure = self.make_coadd_exposure(exposures[0])
stats_ctrl = self.make_stats_ctrl()
stats_ctrl.setCalcErrorFromInputVariance(False)
mask_map = self.make_mask_map(stats_ctrl)
stats_flags = afwMath.stringToStatisticsProperty("MEAN")
clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
masked_image_list = [exp.maskedImage for exp in exposures]
afw_masked_image = afwMath.statisticsStack(masked_image_list,
stats_flags, stats_ctrl,
weights, clipped, mask_map)
mask_threshold_dict = AccumulatorMeanStack.stats_ctrl_to_threshold_dict(
stats_ctrl)
# Make the stack with the online accumulator
# By setting no_good_pixels=None we have the same behavior
# as the default from stats_ctrl.getNoGoodPixelsMask(), but
# covers the alternate code path.
stacker = AccumulatorMeanStack(
coadd_exposure.image.array.shape,
stats_ctrl.getAndMask(),
mask_threshold_dict=mask_threshold_dict,
mask_map=mask_map,
no_good_pixels_mask=None,
calc_error_from_input_variance=stats_ctrl.
getCalcErrorFromInputVariance(),
compute_n_image=True)
for exposure, weight in zip(exposures, weights):
stacker.add_masked_image(exposure.maskedImage, weight=weight)
stacker.fill_stacked_masked_image(coadd_exposure.maskedImage)
online_masked_image = coadd_exposure.maskedImage
# The coadds match at the <1e-5 level.
testing.assert_array_almost_equal(online_masked_image.image.array,
afw_masked_image.image.array,
decimal=5)
# The computed variances match at the <1e-4 level.
testing.assert_array_almost_equal(online_masked_image.variance.array,
afw_masked_image.variance.array,
decimal=4)
testing.assert_array_equal(online_masked_image.mask.array,
afw_masked_image.mask.array)
def measureAndNormalize(
self,
annulus,
statsControl=afwMath.StatisticsControl(),
statsFlag=afwMath.stringToStatisticsProperty("MEAN"),
badMaskPlanes=('BAD', 'SAT', 'NO_DATA')):
"""Compute "annularFlux", the integrated flux within an annulus
around an object's center, and normalize it.
Since the center of bright stars are saturated and/or heavily affected
by ghosts, we measure their flux in an annulus with a large enough
inner radius to avoid the most severe ghosts and contain enough
non-saturated pixels.
Parameters
----------
annulus : `lsst.afw.geom.spanSet.SpanSet`
SpanSet containing the annulus to use for normalization.
statsControl : `lsst.afw.math.statistics.StatisticsControl`, optional
StatisticsControl to be used when computing flux over all pixels
within the annulus.
statsFlag : `lsst.afw.math.statistics.Property`, optional
statsFlag to be passed on to ``afwMath.makeStatistics`` to compute
annularFlux. Defaults to a simple MEAN.
badMaskPlanes : `collections.abc.Collection` [`str`]
Collection of mask planes to ignore when computing annularFlux.
"""
stampSize = self.stamp_im.getDimensions()
# create image with the same pixel values within annulus, NO_DATA
# elsewhere
maskPlaneDict = self.stamp_im.mask.getMaskPlaneDict()
annulusImage = MaskedImageF(stampSize, planeDict=maskPlaneDict)
annulusMask = annulusImage.mask
annulusMask.array[:] = 2**maskPlaneDict['NO_DATA']
annulus.copyMaskedImage(self.stamp_im, annulusImage)
# set mask planes to be ignored
andMask = reduce(ior, (annulusMask.getPlaneBitMask(bm)
for bm in badMaskPlanes))
statsControl.setAndMask(andMask)
# compute annularFlux
annulusStat = afwMath.makeStatistics(annulusImage, statsFlag,
statsControl)
self.annularFlux = annulusStat.getValue()
if np.isnan(self.annularFlux):
raise RuntimeError(
"Annular flux computation failed, likely because no pixels were valid."
)
# normalize stamps
self.stamp_im.image.array /= self.annularFlux
return None
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False):
"""Apply flat correction in place.
Parameters
----------
maskedImage : `lsst.afw.image.MaskedImage`
Image to process. The image is modified.
flatMaskedImage : `lsst.afw.image.MaskedImage`
Flat image of the same size as ``maskedImage``
scalingType : str
Flat scale computation method. Allowed values are 'MEAN',
'MEDIAN', or 'USER'.
userScale : scalar, optional
Scale to use if ``scalingType``='USER'.
invert : `Bool`, optional
If True, unflatten an already flattened image.
trimToFit : `Bool`, optional
If True, raw data is symmetrically trimmed to match
calibration size.
Raises
------
RuntimeError
Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
the same size.
pexExcept.Exception
Raised if ``scalingType`` is not an allowed value.
"""
if trimToFit:
maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)
if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
(maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
# Figure out scale from the data
# Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility
# in the case that the flat is created by some other mechanism.
if scalingType in ('MEAN', 'MEDIAN'):
scalingType = afwMath.stringToStatisticsProperty(scalingType)
flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue()
elif scalingType == 'USER':
flatScale = userScale
else:
raise pexExcept.Exception('%s : %s not implemented' % ("flatCorrection", scalingType))
if not invert:
maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
else:
maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
def test_online_coadd_image(self):
"""Test online coaddition with regular non-masked images."""
exposures, weights = self.make_test_images_to_coadd()
coadd_exposure = self.make_coadd_exposure(exposures[0])
stats_ctrl = self.make_stats_ctrl()
stats_ctrl.setAndMask(0)
stats_ctrl.setCalcErrorFromInputVariance(True)
mask_map = self.make_mask_map(stats_ctrl)
stats_flags = afwMath.stringToStatisticsProperty("MEAN")
clipped = afwImage.Mask.getPlaneBitMask("CLIPPED")
masked_image_list = [exp.maskedImage for exp in exposures]
afw_masked_image = afwMath.statisticsStack(masked_image_list,
stats_flags, stats_ctrl,
weights, clipped, mask_map)
# Make the stack with the online accumulator
stacker = AccumulatorMeanStack(
coadd_exposure.image.array.shape,
stats_ctrl.getAndMask(),
mask_map=mask_map,
no_good_pixels_mask=stats_ctrl.getNoGoodPixelsMask(),
calc_error_from_input_variance=stats_ctrl.
getCalcErrorFromInputVariance(),
compute_n_image=True)
for exposure, weight in zip(exposures, weights):
stacker.add_image(exposure.image, weight=weight)
stacker.fill_stacked_image(coadd_exposure.image)
online_image = coadd_exposure.image
# The unmasked coadd good pixels should match at the <1e-5 level
# The masked pixels will not be correct for straight image stacking.
good_pixels = np.where(afw_masked_image.mask.array == 0)
testing.assert_array_almost_equal(
online_image.array[good_pixels],
afw_masked_image.image.array[good_pixels],
decimal=5)
def measureScale(self, image, skyBackground):
"""Measure scale of background model in image
We treat the sky frame much as we would a fringe frame
(except the length scale of the variations is different):
we measure samples on the input image and the sky frame,
which we will use to determine the scaling factor in the
'solveScales` method.
Parameters
----------
image : `lsst.afw.image.Exposure` or `lsst.afw.image.MaskedImage`
Science image for which to measure scale.
skyBackground : `lsst.afw.math.BackgroundList`
Sky background model.
Returns
-------
imageSamples : `numpy.ndarray`
Sample measurements on image.
skySamples : `numpy.ndarray`
Sample measurements on sky frame.
"""
if isinstance(image, afwImage.Exposure):
image = image.getMaskedImage()
xLimits = numpy.linspace(0, image.getWidth() - 1, self.config.xNumSamples + 1, dtype=int)
yLimits = numpy.linspace(0, image.getHeight() - 1, self.config.yNumSamples + 1, dtype=int)
sky = skyBackground.getImage()
maskVal = image.getMask().getPlaneBitMask(self.config.stats.mask)
ctrl = afwMath.StatisticsControl(self.config.stats.clip, self.config.stats.nIter, maskVal)
statistic = afwMath.stringToStatisticsProperty(self.config.stats.statistic)
imageSamples = []
skySamples = []
for xStart, yStart, xStop, yStop in zip(xLimits[:-1], yLimits[:-1], xLimits[1:], yLimits[1:]):
box = afwGeom.Box2I(afwGeom.Point2I(xStart, yStart), afwGeom.Point2I(xStop, yStop))
subImage = image.Factory(image, box)
subSky = sky.Factory(sky, box)
imageSamples.append(afwMath.makeStatistics(subImage, statistic, ctrl).getValue())
skySamples.append(afwMath.makeStatistics(subSky, statistic, ctrl).getValue())
return imageSamples, skySamples
class StackBrightStarsTask(pipeBase.CmdLineTask):
"""Stack bright stars together to build an extended PSF model.
"""
ConfigClass = StackBrightStarsConfig
_DefaultName = "stack_bright_stars"
def __init__(self, initInputs=None, *args, **kwargs):
pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
def _set_up_stacking(self, example_stamp):
"""Configure stacking statistic and control from config fields.
"""
stats_control = afwMath.StatisticsControl()
stats_control.setNumSigmaClip(self.config.num_sigma_clip)
stats_control.setNumIter(self.config.num_iter)
if bad_masks := self.config.bad_mask_planes:
and_mask = example_stamp.mask.getPlaneBitMask(bad_masks[0])
for bm in bad_masks[1:]:
and_mask = and_mask | example_stamp.mask.getPlaneBitMask(bm)
stats_control.setAndMask(and_mask)
stats_flags = afwMath.stringToStatisticsProperty(
self.config.stacking_statistic)
return stats_control, stats_flags
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
statControl=None, overscanIsInt=True):
"""Apply overscan correction in place.
Parameters
----------
ampMaskedImage : `lsst.afw.image.MaskedImage`
Image of amplifier to correct; modified.
overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
Image of overscan; modified.
fitType : `str`
Type of fit for overscan correction. May be one of:
- ``MEAN``: use mean of overscan.
- ``MEANCLIP``: use clipped mean of overscan.
- ``MEDIAN``: use median of overscan.
- ``POLY``: fit with ordinary polynomial.
- ``CHEB``: fit with Chebyshev polynomial.
- ``LEG``: fit with Legendre polynomial.
- ``NATURAL_SPLINE``: fit with natural spline.
- ``CUBIC_SPLINE``: fit with cubic spline.
- ``AKIMA_SPLINE``: fit with Akima spline.
order : `int`
Polynomial order or number of spline knots; ignored unless
``fitType`` indicates a polynomial or spline.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object. In particular, we pay attention to numSigmaClip
overscanIsInt : `bool`
Treat the overscan region as consisting of integers, even if it's been
converted to float. E.g. handle ties properly.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``imageFit``: Value(s) removed from image (scalar or
`lsst.afw.image.Image`)
- ``overscanFit``: Value(s) removed from overscan (scalar or
`lsst.afw.image.Image`)
- ``overscanImage``: Overscan corrected overscan region
(`lsst.afw.image.Image`)
Raises
------
pexExcept.Exception
Raised if ``fitType`` is not an allowed value.
Notes
-----
The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
subtracted. Note that the ``overscanImage`` should not be a subimage of
the ``ampMaskedImage``, to avoid being subtracted twice.
Debug plots are available for the SPLINE fitTypes by setting the
`debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
plots show the scatter plot of the overscan data (collapsed along
the perpendicular dimension) as a function of position on the CCD
(normalized between +/-1).
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
numSigmaClip = statControl.getNumSigmaClip()
if fitType in ('MEAN', 'MEANCLIP'):
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue()
overscanFit = offImage
elif fitType in ('MEDIAN',):
if overscanIsInt:
# we need an image with integer pixels to handle ties properly
if hasattr(overscanImage, "image"):
imageI = overscanImage.image.convertI()
overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance)
else:
overscanImageI = overscanImage.convertI()
else:
overscanImageI = overscanImage
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImageI, fitType, statControl).getValue()
overscanFit = offImage
if overscanIsInt:
del overscanImageI
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0*numpy.arange(num)/float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
weights=1-collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
with numpy.errstate(invalid="ignore"): # suppress NAN warnings
values = numpy.histogram(indices, bins=numBins,
weights=collapsed.data*~collapsedMask)[0]/numPerBin
binCenters = numpy.histogram(indices, bins=numBins,
weights=indices*~collapsedMask)[0]/numPerBin
interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
plot.xlabel("centered/scaled position along overscan region")
plot.ylabel("pixel value/fit value")
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in ("", "c",):
break
if ans in ("p",):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
plot.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
overscanFit = afwImage.ImageF(overscanImage.getDimensions())
overscanArray = overscanFit.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType))
ampImage -= offImage
overscanImage -= overscanFit
return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs):
"""Warp coadds from multiple tracts to form a template for image diff.
Where the tracts overlap, the resulting template image is averaged.
The PSF on the template is created by combining the CoaddPsf on each
template image into a meta-CoaddPsf.
Parameters
----------
coaddExposures : `list` of `lsst.afw.image.Exposure`
Coadds to be mosaicked
bbox : `lsst.geom.Box2I`
Template Bounding box of the detector geometry onto which to
resample the coaddExposures
wcs : `lsst.afw.geom.SkyWcs`
Template WCS onto which to resample the coaddExposures
dataIds : `list` of `lsst.daf.butler.DataCoordinate`
Record of the tract and patch of each coaddExposure.
**kwargs
Any additional keyword parameters.
Returns
-------
result : `lsst.pipe.base.Struct` containing
- ``outputExposure`` : a template coadd exposure assembled out of patches
"""
# Table for CoaddPSF
tractsSchema = afwTable.ExposureTable.makeMinimalSchema()
tractKey = tractsSchema.addField('tract',
type=np.int32,
doc='Which tract')
patchKey = tractsSchema.addField('patch',
type=np.int32,
doc='Which patch')
weightKey = tractsSchema.addField(
'weight', type=float, doc='Weight for each tract, should be 1')
tractsCatalog = afwTable.ExposureCatalog(tractsSchema)
finalWcs = wcs
bbox.grow(self.config.templateBorderSize)
finalBBox = bbox
nPatchesFound = 0
maskedImageList = []
weightList = []
for coaddExposure, dataId in zip(coaddExposures, dataIds):
# warp to detector WCS
warped = self.warper.warpExposure(finalWcs,
coaddExposure,
maxBBox=finalBBox)
# Check if warped image is viable
if not np.any(np.isfinite(warped.image.array)):
self.log.info("No overlap for warped %s. Skipping" % dataId)
continue
exp = afwImage.ExposureF(finalBBox, finalWcs)
exp.maskedImage.set(np.nan,
afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
exp.maskedImage.assign(warped.maskedImage, warped.getBBox())
maskedImageList.append(exp.maskedImage)
weightList.append(1)
record = tractsCatalog.addNew()
record.setPsf(coaddExposure.getPsf())
record.setWcs(coaddExposure.getWcs())
record.setPhotoCalib(coaddExposure.getPhotoCalib())
record.setBBox(coaddExposure.getBBox())
record.setValidPolygon(
afwGeom.Polygon(
geom.Box2D(coaddExposure.getBBox()).getCorners()))
record.set(tractKey, dataId['tract'])
record.set(patchKey, dataId['patch'])
record.set(weightKey, 1.)
nPatchesFound += 1
if nPatchesFound == 0:
raise pipeBase.NoWorkFound("No patches found to overlap detector")
# Combine images from individual patches together
statsFlags = afwMath.stringToStatisticsProperty('MEAN')
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
templateExposure = afwImage.ExposureF(finalBBox, finalWcs)
templateExposure.maskedImage.set(
np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
xy0 = templateExposure.getXY0()
# Do not mask any values
templateExposure.maskedImage = afwMath.statisticsStack(maskedImageList,
statsFlags,
statsCtrl,
weightList,
clipped=0,
maskMap=[])
templateExposure.maskedImage.setXY0(xy0)
# CoaddPsf centroid not only must overlap image, but must overlap the part of
# image with data. Use centroid of region with data
boolmask = templateExposure.mask.array & templateExposure.mask.getPlaneBitMask(
'NO_DATA') == 0
maskx = afwImage.makeMaskFromArray(boolmask.astype(afwImage.MaskPixel))
centerCoord = afwGeom.SpanSet.fromMask(maskx, 1).computeCentroid()
ctrl = self.config.coaddPsf.makeControl()
coaddPsf = CoaddPsf(tractsCatalog, finalWcs, centerCoord,
ctrl.warpingKernelName, ctrl.cacheSize)
if coaddPsf is None:
raise RuntimeError("CoaddPsf could not be constructed")
templateExposure.setPsf(coaddPsf)
templateExposure.setFilter(coaddExposure.getFilter())
templateExposure.setPhotoCalib(coaddExposure.getPhotoCalib())
return pipeBase.Struct(outputExposure=templateExposure)
def run(self, pixels, coadd_exposure_handles):
"""Run the HighResolutionHipsTask.
Parameters
----------
pixels : `Iterable` [ `int` ]
Iterable of healpix pixels (nest ordering) to warp to.
coadd_exposure_handles : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
Handles for the coadd exposures.
Returns
-------
outputs : `lsst.pipe.base.Struct`
``hips_exposures`` is a dict with pixel (key) and hips_exposure (value)
"""
self.log.info("Generating HIPS images for %d pixels at order %d",
len(pixels), self.config.hips_order)
npix = 2**self.config.shift_order
bbox_hpx = geom.Box2I(corner=geom.Point2I(0, 0),
dimensions=geom.Extent2I(npix, npix))
# For each healpix pixel we will create an empty exposure with the
# correct HPX WCS. We furthermore create a dict to hold each of
# the warps that will go into each HPX exposure.
exp_hpx_dict = {}
warp_dict = {}
for pixel in pixels:
wcs_hpx = afwGeom.makeHpxWcs(self.config.hips_order,
pixel,
shift_order=self.config.shift_order)
exp_hpx = afwImage.ExposureF(bbox_hpx, wcs_hpx)
exp_hpx_dict[pixel] = exp_hpx
warp_dict[pixel] = []
first_handle = True
# Loop over input coadd exposures to minimize i/o (this speeds things
# up by ~8x to batch together pixels that overlap a given coadd).
for handle in coadd_exposure_handles:
coadd_exp = handle.get()
# For each pixel, warp the coadd to the HPX WCS for the pixel.
for pixel in pixels:
warped = self.warper.warpExposure(exp_hpx_dict[pixel].getWcs(),
coadd_exp,
maxBBox=bbox_hpx)
exp = afwImage.ExposureF(exp_hpx_dict[pixel].getBBox(),
exp_hpx_dict[pixel].getWcs())
exp.maskedImage.set(np.nan,
afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
if first_handle:
# Make sure the mask planes, filter, and photocalib of the output
# exposure match the (first) input exposure.
exp_hpx_dict[pixel].mask.conformMaskPlanes(
coadd_exp.mask.getMaskPlaneDict())
exp_hpx_dict[pixel].setFilter(coadd_exp.getFilter())
exp_hpx_dict[pixel].setPhotoCalib(
coadd_exp.getPhotoCalib())
if warped.getBBox().getArea() == 0 or not np.any(
np.isfinite(warped.image.array)):
# There is no overlap, skip.
self.log.debug(
"No overlap between output HPX %d and input exposure %s",
pixel, handle.dataId)
continue
exp.maskedImage.assign(warped.maskedImage, warped.getBBox())
warp_dict[pixel].append(exp.maskedImage)
first_handle = False
stats_flags = afwMath.stringToStatisticsProperty('MEAN')
stats_ctrl = afwMath.StatisticsControl()
stats_ctrl.setNanSafe(True)
stats_ctrl.setWeighted(True)
stats_ctrl.setCalcErrorFromInputVariance(True)
# Loop over pixels and combine the warps for each pixel.
# The combination is done with a simple mean for pixels that
# overlap in neighboring patches.
for pixel in pixels:
exp_hpx_dict[pixel].maskedImage.set(
np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
if not warp_dict[pixel]:
# Nothing in this pixel
self.log.debug("No data in HPX pixel %d", pixel)
# Remove the pixel from the output, no need to persist an
# empty exposure.
exp_hpx_dict.pop(pixel)
continue
exp_hpx_dict[pixel].maskedImage = afwMath.statisticsStack(
warp_dict[pixel],
stats_flags,
stats_ctrl, [1.0] * len(warp_dict[pixel]),
clipped=0,
maskMap=[])
return pipeBase.Struct(hips_exposures=exp_hpx_dict)
def initAndNormalize(cls,
starStamps,
innerRadius,
outerRadius,
nb90Rots=None,
metadata=None,
use_mask=True,
use_variance=False,
use_archive=False,
imCenter=None,
discardNanFluxObjects=True,
statsControl=afwMath.StatisticsControl(),
statsFlag=afwMath.stringToStatisticsProperty("MEAN"),
badMaskPlanes=('BAD', 'SAT', 'NO_DATA')):
"""Normalize a set of bright star stamps and initialize a
BrightStarStamps instance.
Since the center of bright stars are saturated and/or heavily affected
by ghosts, we measure their flux in an annulus with a large enough
inner radius to avoid the most severe ghosts and contain enough
non-saturated pixels.
Parameters
----------
starStamps : `collections.abc.Sequence` [`BrightStarStamp`]
Sequence of star stamps. Cannot contain both normalized and
unnormalized stamps.
innerRadius : `int`
Inner radius value, in pixels. This and ``outerRadius`` define the
annulus used to compute the ``"annularFlux"`` values within each
``starStamp``.
outerRadius : `int`
Outer radius value, in pixels. This and ``innerRadius`` define the
annulus used to compute the ``"annularFlux"`` values within each
``starStamp``.
nb90Rots : `int`, optional
Number of 90 degree rotations required to compensate for detector
orientation.
metadata : `lsst.daf.base.PropertyList`, optional
Metadata associated with the bright stars.
use_mask : `bool`
If `True` read and write mask data. Default `True`.
use_variance : `bool`
If ``True`` read and write variance data. Default ``False``.
use_archive : `bool`
If ``True`` read and write an Archive that contains a Persistable
associated with each stamp. In the case of bright stars, this is
usually a ``TransformPoint2ToPoint2``, used to warp each stamp
to the same pixel grid before stacking.
imCenter : `collections.abc.Sequence`, optional
Center of the object, in pixels. If not provided, the center of the
first stamp's pixel grid will be used.
discardNanFluxObjects : `bool`
Whether objects with NaN annular flux should be discarded.
If False, these objects will not be normalized.
statsControl : `lsst.afw.math.statistics.StatisticsControl`, optional
StatisticsControl to be used when computing flux over all pixels
within the annulus.
statsFlag : `lsst.afw.math.statistics.Property`, optional
statsFlag to be passed on to ``afwMath.makeStatistics`` to compute
annularFlux. Defaults to a simple MEAN.
badMaskPlanes : `collections.abc.Collection` [`str`]
Collection of mask planes to ignore when computing annularFlux.
Raises
------
ValueError
Raised if one of the star stamps provided does not contain the
required keys.
AttributeError
Raised if there is a mix-and-match of normalized and unnormalized
stamps, stamps normalized with different annulus definitions, or if
stamps are to be normalized but annular radii were not provided.
"""
if imCenter is None:
stampSize = starStamps[0].stamp_im.getDimensions()
imCenter = stampSize[0] // 2, stampSize[1] // 2
# Create SpanSet of annulus
outerCircle = afwGeom.SpanSet.fromShape(outerRadius,
afwGeom.Stencil.CIRCLE,
offset=imCenter)
innerCircle = afwGeom.SpanSet.fromShape(innerRadius,
afwGeom.Stencil.CIRCLE,
offset=imCenter)
annulus = outerCircle.intersectNot(innerCircle)
# Initialize (unnormalized) brightStarStamps instance
bss = cls(starStamps,
innerRadius=None,
outerRadius=None,
nb90Rots=nb90Rots,
metadata=metadata,
use_mask=use_mask,
use_variance=use_variance,
use_archive=use_archive)
# Ensure no stamps had already been normalized
bss._checkNormalization(True, innerRadius, outerRadius)
bss._innerRadius, bss._outerRadius = innerRadius, outerRadius
# Apply normalization
for j, stamp in enumerate(bss._stamps):
try:
stamp.measureAndNormalize(annulus,
statsControl=statsControl,
statsFlag=statsFlag,
badMaskPlanes=badMaskPlanes)
except RuntimeError:
# Optionally keep NaN flux objects, for bookkeeping purposes,
# and to avoid having to re-find and redo the preprocessing
# steps needed before bright stars can be subtracted.
if discardNanFluxObjects:
bss._stamps.pop(j)
else:
stamp.annularFlux = np.nan
bss.normalized = True
return bss
def run(self, exposure, sensorRef, templateIdList=None):
"""Retrieve and mosaic a template coadd that overlaps the exposure where
the template spans multiple tracts.
The resulting template image will be an average of all the input templates from
the separate tracts.
The PSF on the template is created by combining the CoaddPsf on each template image
into a meta-CoaddPsf.
Parameters
----------
exposure: `lsst.afw.image.Exposure`
an exposure for which to generate an overlapping template
sensorRef : TYPE
a Butler data reference that can be used to obtain coadd data
templateIdList : TYPE, optional
list of data ids (unused)
Returns
-------
result : `struct`
return a pipeBase.Struct:
- ``exposure`` : a template coadd exposure assembled out of patches
- ``sources`` : None for this subtask
"""
# Table for CoaddPSF
tractsSchema = afwTable.ExposureTable.makeMinimalSchema()
tractKey = tractsSchema.addField('tract',
type=np.int32,
doc='Which tract')
patchKey = tractsSchema.addField('patch',
type=np.int32,
doc='Which patch')
weightKey = tractsSchema.addField(
'weight', type=float, doc='Weight for each tract, should be 1')
tractsCatalog = afwTable.ExposureCatalog(tractsSchema)
skyMap = sensorRef.get(datasetType=self.config.coaddName +
"Coadd_skyMap")
expWcs = exposure.getWcs()
expBoxD = geom.Box2D(exposure.getBBox())
expBoxD.grow(self.config.templateBorderSize)
ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter())
centralTractInfo = skyMap.findTract(ctrSkyPos)
if not centralTractInfo:
raise RuntimeError("No suitable tract found for central point")
self.log.info("Central skyMap tract %s" % (centralTractInfo.getId(), ))
skyCorners = [
expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()
]
tractPatchList = skyMap.findTractPatchList(skyCorners)
if not tractPatchList:
raise RuntimeError("No suitable tract found")
self.log.info("All overlapping skyMap tracts %s" %
([a[0].getId() for a in tractPatchList]))
# Move central tract to front of the list and use as the reference
tracts = [tract[0].getId() for tract in tractPatchList]
centralIndex = tracts.index(centralTractInfo.getId())
tracts.insert(0, tracts.pop(centralIndex))
tractPatchList.insert(0, tractPatchList.pop(centralIndex))
coaddPsf = None
coaddFilter = None
nPatchesFound = 0
maskedImageList = []
weightList = []
for itract, tract in enumerate(tracts):
tractInfo = tractPatchList[itract][0]
coaddWcs = tractInfo.getWcs()
coaddBBox = geom.Box2D()
for skyPos in skyCorners:
coaddBBox.include(coaddWcs.skyToPixel(skyPos))
coaddBBox = geom.Box2I(coaddBBox)
if itract == 0:
# Define final wcs and bounding box from the reference tract
finalWcs = coaddWcs
finalBBox = coaddBBox
patchList = tractPatchList[itract][1]
for patchInfo in patchList:
self.log.info('Adding patch %s from tract %s' %
(patchInfo.getIndex(), tract))
# Local patch information
patchSubBBox = geom.Box2I(patchInfo.getInnerBBox())
patchSubBBox.clip(coaddBBox)
patchInt = int(
f"{patchInfo.getIndex()[0]}{patchInfo.getIndex()[1]}")
innerBBox = geom.Box2I(tractInfo._minimumBoundingBox(finalWcs))
if itract == 0:
# clip to image and tract boundaries
patchSubBBox.clip(finalBBox)
patchSubBBox.clip(innerBBox)
if patchSubBBox.getArea() == 0:
self.log.debug("No ovlerap for patch %s" % patchInfo)
continue
patchArgDict = dict(
datasetType="deepCoadd_sub",
bbox=patchSubBBox,
tract=tractInfo.getId(),
patch="%s,%s" %
(patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
filter=exposure.getFilter().getName())
coaddPatch = sensorRef.get(**patchArgDict)
if coaddFilter is None:
coaddFilter = coaddPatch.getFilter()
# create full image from final bounding box
exp = afwImage.ExposureF(finalBBox, finalWcs)
exp.maskedImage.set(
np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
exp.maskedImage.assign(coaddPatch.maskedImage,
patchSubBBox)
maskedImageList.append(exp.maskedImage)
weightList.append(1)
record = tractsCatalog.addNew()
record.setPsf(coaddPatch.getPsf())
record.setWcs(coaddPatch.getWcs())
record.setPhotoCalib(coaddPatch.getPhotoCalib())
record.setBBox(patchSubBBox)
record.set(tractKey, tract)
record.set(patchKey, patchInt)
record.set(weightKey, 1.)
nPatchesFound += 1
else:
# compute the exposure bounding box in a tract that is not the reference tract
localBox = geom.Box2I()
for skyPos in skyCorners:
localBox.include(
geom.Point2I(
tractInfo.getWcs().skyToPixel(skyPos)))
# clip to patch bounding box
localBox.clip(patchSubBBox)
# grow border to deal with warping at edges
localBox.grow(self.config.templateBorderSize)
# clip to tract inner bounding box
localInnerBBox = geom.Box2I(
tractInfo._minimumBoundingBox(tractInfo.getWcs()))
localBox.clip(localInnerBBox)
patchArgDict = dict(
datasetType="deepCoadd_sub",
bbox=localBox,
tract=tractInfo.getId(),
patch="%s,%s" %
(patchInfo.getIndex()[0], patchInfo.getIndex()[1]),
filter=exposure.getFilter().getName())
coaddPatch = sensorRef.get(**patchArgDict)
# warp to reference tract wcs
xyTransform = afwGeom.makeWcsPairTransform(
coaddPatch.getWcs(), finalWcs)
psfWarped = WarpedPsf(coaddPatch.getPsf(), xyTransform)
warped = self.warper.warpExposure(finalWcs,
coaddPatch,
maxBBox=finalBBox)
# check if warpped image is viable
if warped.getBBox().getArea() == 0:
self.log.info(
"No ovlerap for warped patch %s. Skipping" %
patchInfo)
continue
warped.setPsf(psfWarped)
exp = afwImage.ExposureF(finalBBox, finalWcs)
exp.maskedImage.set(
np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
exp.maskedImage.assign(warped.maskedImage,
warped.getBBox())
maskedImageList.append(exp.maskedImage)
weightList.append(1)
record = tractsCatalog.addNew()
record.setPsf(psfWarped)
record.setWcs(finalWcs)
record.setPhotoCalib(coaddPatch.getPhotoCalib())
record.setBBox(warped.getBBox())
record.set(tractKey, tract)
record.set(patchKey, patchInt)
record.set(weightKey, 1.)
nPatchesFound += 1
if nPatchesFound == 0:
raise RuntimeError("No patches found!")
# Combine images from individual patches together
# Do not mask any values
statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
maskMap = []
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
coaddExposure = afwImage.ExposureF(finalBBox, finalWcs)
coaddExposure.maskedImage.set(np.nan,
afwImage.Mask.getPlaneBitMask("NO_DATA"),
np.nan)
xy0 = coaddExposure.getXY0()
coaddExposure.maskedImage = afwMath.statisticsStack(
maskedImageList, statsFlags, statsCtrl, weightList, 0, maskMap)
coaddExposure.maskedImage.setXY0(xy0)
coaddPsf = CoaddPsf(tractsCatalog, finalWcs,
self.config.coaddPsf.makeControl())
if coaddPsf is None:
raise RuntimeError("No coadd Psf found!")
coaddExposure.setPsf(coaddPsf)
coaddExposure.setFilter(coaddFilter)
return pipeBase.Struct(exposure=coaddExposure, sources=None)
def overscanCorrection(ampMaskedImage,
overscanImage,
fitType='MEDIAN',
order=1,
collapseRej=3.0,
statControl=None,
overscanIsInt=True):
"""Apply overscan correction in place.
Parameters
----------
ampMaskedImage : `lsst.afw.image.MaskedImage`
Image of amplifier to correct; modified.
overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
Image of overscan; modified.
fitType : `str`
Type of fit for overscan correction. May be one of:
- ``MEAN``: use mean of overscan.
- ``MEANCLIP``: use clipped mean of overscan.
- ``MEDIAN``: use median of overscan.
- ``POLY``: fit with ordinary polynomial.
- ``CHEB``: fit with Chebyshev polynomial.
- ``LEG``: fit with Legendre polynomial.
- ``NATURAL_SPLINE``: fit with natural spline.
- ``CUBIC_SPLINE``: fit with cubic spline.
- ``AKIMA_SPLINE``: fit with Akima spline.
order : `int`
Polynomial order or number of spline knots; ignored unless
``fitType`` indicates a polynomial or spline.
statControl : `lsst.afw.math.StatisticsControl`
Statistics control object. In particular, we pay attention to numSigmaClip
overscanIsInt : `bool`
Treat the overscan region as consisting of integers, even if it's been
converted to float. E.g. handle ties properly.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``imageFit``: Value(s) removed from image (scalar or
`lsst.afw.image.Image`)
- ``overscanFit``: Value(s) removed from overscan (scalar or
`lsst.afw.image.Image`)
- ``overscanImage``: Overscan corrected overscan region
(`lsst.afw.image.Image`)
Raises
------
pexExcept.Exception
Raised if ``fitType`` is not an allowed value.
Notes
-----
The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
subtracted. Note that the ``overscanImage`` should not be a subimage of
the ``ampMaskedImage``, to avoid being subtracted twice.
Debug plots are available for the SPLINE fitTypes by setting the
`debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
plots show the scatter plot of the overscan data (collapsed along
the perpendicular dimension) as a function of position on the CCD
(normalized between +/-1).
"""
ampImage = ampMaskedImage.getImage()
if statControl is None:
statControl = afwMath.StatisticsControl()
numSigmaClip = statControl.getNumSigmaClip()
if fitType in ('MEAN', 'MEANCLIP'):
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImage, fitType,
statControl).getValue()
overscanFit = offImage
elif fitType in ('MEDIAN', ):
if overscanIsInt:
# we need an image with integer pixels to handle ties properly
if hasattr(overscanImage, "image"):
imageI = overscanImage.image.convertI()
overscanImageI = afwImage.MaskedImageI(imageI,
overscanImage.mask,
overscanImage.variance)
else:
overscanImageI = overscanImage.convertI()
else:
overscanImageI = overscanImage
fitType = afwMath.stringToStatisticsProperty(fitType)
offImage = afwMath.makeStatistics(overscanImageI, fitType,
statControl).getValue()
overscanFit = offImage
if overscanIsInt:
del overscanImageI
elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE',
'AKIMA_SPLINE'):
if hasattr(overscanImage, "getImage"):
biasArray = overscanImage.getImage().getArray()
biasArray = numpy.ma.masked_where(
overscanImage.getMask().getArray() & statControl.getAndMask(),
biasArray)
else:
biasArray = overscanImage.getArray()
# Fit along the long axis, so collapse along each short row and fit the resulting array
shortInd = numpy.argmin(biasArray.shape)
if shortInd == 0:
# Convert to some 'standard' representation to make things easier
biasArray = numpy.transpose(biasArray)
# Do a single round of clipping to weed out CR hits and signal leaking into the overscan
percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
medianBiasArr = percentiles[1]
stdevBiasArr = 0.74 * (percentiles[2] - percentiles[0]) # robust stdev
diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
biasMaskedArr = numpy.ma.masked_where(
diff > numSigmaClip * stdevBiasArr[:, numpy.newaxis], biasArray)
collapsed = numpy.mean(biasMaskedArr, axis=1)
if collapsed.mask.sum() > 0:
collapsed.data[collapsed.mask] = numpy.mean(
biasArray.data[collapsed.mask], axis=1)
del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
if shortInd == 0:
collapsed = numpy.transpose(collapsed)
num = len(collapsed)
indices = 2.0 * numpy.arange(num) / float(num) - 1.0
if fitType in ('POLY', 'CHEB', 'LEG'):
# A numpy polynomial
poly = numpy.polynomial
fitter, evaler = {
"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
"CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
"LEG": (poly.legendre.legfit, poly.legendre.legval),
}[fitType]
coeffs = fitter(indices, collapsed, order)
fitBiasArr = evaler(indices, coeffs)
elif 'SPLINE' in fitType:
# An afw interpolation
numBins = order
#
# numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
# no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
#
# Issue DM-415
#
collapsedMask = collapsed.mask
try:
if collapsedMask == numpy.ma.nomask:
collapsedMask = numpy.array(
len(collapsed) * [numpy.ma.nomask])
except ValueError: # If collapsedMask is an array the test fails [needs .all()]
pass
numPerBin, binEdges = numpy.histogram(indices,
bins=numBins,
weights=1 -
collapsedMask.astype(int))
# Binning is just a histogram, with weights equal to the values.
# Use a similar trick to get the bin centers (this deals with different numbers per bin).
with numpy.errstate(invalid="ignore"): # suppress NAN warnings
values = numpy.histogram(
indices,
bins=numBins,
weights=collapsed.data * ~collapsedMask)[0] / numPerBin
binCenters = numpy.histogram(
indices, bins=numBins,
weights=indices * ~collapsedMask)[0] / numPerBin
interp = afwMath.makeInterpolate(
binCenters.astype(float)[numPerBin > 0],
values.astype(float)[numPerBin > 0],
afwMath.stringToInterpStyle(fitType))
fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
import lsstDebug
if lsstDebug.Info(__name__).display:
import matplotlib.pyplot as plot
figure = plot.figure(1)
figure.clear()
axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
if collapsedMask.sum() > 0:
axes.plot(indices[collapsedMask],
collapsed.data[collapsedMask], 'b+')
axes.plot(indices, fitBiasArr, 'r-')
plot.xlabel("centered/scaled position along overscan region")
plot.ylabel("pixel value/fit value")
figure.show()
prompt = "Press Enter or c to continue [chp]... "
while True:
ans = input(prompt).lower()
if ans in (
"",
"c",
):
break
if ans in ("p", ):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print("h[elp] c[ontinue] p[db]")
plot.close()
offImage = ampImage.Factory(ampImage.getDimensions())
offArray = offImage.getArray()
overscanFit = afwImage.ImageF(overscanImage.getDimensions())
overscanArray = overscanFit.getArray()
if shortInd == 1:
offArray[:, :] = fitBiasArr[:, numpy.newaxis]
overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
else:
offArray[:, :] = fitBiasArr[numpy.newaxis, :]
overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
# We don't trust any extrapolation: mask those pixels as SUSPECT
# This will occur when the top and or bottom edges of the overscan
# contain saturated values. The values will be extrapolated from
# the surrounding pixels, but we cannot entirely trust the value of
# the extrapolation, and will mark the image mask plane to flag the
# image as such.
mask = ampMaskedImage.getMask()
maskArray = mask.getArray() if shortInd == 1 else mask.getArray(
).transpose()
suspect = mask.getPlaneBitMask("SUSPECT")
try:
if collapsed.mask == numpy.ma.nomask:
# There is no mask, so the whole array is fine
pass
except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
for low in range(num):
if not collapsed.mask[low]:
break
if low > 0:
maskArray[:low, :] |= suspect
for high in range(1, num):
if not collapsed.mask[-high]:
break
if high > 1:
maskArray[-high:, :] |= suspect
else:
raise pexExcept.Exception('%s : %s an invalid overscan type' %
("overscanCorrection", fitType))
ampImage -= offImage
overscanImage -= overscanFit
return Struct(imageFit=offImage,
overscanFit=overscanFit,
overscanImage=overscanImage)
