def processSingle(self, sensorRef):
"""Process a single CCD
Besides the regular ISR, also masks cosmic-rays.
"""
if not self.config.rerunISR:
try:
exposure = sensorRef.get('postISRCCD')
self.log.debug("Obtained postISRCCD from butler for %s" % sensorRef.dataId)
return exposure
except NoResults:
pass # ah well. We'll have to run the ISR
exposure = super().processSingle(sensorRef)
if self.config.doRepair:
psf = DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
exposure.setPsf(psf)
self.repair.run(exposure, keepCRs=False)
if self.config.crGrow > 0:
mask = exposure.getMaskedImage().getMask().clone()
mask &= mask.getPlaneBitMask("CR")
fpSet = FootprintSet(mask, Threshold(0.5))
fpSet = FootprintSet(fpSet, self.config.crGrow, True)
fpSet.setMask(exposure.getMaskedImage().getMask(), "CR")
if self.debugInfo.display and self.debugInfo.inputsFrame >= 0:
display = Display(frame=self.debugInfo.inputsFrame)
display.mtv(exposure, "raw %(visit)d" % sensorRef.dataId)
return exposure
def extractCrosstalkRatios(exposure,
threshold=30000,
badPixels=["SAT", "BAD", "INTRP"]):
"""Extract crosstalk ratios between different amplifiers
For pixels above ``threshold``, we calculate the ratio between each
target amp and source amp. We return a list of ratios for each pixel
for each target/source combination, as a matrix of lists.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure for which to measure crosstalk.
threshold : `float`
Lower limit on pixels for which we measure crosstalk.
badPixels : `list` of `str`
Mask planes indicating a pixel is bad.
Returns
-------
ratios : `list` of `list` of `numpy.ndarray`
A matrix of pixel arrays. ``ratios[i][j]`` is an array of
the fraction of the ``j``-th amp present on the ``i``-th amp.
The value is `None` for the diagonal elements.
"""
mi = exposure.getMaskedImage()
FootprintSet(mi, Threshold(threshold), "DETECTED")
detected = mi.getMask().getPlaneBitMask("DETECTED")
bad = mi.getMask().getPlaneBitMask(badPixels)
bg = calculateBackground(mi, badPixels + ["DETECTED"])
ccd = exposure.getDetector()
ratios = [[None for iAmp in ccd] for jAmp in ccd]
for ii, iAmp in enumerate(ccd):
iImage = mi[iAmp.getBBox()]
iMask = iImage.mask.array
select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(
iImage.image.array)
for jj, jAmp in enumerate(ccd):
if ii == jj:
continue
jImage = extractAmp(mi.image,
jAmp,
iAmp.getReadoutCorner(),
isTrimmed=True)
ratios[jj][ii] = (jImage.array[select] -
bg) / iImage.image.array[select]
return ratios
def find_bright_columns(imarr, threshold):
"""Find bright columns in an image array.
Parameters
----------
imarr : `numpy.ndarrawy`, (Nx, Ny)
An array representing an image to analyze.
threshold : `float`
Pixel value threshold defining a bright column.
Returns
-------
bright_cols : `list`
List of column indices corresponding to bright columns.
"""
image = afwImage.ImageF(imarr)
fp_set = FootprintSet(image, Threshold(threshold))
columns = dict([(x, []) for x in range(0, image.getWidth())])
for footprint in fp_set.getFootprints():
for span in footprint.getSpans():
y = span.getY()
for x in range(span.getX0(), span.getX1() + 1):
columns[x].append(y)
bright_cols = []
x0 = image.getX0()
y0 = image.getY0()
for x in columns:
if bad_column(columns[x], 20):
bright_cols.append(x - x0)
#
# Sort the output.
#
bright_cols.sort()
return bright_cols
def run(self, inputExp, sourceExps=[]):
"""Measure pixel ratios between amplifiers in inputExp.
Extract crosstalk ratios between different amplifiers.
For pixels above ``config.threshold``, we calculate the ratio
between each background-subtracted target amp and the source
amp. We return a list of ratios for each pixel for each
target/source combination, as nested dictionary containing the
ratio.
Parameters
----------
inputExp : `lsst.afw.image.Exposure`
Input exposure to measure pixel ratios on.
sourceExp : `list` [`lsst.afw.image.Exposure`], optional
List of chips to use as sources to measure inter-chip
crosstalk.
Returns
-------
results : `lsst.pipe.base.Struct`
The results struct containing:
``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
A catalog of ratio lists. The dictionaries are
indexed such that:
outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
contains the ratio list for that combination.
``outputFluxes`` : `dict` [`dict` [`list`]]
A catalog of flux lists. The dictionaries are
indexed such that:
outputFluxes[sourceChip][sourceAmp]
contains the flux list used in the outputRatios.
Notes
-----
The lsstDebug.Info() method can be rewritten for __name__ =
`lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
debug.display['extract'] : `bool`
Display the exposure under consideration, with the pixels used
for crosstalk measurement indicated by the DETECTED mask plane.
debug.display['pixels'] : `bool`
Display a plot of the ratio calculated for each pixel used in this
exposure, split by amplifier pairs. The median value is listed
for reference.
"""
outputRatios = defaultdict(lambda: defaultdict(dict))
outputFluxes = defaultdict(lambda: defaultdict(dict))
threshold = self.config.threshold
badPixels = list(self.config.badMask)
targetDetector = inputExp.getDetector()
targetChip = targetDetector.getName()
# Always look at the target chip first, then go to any other supplied exposures.
sourceExtractExps = [inputExp]
sourceExtractExps.extend(sourceExps)
self.log.info("Measuring full detector background for target: %s", targetChip)
targetIm = inputExp.getMaskedImage()
FootprintSet(targetIm, Threshold(threshold), "DETECTED")
detected = targetIm.getMask().getPlaneBitMask("DETECTED")
bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"])
self.debugView('extract', inputExp)
for sourceExp in sourceExtractExps:
sourceDetector = sourceExp.getDetector()
sourceChip = sourceDetector.getName()
sourceIm = sourceExp.getMaskedImage()
bad = sourceIm.getMask().getPlaneBitMask(badPixels)
self.log.info("Measuring crosstalk from source: %s", sourceChip)
if sourceExp != inputExp:
FootprintSet(sourceIm, Threshold(threshold), "DETECTED")
detected = sourceIm.getMask().getPlaneBitMask("DETECTED")
# The dictionary of amp-to-amp ratios for this pair of source->target detectors.
ratioDict = defaultdict(lambda: defaultdict(list))
extractedCount = 0
for sourceAmp in sourceDetector:
sourceAmpName = sourceAmp.getName()
sourceAmpImage = sourceIm[sourceAmp.getBBox()]
sourceMask = sourceAmpImage.mask.array
select = ((sourceMask & detected > 0) &
(sourceMask & bad == 0) &
np.isfinite(sourceAmpImage.image.array))
count = np.sum(select)
self.log.debug(" Source amplifier: %s", sourceAmpName)
outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist()
for targetAmp in targetDetector:
# iterate over targetExposure
targetAmpName = targetAmp.getName()
if sourceAmpName == targetAmpName and sourceChip == targetChip:
ratioDict[sourceAmpName][targetAmpName] = []
continue
self.log.debug(" Target amplifier: %s", targetAmpName)
targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
targetAmp, sourceAmp,
isTrimmed=self.config.isTrimmed)
ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
extractedCount += count
self.debugPixels('pixels',
sourceAmpImage.image.array[select],
targetAmpImage.array[select] - bg,
sourceAmpName, targetAmpName)
self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)",
extractedCount, sourceChip, targetChip, bg)
outputRatios[targetChip][sourceChip] = ratioDict
return pipeBase.Struct(
outputRatios=ddict2dict(outputRatios),
outputFluxes=ddict2dict(outputFluxes)
)
def extractCrosstalkRatios(self, exposure, threshold=None, badPixels=None):
"""Extract crosstalk ratios between different amplifiers.
For pixels above ``threshold``, we calculate the ratio between
each background-subtracted target amp and the source amp. We
return a list of ratios for each pixel for each target/source
combination, as a matrix of lists.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure for which to measure crosstalk.
threshold : `float`, optional
Lower limit on pixels for which we measure crosstalk.
badPixels : `list` of `str`, optional
Mask planes indicating a pixel is bad.
Returns
-------
ratios : `list` of `list` of `numpy.ndarray`
A matrix of pixel arrays. ``ratios[i][j]`` is an array of
the fraction of the ``j``-th amp present on the ``i``-th amp.
The value is `None` for the diagonal elements.
Notes
-----
This has been moved into MeasureCrosstalkTask to allow for easier
debugging.
The lsstDebug.Info() method can be rewritten for __name__ =
`lsst.ip.isr.measureCrosstalk`, and supports the parameters:
debug.display['extract'] : `bool`
Display the exposure under consideration, with the pixels used
for crosstalk measurement indicated by the DETECTED mask plane.
debug.display['pixels'] : `bool`
Display a plot of the ratio calculated for each pixel used in this
exposure, split by amplifier pairs. The median value is listed
for reference.
"""
if threshold is None:
threshold = self.config.threshold
if badPixels is None:
badPixels = list(self.config.badMask)
mi = exposure.getMaskedImage()
FootprintSet(mi, Threshold(threshold), "DETECTED")
detected = mi.getMask().getPlaneBitMask("DETECTED")
bad = mi.getMask().getPlaneBitMask(badPixels)
bg = calculateBackground(mi, badPixels + ["DETECTED"])
self.debugView('extract', exposure)
ccd = exposure.getDetector()
ratios = [[None for iAmp in ccd] for jAmp in ccd]
for ii, iAmp in enumerate(ccd):
iImage = mi[iAmp.getBBox()]
iMask = iImage.mask.array
select = (iMask & detected > 0) & (iMask & bad == 0) & np.isfinite(
iImage.image.array)
for jj, jAmp in enumerate(ccd):
if ii == jj:
continue
jImage = extractAmp(mi.image,
jAmp,
iAmp.getReadoutCorner(),
isTrimmed=self.config.isTrimmed)
ratios[jj][ii] = (jImage.array[select] -
bg) / iImage.image.array[select]
self.debugPixels('pixels', iImage.image.array[select],
jImage.array[select] - bg, ii, jj)
return ratios
def calculateThreshold(self, exposure, seed, sigma=None):
"""Calculate new threshold
This is the main functional addition to the vanilla
`SourceDetectionTask`.
We identify sky objects and perform forced PSF photometry on
them. Using those PSF flux measurements and estimated errors,
we set the threshold so that the stdev of the measurements
matches the median estimated error.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure on which we're detecting sources.
seed : `int`
RNG seed to use for finding sky objects.
sigma : `float`, optional
Gaussian sigma of smoothing kernel; if not provided,
will be deduced from the exposure's PSF.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``multiplicative``: multiplicative factor to be applied to the
configured detection threshold (`float`).
- ``additive``: additive factor to be applied to the background
level (`float`).
"""
# Make a catalog of sky objects
fp = self.skyObjects.run(exposure.maskedImage.mask, seed)
skyFootprints = FootprintSet(exposure.getBBox())
skyFootprints.setFootprints(fp)
table = SourceTable.make(self.skyMeasurement.schema)
catalog = SourceCatalog(table)
catalog.reserve(len(skyFootprints.getFootprints()))
skyFootprints.makeSources(catalog)
key = catalog.getCentroidKey()
for source in catalog:
peaks = source.getFootprint().getPeaks()
assert len(peaks) == 1
source.set(key, peaks[0].getF())
source.updateCoord(exposure.getWcs())
# Forced photometry on sky objects
self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs())
# Calculate new threshold
fluxes = catalog["base_PsfFlux_instFlux"]
area = catalog["base_PsfFlux_area"]
bg = catalog["base_LocalBackground_instFlux"]
good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] &
np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg))
if good.sum() < self.config.minNumSources:
self.log.warn("Insufficient good flux measurements (%d < %d) for dynamic threshold calculation",
good.sum(), self.config.minNumSources)
return Struct(multiplicative=1.0, additive=0.0)
bgMedian = np.median((fluxes/area)[good])
lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0])
stdevMeas = 0.741*(uq - lq)
medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good])
return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian)
def calculateThreshold(self, exposure, seed, sigma=None):
"""Calculate new threshold
This is the main functional addition to the vanilla
`SourceDetectionTask`.
We identify sky objects and perform forced PSF photometry on
them. Using those PSF flux measurements and estimated errors,
we set the threshold so that the stdev of the measurements
matches the median estimated error.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure on which we're detecting sources.
seed : `int`
RNG seed to use for finding sky objects.
sigma : `float`, optional
Gaussian sigma of smoothing kernel; if not provided,
will be deduced from the exposure's PSF.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``multiplicative``: multiplicative factor to be applied to the
configured detection threshold (`float`).
- ``additive``: additive factor to be applied to the background
level (`float`).
"""
# Make a catalog of sky objects
fp = self.skyObjects.run(exposure.maskedImage.mask, seed)
skyFootprints = FootprintSet(exposure.getBBox())
skyFootprints.setFootprints(fp)
table = SourceTable.make(self.skyMeasurement.schema)
catalog = SourceCatalog(table)
catalog.reserve(len(skyFootprints.getFootprints()))
skyFootprints.makeSources(catalog)
key = catalog.getCentroidKey()
for source in catalog:
peaks = source.getFootprint().getPeaks()
assert len(peaks) == 1
source.set(key, peaks[0].getF())
source.updateCoord(exposure.getWcs())
# Forced photometry on sky objects
self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs())
# Calculate new threshold
fluxes = catalog["base_PsfFlux_instFlux"]
area = catalog["base_PsfFlux_area"]
bg = catalog["base_LocalBackground_instFlux"]
good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] &
np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg))
if good.sum() < self.config.minNumSources:
self.log.warn("Insufficient good flux measurements (%d < %d) for dynamic threshold calculation",
good.sum(), self.config.minNumSources)
return Struct(multiplicative=1.0, additive=0.0)
bgMedian = np.median((fluxes/area)[good])
lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0])
stdevMeas = 0.741*(uq - lq)
medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good])
return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian)