def testTaskAPI(self):
"""Test that the Tasks work
Checks both MeasureCrosstalkTask and the CrosstalkTask.
"""
# make exposure available to NullIsrTask
# without NullIsrTask's `self` hiding this test class's `self`
exposure = self.exposure
class NullIsrTask(IsrTask):
def runDataRef(self, dataRef):
return Struct(exposure=exposure)
config = MeasureCrosstalkTask.ConfigClass()
config.isr.retarget(NullIsrTask)
config.threshold = self.value - 1
measure = MeasureCrosstalkTask(config=config)
fakeDataRef = Struct(dataId={'fake': 1})
coeff, coeffErr, coeffNum = measure.reduce(
[measure.runDataRef(fakeDataRef)])
self.checkCoefficients(coeff, coeffErr, coeffNum)
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
calib = CrosstalkCalib().fromDetector(self.exposure.getDetector(),
coeffVector=coeff.transpose())
isr.crosstalk.run(self.exposure, crosstalk=calib)
self.checkSubtracted(self.exposure)
def filterCrosstalkCalib(inCalib):
"""Apply valid constraints to the measured values.
Any measured coefficient that is determined to be invalid is
set to zero, and has the error set to nan. The validation is
determined by checking that the measured coefficient is larger
than the calculated standard error of the mean.
Parameters
----------
inCalib : `lsst.ip.isr.CrosstalkCalib`
Input calibration to filter.
Returns
-------
outCalib : `lsst.ip.isr.CrosstalkCalib`
Filtered calibration.
"""
outCalib = CrosstalkCalib()
outCalib.numAmps = inCalib.numAmps
outCalib.coeffs = inCalib.coeffs
outCalib.coeffs[~inCalib.coeffValid] = 0.0
outCalib.coeffErr = inCalib.coeffErr
outCalib.coeffErr[~inCalib.coeffValid] = np.nan
outCalib.coeffNum = inCalib.coeffNum
outCalib.coeffValid = inCalib.coeffValid
return outCalib
def test_interChip(self):
"""Test that passing an external exposure as the crosstalk source
works.
"""
exposure = self.exposure
ctSources = [self.ctSource]
coeff = np.array(self.crosstalk).transpose()
calib = CrosstalkCalib().fromDetector(exposure.getDetector(),
coeffVector=coeff)
# Now convert this into zero intra-chip, full inter-chip:
calib.interChip['detector 2'] = coeff
calib.coeffs = np.zeros_like(coeff)
# Process and check as above
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
isr.crosstalk.run(exposure,
crosstalk=calib,
crosstalkSources=ctSources)
self.checkSubtracted(exposure)
def testDirectAPI(self):
"""Test that individual function calls work"""
calib = CrosstalkCalib()
calib.coeffs = np.array(self.crosstalk).transpose()
calib.subtractCrosstalk(self.exposure,
crosstalkCoeffs=calib.coeffs,
minPixelToMask=self.value - 1,
crosstalkStr=self.crosstalkStr)
self.checkSubtracted(self.exposure)
outPath = tempfile.mktemp(
) if outputName is None else "{}-isrCrosstalk".format(outputName)
outPath += '.yaml'
calib.writeText(outPath)
def test_crosstalkIO(self):
"""Test that crosstalk doesn't change on being converted to persistable
formats.
"""
# Add the interchip crosstalk as in the previous test.
exposure = self.exposure
coeff = np.array(self.crosstalk).transpose()
calib = CrosstalkCalib().fromDetector(exposure.getDetector(),
coeffVector=coeff)
# Now convert this into zero intra-chip, full inter-chip:
calib.interChip['detector 2'] = coeff
outPath = tempfile.mktemp() + '.yaml'
calib.writeText(outPath)
newCrosstalk = CrosstalkCalib().readText(outPath)
self.assertEqual(calib, newCrosstalk)
outPath = tempfile.mktemp() + '.fits'
calib.writeFits(outPath)
newCrosstalk = CrosstalkCalib().readFits(outPath)
self.assertEqual(calib, newCrosstalk)
def testDirectAPI(self):
"""Test that individual function calls work"""
config = MeasureCrosstalkTask.ConfigClass()
measure = MeasureCrosstalkTask(config=config)
ratios = measure.extractCrosstalkRatios(self.exposure,
threshold=self.value - 1)
coeff, coeffErr, coeffNum = measure.measureCrosstalkCoefficients(
ratios)
self.checkCoefficients(coeff, coeffErr, coeffNum)
calib = CrosstalkCalib()
calib.coeffs = coeff.transpose()
calib.coeffErr = coeffErr.transpose()
calib.coeffNum = coeffNum.transpose()
calib.subtractCrosstalk(self.exposure,
crosstalkCoeffs=coeff.transpose(),
minPixelToMask=self.value - 1,
crosstalkStr=self.crosstalkStr)
self.checkSubtracted(self.exposure)
outPath = tempfile.mktemp(
) if outputName is None else "{}-isrCrosstalk".format(outputName)
outPath += '.yaml'
calib.writeText(outPath)
def testTaskAPI(self):
"""Test that the Tasks work
Checks both MeasureCrosstalkTask and the CrosstalkTask.
"""
# make exposure available to NullIsrTask
# without NullIsrTask's `self` hiding this test class's `self`
exposure = self.exposure
class NullIsrTask(IsrTask):
def runDataRef(self, dataRef):
return Struct(exposure=exposure)
coeff = np.array(self.crosstalk).transpose()
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
calib = CrosstalkCalib().fromDetector(self.exposure.getDetector(),
coeffVector=coeff)
isr.crosstalk.run(self.exposure, crosstalk=calib)
self.checkSubtracted(self.exposure)
def measureCrosstalkCoefficients(self, ratios, rejIter, rejSigma):
"""Measure crosstalk coefficients from the ratios.
Given a list of ratios for each target/source amp combination,
we measure a sigma clipped mean and error.
The coefficient errors returned are the standard deviation of
the final set of clipped input ratios.
Parameters
----------
ratios : `dict` of `dict` of `numpy.ndarray`
Catalog of arrays of ratios.
rejIter : `int`
Number of rejection iterations.
rejSigma : `float`
Rejection threshold (sigma).
Returns
-------
calib : `lsst.ip.isr.CrosstalkCalib`
The output crosstalk calibration.
Notes
-----
The lsstDebug.Info() method can be rewritten for __name__ =
`lsst.ip.isr.measureCrosstalk`, and supports the parameters:
debug.display['measure'] : `bool`
Display the CDF of the combined ratio measurements for
a pair of source/target amplifiers from the final set of
clipped input ratios.
"""
calib = CrosstalkCalib(nAmp=len(ratios))
# Calibration stores coefficients as a numpy ndarray.
ordering = list(ratios.keys())
for ii, jj in itertools.product(range(calib.nAmp), range(calib.nAmp)):
if ii == jj:
values = [0.0]
else:
values = np.array(ratios[ordering[ii]][ordering[jj]])
values = values[np.abs(values) < 1.0] # Discard unreasonable values
calib.coeffNum[ii][jj] = len(values)
if len(values) == 0:
self.log.warn("No values for matrix element %d,%d" % (ii, jj))
calib.coeffs[ii][jj] = np.nan
calib.coeffErr[ii][jj] = np.nan
calib.coeffValid[ii][jj] = False
else:
if ii != jj:
for rej in range(rejIter):
lo, med, hi = np.percentile(values, [25.0, 50.0, 75.0])
sigma = 0.741*(hi - lo)
good = np.abs(values - med) < rejSigma*sigma
if good.sum() == len(good):
break
values = values[good]
calib.coeffs[ii][jj] = np.mean(values)
if calib.coeffNum[ii][jj] == 1:
calib.coeffErr[ii][jj] = np.nan
else:
correctionFactor = self.sigmaClipCorrection(rejSigma)
calib.coeffErr[ii][jj] = np.std(values) * correctionFactor
calib.coeffValid[ii][jj] = (np.abs(calib.coeffs[ii][jj]) >
calib.coeffErr[ii][jj] / np.sqrt(calib.coeffNum[ii][jj]))
if calib.coeffNum[ii][jj] > 1:
self.debugRatios('measure', ratios, ordering[ii], ordering[jj],
calib.coeffs[ii][jj], calib.coeffValid[ii][jj])
return calib
def run(self, inputExp, sourceExps=[]):
"""Measure pixel ratios between amplifiers in inputExp.
Extract crosstalk ratios between different amplifiers.
For pixels above ``config.threshold``, we calculate the ratio
between each background-subtracted target amp and the source
amp. We return a list of ratios for each pixel for each
target/source combination, as nested dictionary containing the
ratio.
Parameters
----------
inputExp : `lsst.afw.image.Exposure`
Input exposure to measure pixel ratios on.
sourceExp : `list` [`lsst.afw.image.Exposure`], optional
List of chips to use as sources to measure inter-chip
crosstalk.
Returns
-------
results : `lsst.pipe.base.Struct`
The results struct containing:
``outputRatios`` : `dict` [`dict` [`dict` [`dict` [`list`]]]]
A catalog of ratio lists. The dictionaries are
indexed such that:
outputRatios[targetChip][sourceChip][targetAmp][sourceAmp]
contains the ratio list for that combination.
``outputFluxes`` : `dict` [`dict` [`list`]]
A catalog of flux lists. The dictionaries are
indexed such that:
outputFluxes[sourceChip][sourceAmp]
contains the flux list used in the outputRatios.
Notes
-----
The lsstDebug.Info() method can be rewritten for __name__ =
`lsst.cp.pipe.measureCrosstalk`, and supports the parameters:
debug.display['extract'] : `bool`
Display the exposure under consideration, with the pixels used
for crosstalk measurement indicated by the DETECTED mask plane.
debug.display['pixels'] : `bool`
Display a plot of the ratio calculated for each pixel used in this
exposure, split by amplifier pairs. The median value is listed
for reference.
"""
outputRatios = defaultdict(lambda: defaultdict(dict))
outputFluxes = defaultdict(lambda: defaultdict(dict))
threshold = self.config.threshold
badPixels = list(self.config.badMask)
targetDetector = inputExp.getDetector()
targetChip = targetDetector.getName()
# Always look at the target chip first, then go to any other supplied exposures.
sourceExtractExps = [inputExp]
sourceExtractExps.extend(sourceExps)
self.log.info("Measuring full detector background for target: %s", targetChip)
targetIm = inputExp.getMaskedImage()
FootprintSet(targetIm, Threshold(threshold), "DETECTED")
detected = targetIm.getMask().getPlaneBitMask("DETECTED")
bg = CrosstalkCalib.calculateBackground(targetIm, badPixels + ["DETECTED"])
self.debugView('extract', inputExp)
for sourceExp in sourceExtractExps:
sourceDetector = sourceExp.getDetector()
sourceChip = sourceDetector.getName()
sourceIm = sourceExp.getMaskedImage()
bad = sourceIm.getMask().getPlaneBitMask(badPixels)
self.log.info("Measuring crosstalk from source: %s", sourceChip)
if sourceExp != inputExp:
FootprintSet(sourceIm, Threshold(threshold), "DETECTED")
detected = sourceIm.getMask().getPlaneBitMask("DETECTED")
# The dictionary of amp-to-amp ratios for this pair of source->target detectors.
ratioDict = defaultdict(lambda: defaultdict(list))
extractedCount = 0
for sourceAmp in sourceDetector:
sourceAmpName = sourceAmp.getName()
sourceAmpImage = sourceIm[sourceAmp.getBBox()]
sourceMask = sourceAmpImage.mask.array
select = ((sourceMask & detected > 0) &
(sourceMask & bad == 0) &
np.isfinite(sourceAmpImage.image.array))
count = np.sum(select)
self.log.debug(" Source amplifier: %s", sourceAmpName)
outputFluxes[sourceChip][sourceAmpName] = sourceAmpImage.image.array[select].tolist()
for targetAmp in targetDetector:
# iterate over targetExposure
targetAmpName = targetAmp.getName()
if sourceAmpName == targetAmpName and sourceChip == targetChip:
ratioDict[sourceAmpName][targetAmpName] = []
continue
self.log.debug(" Target amplifier: %s", targetAmpName)
targetAmpImage = CrosstalkCalib.extractAmp(targetIm.image,
targetAmp, sourceAmp,
isTrimmed=self.config.isTrimmed)
ratios = (targetAmpImage.array[select] - bg)/sourceAmpImage.image.array[select]
ratioDict[targetAmpName][sourceAmpName] = ratios.tolist()
extractedCount += count
self.debugPixels('pixels',
sourceAmpImage.image.array[select],
targetAmpImage.array[select] - bg,
sourceAmpName, targetAmpName)
self.log.info("Extracted %d pixels from %s -> %s (targetBG: %f)",
extractedCount, sourceChip, targetChip, bg)
outputRatios[targetChip][sourceChip] = ratioDict
return pipeBase.Struct(
outputRatios=ddict2dict(outputRatios),
outputFluxes=ddict2dict(outputFluxes)
)
def run(self, inputExp, rawExp):
## run() method
outputRatios = defaultdict(lambda: defaultdict(dict))
outputFluxes = defaultdict(lambda: defaultdict(dict))
outputZOffsets = defaultdict(lambda: defaultdict(dict))
outputYTilts = defaultdict(lambda: defaultdict(dict))
outputXTilts = defaultdict(lambda: defaultdict(dict))
badPixels = list(self.config.badMask)
targetDetector = inputExp.getDetector()
targetChip = targetDetector.getName()
targetIm = inputExp.getMaskedImage()
## loop on sourceExp would go here
sourceExp = inputExp ## Ignore other exposures for now
sourceDetector = sourceExp.getDetector()
sourceChip = sourceDetector.getName()
sourceIm = sourceExp.getMaskedImage()
bad = sourceIm.getMask().getPlaneBitMask(badPixels)
self.log.info("Measuring crosstalk from source: %s", sourceChip)
ratioDict = defaultdict(lambda: defaultdict(list))
zoffsetDict = defaultdict(lambda: defaultdict(list))
ytiltDict = defaultdict(lambda: defaultdict(list))
xtiltDict = defaultdict(lambda: defaultdict(list))
extractedCount = 0
for sourceAmp in sourceDetector:
sourceAmpName = sourceAmp.getName()
sourceAmpImage = sourceIm[sourceAmp.getBBox()]
sourceMask = sourceAmpImage.mask.array
sourceAmpArray = sourceAmpImage.image.array
tested_angles = np.linspace(-np.pi / 2, np.pi / 2, 1000)
edges = feature.canny(sourceAmpArray,
sigma=self.config.cannySigma,
low_threshold=self.config.thresholdLow,
high_threshold=self.config.thresholdHigh)
h, theta, d = hough_line(edges, theta=tested_angles)
_, angle, dist = hough_line_peaks(h, theta, d)
if len(angle) != 2:
continue
mean_angle = np.mean(angle)
mean_dist = np.mean(dist)
select = mixCrosstalk.satellite_mask(sourceAmpArray,
mean_angle,
mean_dist,
width=self.config.maskWidth)
signal = np.max(sourceAmpArray[select])
self.log.debug(" Source amplifier: %s", sourceAmpName)
outputFluxes[sourceChip][sourceAmpName] = [float(signal)]
for targetAmp in targetDetector:
# iterate over targetExposure
targetAmpName = targetAmp.getName()
if sourceAmpName == targetAmpName and sourceChip == targetChip:
ratioDict[sourceAmpName][targetAmpName] = []
continue
self.log.debug(" Target amplifier: %s", targetAmpName)
if self.config.correctNoiseCovariance:
covariance = mixCrosstalk.calculate_covariance(
rawExp, sourceAmp, targetAmp)
else:
noise = np.asarray(
[[sourceAmp.getReadNoise() / sourceAmp.getGain(), 0.],
[0.,
targetAmp.getReadNoise() / targetAmp.getGain()]])
covariance = np.square(noise)
targetAmpImage = CrosstalkCalib.extractAmp(
targetIm.image,
targetAmp,
sourceAmp,
isTrimmed=self.config.isTrimmed)
targetAmpArray = targetAmpImage.array
results = mixCrosstalk.crosstalk_fit(
sourceAmpArray,
targetAmpArray,
select,
covariance=covariance,
correct_covariance=self.config.correctNoiseCovariance,
seed=189)
ratioDict[targetAmpName][sourceAmpName] = [float(results[0])]
zoffsetDict[targetAmpName][sourceAmpName] = [float(results[1])]
ytiltDict[targetAmpName][sourceAmpName] = [float(results[2])]
xtiltDict[targetAmpName][sourceAmpName] = [float(results[3])]
extractedCount += 1
self.log.info("Extracted %d pixels from %s -> %s", extractedCount,
sourceChip, targetChip)
outputRatios[targetChip][sourceChip] = ratioDict
outputZOffsets[targetChip][sourceChip] = zoffsetDict
outputYTilts[targetChip][sourceChip] = ytiltDict
outputXTilts[targetChip][sourceChip] = xtiltDict
return pipeBase.Struct(outputRatios=ddict2dict(outputRatios),
outputFluxes=ddict2dict(outputFluxes),
outputZOffsets=ddict2dict(outputZOffsets),
outputYTilts=ddict2dict(outputYTilts),
outputXTilts=ddict2dict(outputXTilts))
def run(self, inputExp, rawExp):
## run() method
outputRatios = defaultdict(lambda: defaultdict(dict))
outputFluxes = defaultdict(lambda: defaultdict(dict))
outputZOffsets = defaultdict(lambda: defaultdict(dict))
outputYTilts = defaultdict(lambda: defaultdict(dict))
outputXTilts = defaultdict(lambda: defaultdict(dict))
badPixels = list(self.config.badMask)
targetDetector = inputExp.getDetector()
targetChip = targetDetector.getName()
targetIm = inputExp.getMaskedImage()
## loop on sourceExp would go here
sourceExp = inputExp ## Ignore other exposures for now
sourceDetector = sourceExp.getDetector()
sourceChip = sourceDetector.getName()
sourceIm = sourceExp.getMaskedImage()
bad = sourceIm.getMask().getPlaneBitMask(badPixels)
self.log.info("Measuring crosstalk from source: %s", sourceChip)
ratioDict = defaultdict(lambda: defaultdict(list))
zoffsetDict = defaultdict(lambda: defaultdict(list))
ytiltDict = defaultdict(lambda: defaultdict(list))
xtiltDict = defaultdict(lambda: defaultdict(list))
extractedCount = 0
for sourceAmp in sourceDetector:
sourceAmpName = sourceAmp.getName()
sourceAmpImage = sourceIm[sourceAmp.getBBox()]
sourceMask = sourceAmpImage.mask.array
sourceAmpArray = sourceAmpImage.image.array
columns = mixCrosstalk.find_bright_columns(sourceAmpArray,
self.config.threshold)
if len(columns) == 0: continue
select = mixCrosstalk.rectangular_mask(sourceAmpArray,
1000,
columns[0],
ly=self.config.maskLengthY,
lx=self.config.maskLengthX)
signal = np.mean(sourceAmpArray[:, columns[0]])
self.log.debug(" Source amplifier: %s", sourceAmpName)
outputFluxes[sourceChip][sourceAmpName] = [float(signal)]
for targetAmp in targetDetector:
# iterate over targetExposure
targetAmpName = targetAmp.getName()
if sourceAmpName == targetAmpName and sourceChip == targetChip:
ratioDict[sourceAmpName][targetAmpName] = []
continue
self.log.debug(" Target amplifier: %s", targetAmpName)
if self.config.correctNoiseCovariance:
covariance = mixCrosstalk.calculate_covariance(
rawExp, sourceAmp, targetAmp)
else:
noise = np.asarray(
[[sourceAmp.getReadNoise() / sourceAmp.getGain(), 0.],
[0.,
targetAmp.getReadNoise() / targetAmp.getGain()]])
covariance = np.square(noise)
targetAmpImage = CrosstalkCalib.extractAmp(
targetIm.image,
targetAmp,
sourceAmp,
isTrimmed=self.config.isTrimmed)
targetAmpArray = targetAmpImage.array
results = mixCrosstalk.crosstalk_fit(
sourceAmpArray,
targetAmpArray,
select,
covariance=covariance,
correct_covariance=self.config.correctNoiseCovariance,
seed=189)
ratioDict[targetAmpName][sourceAmpName] = [float(results[0])]
zoffsetDict[targetAmpName][sourceAmpName] = [float(results[1])]
ytiltDict[targetAmpName][sourceAmpName] = [float(results[2])]
xtiltDict[targetAmpName][sourceAmpName] = [float(results[3])]
extractedCount += 1
self.log.info("Extracted %d pixels from %s -> %s", extractedCount,
sourceChip, targetChip)
outputRatios[targetChip][sourceChip] = ratioDict
outputZOffsets[targetChip][sourceChip] = zoffsetDict
outputYTilts[targetChip][sourceChip] = ytiltDict
outputXTilts[targetChip][sourceChip] = xtiltDict
return pipeBase.Struct(outputRatios=ddict2dict(outputRatios),
outputFluxes=ddict2dict(outputFluxes),
outputZOffsets=ddict2dict(outputZOffsets),
outputYTilts=ddict2dict(outputYTilts),
outputXTilts=ddict2dict(outputXTilts))