def testBasics(self):
"""!Test basic functionality of LinearizeSquared
"""
for imageClass in (afwImage.ImageF, afwImage.ImageD):
inImage = makeRampImage(bbox=self.bbox,
start=-5,
stop=2500,
imageClass=imageClass)
measImage = inImage.Factory(inImage, True)
linCorr = Linearizer(detector=self.detector)
linRes = linCorr.applyLinearity(image=measImage,
detector=self.detector)
desNumLinearized = np.sum(self.sqCoeffs.flatten() > 0)
self.assertEqual(linRes.numLinearized, desNumLinearized)
self.assertEqual(linRes.numAmps,
len(self.detector.getAmplifiers()))
refImage = inImage.Factory(inImage, True)
refLinearizeSquared(image=refImage, detector=self.detector)
self.assertImagesAlmostEqual(refImage, measImage)
# make sure logging is accepted
log = logging.getLogger("lsst.ip.isr.LinearizeSquared")
linRes = linCorr.applyLinearity(image=measImage,
detector=self.detector,
log=log)
def testBasics(self):
"""!Test basic functionality of LinearizeLookupTable
"""
for imageClass in (afwImage.ImageF, afwImage.ImageD):
inImage = makeRampImage(bbox=self.bbox,
start=-5,
stop=250,
imageClass=imageClass)
table = self.makeTable(inImage)
log = logging.getLogger("lsst.ip.isr.LinearizeLookupTable")
measImage = inImage.Factory(inImage, True)
llt = Linearizer(table=table, detector=self.detector)
linRes = llt.applyLinearity(measImage,
detector=self.detector,
log=log)
refImage = inImage.Factory(inImage, True)
refNumOutOfRange = refLinearize(image=refImage,
detector=self.detector,
table=table)
self.assertEqual(linRes.numAmps,
len(self.detector.getAmplifiers()))
self.assertEqual(linRes.numAmps, linRes.numLinearized)
self.assertEqual(linRes.numOutOfRange, refNumOutOfRange)
self.assertImagesAlmostEqual(refImage, measImage)
# make sure logging is accepted
log = logging.getLogger("lsst.ip.isr.LinearizeLookupTable")
linRes = llt.applyLinearity(image=measImage,
detector=self.detector,
log=log)
def testKnown(self):
"""!Test a few known values
"""
numAmps = (2, 2)
bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
lsst.geom.Extent2I(4, 4))
# make a 4x4 image with 4 identical 2x2 subregions that flatten
# to -1, 0, 1, 2
im = afwImage.ImageF(bbox)
imArr = im.getArray()
imArr[:, :] = np.array(
((-1, 0, -1, 0), (1, 2, 1, 2), (-1, 0, -1, 0), (1, 2, 1, 2)),
dtype=imArr.dtype)
def castAndReshape(arr):
arr = np.array(arr, dtype=float)
arr.shape = numAmps
return arr
rowInds = castAndReshape(
(3, 2, 1,
0)) # avoid the trivial mapping to exercise more of the code
colIndOffsets = castAndReshape((0, 0, 1, 1))
detector = self.makeDetector(bbox=bbox,
numAmps=numAmps,
rowInds=rowInds,
colIndOffsets=colIndOffsets)
ampInfoCat = detector.getAmplifiers()
# note: table rows are reversed relative to amplifier order because
# rowInds is a descending ramp
table = np.array(
((7, 6, 5, 4), (1, 1, 1, 1), (5, 4, 3, 2), (0, 0, 0, 0)),
dtype=imArr.dtype)
llt = Linearizer(table=table, detector=detector)
lltRes = llt.applyLinearity(image=im, detector=detector)
self.assertEqual(lltRes.numOutOfRange, 2)
# amp 0 is a constant correction of 0; one image value is out of range,
# but it doesn't matter
imArr0 = im.Factory(im, ampInfoCat[0].getBBox()).getArray()
self.assertFloatsAlmostEqual(imArr0.flatten(), (-1, 0, 1, 2))
# amp 1 is a correction of (5, 4, 3, 2), but the first image value is
# under range
imArr1 = im.Factory(im, ampInfoCat[1].getBBox()).getArray()
self.assertFloatsAlmostEqual(imArr1.flatten(), (4, 5, 5, 5))
# amp 2 is a constant correction of +1; all image values are in range,
# but it doesn't matter
imArr2 = im.Factory(im, ampInfoCat[2].getBBox()).getArray()
self.assertFloatsAlmostEqual(imArr2.flatten(), (0, 1, 2, 3))
# amp 3 is a correction of (7, 6, 5, 4); all image values in range
imArr1 = im.Factory(im, ampInfoCat[3].getBBox()).getArray()
self.assertFloatsAlmostEqual(imArr1.flatten(), (6, 6, 6, 6))
def bypass_linearizer(self, datasetType, pythonType, butlerLocation, dataId):
"""Return a linearizer for the given detector.
On each call, a fresh instance of `Linearizer` is returned; the caller
is responsible for initializing it appropriately for the detector.
Parameters
----------
datasetType : `str``
The dataset type.
pythonType : `str` or `type`
Type of python object.
butlerLocation : `lsst.daf.persistence.ButlerLocation`
Struct-like class that holds information needed to persist and
retrieve an object using the LSST Persistence Framework.
dataId : `dict`
dataId passed to map location.
Returns
-------
Linearizer : `lsst.ip.isr.Linearizer`
Linearizer object for the given detector.
Notes
-----
Linearizers are not saved to persistent storage; rather, they are
managed entirely in memory. On each call, this function will return a
new instance of `Linearizer`, which must be managed (including setting
it up for use with a particular detector) by the caller. Calling
`bypass_linearizer` twice for the same detector will return
_different_ instances of `Linearizer`, which share no state.
"""
return Linearizer(detectorId=dataId.get('ccd', None))
def testGetLinearizer(self):
"""Test that we can get a linearizer"""
camera = self.butler.get("camera")
for ccdnum in (1, 62):
detector = camera[ccdnum]
linearizer = Linearizer(table=self.butler.get(
"linearizer", dataId=dict(ccdnum=ccdnum), immediate=True),
detector=detector)
for amp in detector:
self.assertEqual(linearizer.linearityType[amp.getName()],
LinearizeLookupTable.LinearityType)
linearizer.validate(detector=detector)
for badccdnum in (0, 63):
with self.assertRaises(Exception):
self.butler.get("linearizer",
dataId=dict(ccdnum=badccdnum),
immediate=True)
def testPickle(self):
"""!Test that a LinearizeLookupTable can be pickled and unpickled
"""
inImage = makeRampImage(bbox=self.bbox, start=-5, stop=2500)
table = self.makeTable(inImage)
llt = Linearizer(table=table, detector=self.detector)
refImage = inImage.Factory(inImage, True)
refNumOutOfRange = llt.applyLinearity(refImage, self.detector)
pickledStr = pickle.dumps(llt)
restoredLlt = pickle.loads(pickledStr)
measImage = inImage.Factory(inImage, True)
measNumOutOfRange = restoredLlt.applyLinearity(measImage,
self.detector)
self.assertEqual(refNumOutOfRange, measNumOutOfRange)
self.assertImagesAlmostEqual(refImage, measImage)
def testKnown(self):
"""!Test a few known values
"""
numAmps = (2, 2)
bbox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
lsst.geom.Extent2I(4, 4))
# make a 4x4 image with 4 identical 2x2 subregions that flatten
# to -1, 0, 1, 2
im = afwImage.ImageF(bbox)
imArr = im.getArray()
imArr[:, :] = np.array(
((-1, 0, -1, 0), (1, 2, 1, 2), (-1, 0, -1, 0), (1, 2, 1, 2)),
dtype=imArr.dtype)
sqCoeffs = np.array(((0, 0.11), (-0.15, -12)))
detector = self.makeDetector(bbox=bbox,
numAmps=numAmps,
sqCoeffs=sqCoeffs)
ampInfoCat = detector.getAmplifiers()
linSq = Linearizer(detector=detector)
linSq.applyLinearity(im, detector=detector)
# amp 0 has 0 squared coefficient and so makes no correction
imArr0 = im.Factory(im, ampInfoCat[0].getBBox()).getArray()
linCoeff0 = ampInfoCat[0].getLinearityCoeffs()[0]
self.assertEqual(0, linCoeff0)
self.assertFloatsAlmostEqual(imArr0.flatten(), (-1, 0, 1, 2))
# test all amps
for ampInfo in ampInfoCat:
imArr = im.Factory(im, ampInfo.getBBox()).getArray()
linCoeff = ampInfo.getLinearityCoeffs()[0]
expect = np.array(
(-1 + linCoeff, 0, 1 + linCoeff, 2 + 4 * linCoeff),
dtype=imArr.dtype)
self.assertFloatsAlmostEqual(imArr.flatten(), expect)
def testErrorHandling(self):
"""!Test error handling in LinearizeLookupTable
"""
image = makeRampImage(bbox=self.bbox, start=-5, stop=250)
table = self.makeTable(image)
llt = Linearizer(table=table, detector=self.detector)
# bad name
detBadName = self.makeDetector(detName="bad_detector_name")
with self.assertRaises(RuntimeError):
llt.applyLinearity(image, detBadName)
# bad serial
detBadSerial = self.makeDetector(detSerial="bad_detector_serial")
with self.assertRaises(RuntimeError):
llt.applyLinearity(image, detBadSerial)
# bad number of amplifiers
badNumAmps = (self.numAmps[0] - 1, self.numAmps[1])
detBadNumMaps = self.makeDetector(numAmps=badNumAmps)
with self.assertRaises(RuntimeError):
llt.applyLinearity(image, detBadNumMaps)
# bad linearity type
detBadLinType = self.makeDetector(linearityType="bad_linearity_type")
with self.assertRaises(RuntimeError):
llt.applyLinearity(image, detBadLinType)
# wrong dimension
badTable = table[..., np.newaxis]
with self.assertRaises(RuntimeError):
Linearizer(table=badTable, detector=self.detector)
# wrong size
badTable = np.transpose(table)
with self.assertRaises(RuntimeError):
Linearizer(table=badTable, detector=self.detector)
def run(self, inputPtc, camera, inputDims):
"""Fit non-linearity to PTC data, returning the correct Linearizer
object.
Parameters
----------
inputPtc : `lsst.cp.pipe.PtcDataset`
Pre-measured PTC dataset.
camera : `lsst.afw.cameraGeom.Camera`
Camera geometry.
inputDims : `lsst.daf.butler.DataCoordinate` or `dict`
DataIds to use to populate the output calibration.
Returns
-------
results : `lsst.pipe.base.Struct`
The results struct containing:
``outputLinearizer`` : `lsst.ip.isr.Linearizer`
Final linearizer calibration.
``outputProvenance`` : `lsst.ip.isr.IsrProvenance`
Provenance data for the new calibration.
Notes
-----
This task currently fits only polynomial-defined corrections,
where the correction coefficients are defined such that:
corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i)
These `c_i` are defined in terms of the direct polynomial fit:
meanVector ~ P(x=timeVector) = sum_j k_j x^j
such that c_(j-2) = -k_j/(k_1^j) in units of DN^(1-j) (c.f.,
Eq. 37 of 2003.05978). The `config.polynomialOrder` or
`config.splineKnots` define the maximum order of x^j to fit.
As k_0 and k_1 are degenerate with bias level and gain, they
are not included in the non-linearity correction.
"""
detector = camera[inputDims['detector']]
if self.config.linearityType == 'LookupTable':
table = np.zeros((len(detector), self.config.maxLookupTableAdu),
dtype=np.float32)
tableIndex = 0
else:
table = None
tableIndex = None # This will fail if we increment it.
if self.config.linearityType == 'Spline':
fitOrder = self.config.splineKnots
else:
fitOrder = self.config.polynomialOrder
# Initialize the linearizer.
linearizer = Linearizer(detector=detector, table=table, log=self.log)
for i, amp in enumerate(detector):
ampName = amp.getName()
if (len(inputPtc.expIdMask[ampName]) == 0):
self.log.warn(
f"Mask not found for {ampName} in non-linearity fit. Using all points."
)
mask = np.repeat(True, len(inputPtc.expIdMask[ampName]))
else:
mask = inputPtc.expIdMask[ampName]
inputAbscissa = np.array(inputPtc.rawExpTimes[ampName])[mask]
inputOrdinate = np.array(inputPtc.rawMeans[ampName])[mask]
# Determine proxy-to-linear-flux transformation
fluxMask = inputOrdinate < self.config.maxLinearAdu
lowMask = inputOrdinate > self.config.minLinearAdu
fluxMask = fluxMask & lowMask
linearAbscissa = inputAbscissa[fluxMask]
linearOrdinate = inputOrdinate[fluxMask]
linearFit, linearFitErr, chiSq, weights = irlsFit([0.0, 100.0],
linearAbscissa,
linearOrdinate,
funcPolynomial)
# Convert this proxy-to-flux fit into an expected linear flux
linearOrdinate = linearFit[0] + linearFit[1] * inputAbscissa
# Exclude low end outliers
threshold = self.config.nSigmaClipLinear * np.sqrt(linearOrdinate)
fluxMask = np.abs(inputOrdinate - linearOrdinate) < threshold
linearOrdinate = linearOrdinate[fluxMask]
fitOrdinate = inputOrdinate[fluxMask]
self.debugFit('linearFit', inputAbscissa, inputOrdinate,
linearOrdinate, fluxMask, ampName)
# Do fits
if self.config.linearityType in [
'Polynomial', 'Squared', 'LookupTable'
]:
polyFit = np.zeros(fitOrder + 1)
polyFit[1] = 1.0
polyFit, polyFitErr, chiSq, weights = irlsFit(
polyFit, linearOrdinate, fitOrdinate, funcPolynomial)
# Truncate the polynomial fit
k1 = polyFit[1]
linearityFit = [
-coeff / (k1**order) for order, coeff in enumerate(polyFit)
]
significant = np.where(
np.abs(linearityFit) > 1e-10, True, False)
self.log.info(f"Significant polynomial fits: {significant}")
modelOrdinate = funcPolynomial(polyFit, linearAbscissa)
self.debugFit('polyFit', linearAbscissa, fitOrdinate,
modelOrdinate, None, ampName)
if self.config.linearityType == 'Squared':
linearityFit = [linearityFit[2]]
elif self.config.linearityType == 'LookupTable':
# Use linear part to get time at wich signal is maxAduForLookupTableLinearizer DN
tMax = (self.config.maxLookupTableAdu -
polyFit[0]) / polyFit[1]
timeRange = np.linspace(0, tMax,
self.config.maxLookupTableAdu)
signalIdeal = polyFit[0] + polyFit[1] * timeRange
signalUncorrected = funcPolynomial(polyFit, timeRange)
lookupTableRow = signalIdeal - signalUncorrected # LinearizerLookupTable has correction
linearizer.tableData[tableIndex, :] = lookupTableRow
linearityFit = [tableIndex, 0]
tableIndex += 1
elif self.config.linearityType in ['Spline']:
# See discussion in `lsst.ip.isr.linearize.py` before modifying.
numPerBin, binEdges = np.histogram(linearOrdinate,
bins=fitOrder)
with np.errstate(invalid="ignore"):
# Algorithm note: With the counts of points per
# bin above, the next histogram calculates the
# values to put in each bin by weighting each
# point by the correction value.
values = np.histogram(
linearOrdinate,
bins=fitOrder,
weights=(inputOrdinate[fluxMask] -
linearOrdinate))[0] / numPerBin
# After this is done, the binCenters are
# calculated by weighting by the value we're
# binning over. This ensures that widely
# spaced/poorly sampled data aren't assigned to
# the midpoint of the bin (as could be done using
# the binEdges above), but to the weighted mean of
# the inputs. Note that both histograms are
# scaled by the count per bin to normalize what
# the histogram returns (a sum of the points
# inside) into an average.
binCenters = np.histogram(
linearOrdinate, bins=fitOrder,
weights=linearOrdinate)[0] / numPerBin
values = values[numPerBin > 0]
binCenters = binCenters[numPerBin > 0]
self.debugFit('splineFit', binCenters, np.abs(values), values,
None, ampName)
interp = afwMath.makeInterpolate(
binCenters.tolist(), values.tolist(),
afwMath.stringToInterpStyle("AKIMA_SPLINE"))
modelOrdinate = linearOrdinate + interp.interpolate(
linearOrdinate)
self.debugFit('splineFit', linearOrdinate, fitOrdinate,
modelOrdinate, None, ampName)
# If we exclude a lot of points, we may end up with
# less than fitOrder points. Pad out the low-flux end
# to ensure equal lengths.
if len(binCenters) != fitOrder:
padN = fitOrder - len(binCenters)
binCenters = np.pad(binCenters, (padN, 0),
'linear_ramp',
end_values=(binCenters.min() - 1.0, ))
# This stores the correction, which is zero at low values.
values = np.pad(values, (padN, 0))
# Pack the spline into a single array.
linearityFit = np.concatenate(
(binCenters.tolist(), values.tolist())).tolist()
polyFit = [0.0]
polyFitErr = [0.0]
chiSq = np.nan
else:
polyFit = [0.0]
polyFitErr = [0.0]
chiSq = np.nan
linearityFit = [0.0]
linearizer.linearityType[ampName] = self.config.linearityType
linearizer.linearityCoeffs[ampName] = np.array(linearityFit)
linearizer.linearityBBox[ampName] = amp.getBBox()
linearizer.fitParams[ampName] = np.array(polyFit)
linearizer.fitParamsErr[ampName] = np.array(polyFitErr)
linearizer.fitChiSq[ampName] = chiSq
image = afwImage.ImageF(len(inputOrdinate), 1)
image.getArray()[:, :] = inputOrdinate
linearizeFunction = linearizer.getLinearityTypeByName(
linearizer.linearityType[ampName])
linearizeFunction()(image, **{
'coeffs': linearizer.linearityCoeffs[ampName],
'table': linearizer.tableData,
'log': linearizer.log
})
linearizeModel = image.getArray()[0, :]
self.debugFit('solution', inputOrdinate[fluxMask], linearOrdinate,
linearizeModel[fluxMask], None, ampName)
linearizer.hasLinearity = True
linearizer.validate()
linearizer.updateMetadata(camera=camera,
detector=detector,
filterName='NONE')
linearizer.updateMetadata(setDate=True, setCalibId=True)
provenance = IsrProvenance(calibType='linearizer')
return pipeBase.Struct(
outputLinearizer=linearizer,
outputProvenance=provenance,
)
def makeLinearizer(self, linearityType, bbox=None):
"""Construct a linearizer with the test coefficients.
Parameters
----------
linearityType : `str`
Type of linearity to use. The coefficients are set by the
setUp method.
bbox : `lsst.geom.Box2I`
Bounding box for the full detector. Used to assign
amp-based bounding boxes.
Returns
-------
linearizer : `lsst.ip.isr.Linearizer`
A fully constructed, persistable linearizer.
"""
bbox = bbox if bbox is not None else self.bbox
numAmps = self.ampArrangement
boxArr = BoxGrid(box=bbox, numColRow=numAmps)
linearizer = Linearizer()
linearizer.hasLinearity = True
for i in range(numAmps[0]):
for j in range(numAmps[1]):
ampName = f"amp {i+1}_{j+1}"
ampBox = boxArr[i, j]
linearizer.ampNames.append(ampName)
if linearityType == 'Squared':
linearizer.linearityCoeffs[ampName] = np.array(
[self.sqCoeffs[i, j]])
elif linearityType == 'LookupTable':
linearizer.linearityCoeffs[ampName] = np.array(
self.lookupIndices[i, j])
linearizer.tableData = self.table
elif linearityType == 'Polynomial':
linearizer.linearityCoeffs[ampName] = np.array(
self.polyCoeffs[i, j])
elif linearityType == 'Spline':
linearizer.linearityCoeffs[ampName] = np.array(
self.splineCoeffs)
linearizer.linearityType[ampName] = linearityType
linearizer.linearityBBox[ampName] = ampBox
linearizer.fitParams[ampName] = np.array([])
linearizer.fitParamsErr[ampName] = np.array([])
linearizer.fitChiSq[ampName] = np.nan
linearizer.fitResiduals[ampName] = np.array([])
linearizer.linearFit[ampName] = np.array([])
return linearizer
def testBasics(self):
"""Test basic linearization functionality.
"""
for imageClass in (afwImage.ImageF, afwImage.ImageD):
inImage = makeRampImage(bbox=self.bbox,
start=-5,
stop=2500,
imageClass=imageClass)
for linearityType in ('Squared', 'LookupTable', 'Polynomial',
'Spline'):
detector = self.makeDetector(linearityType)
table = None
inputData = {
'Squared': self.sqCoeffs,
'LookupTable': self.lookupIndices,
'Polynomial': self.polyCoeffs,
'Spline': self.splineCoeffs
}[linearityType]
if linearityType == 'LookupTable':
table = np.array(self.table,
dtype=inImage.getArray().dtype)
linearizer = Linearizer(detector=detector, table=table)
measImage = inImage.Factory(inImage, True)
result = linearizer.applyLinearity(measImage,
detector=detector,
log=self.log)
refImage, refNumOutOfRange = referenceImage(
inImage.Factory(inImage, True), detector, linearityType,
inputData, table)
# This is necessary for the same tests to be used on
# all types. The first amplifier has 0.0 for the
# coefficient, which should be tested (it has a log
# message), but we are not linearizing an amplifier
# with no correction, so it fails the test that
# numLinearized == numAmps.
zeroLinearity = 1 if linearityType == 'Squared' else 0
self.compareResults(measImage, result.numOutOfRange,
result.numLinearized, result.numAmps,
refImage, refNumOutOfRange,
self.numAmps - zeroLinearity, self.numAmps)
# Test a stand alone linearizer. This ignores validate checks.
measImage = inImage.Factory(inImage, True)
storedLinearizer = self.makeLinearizer(linearityType)
storedResult = storedLinearizer.applyLinearity(measImage,
log=self.log)
self.compareResults(measImage, storedResult.numOutOfRange,
storedResult.numLinearized,
storedResult.numAmps, refImage,
refNumOutOfRange,
self.numAmps - zeroLinearity, self.numAmps)
# "Save to yaml" and test again
storedDict = storedLinearizer.toDict()
storedLinearizer = Linearizer().fromDict(storedDict)
measImage = inImage.Factory(inImage, True)
storedLinearizer = self.makeLinearizer(linearityType)
storedResult = storedLinearizer.applyLinearity(measImage,
log=self.log)
self.compareResults(measImage, storedResult.numOutOfRange,
storedResult.numLinearized,
storedResult.numAmps, refImage,
refNumOutOfRange,
self.numAmps - zeroLinearity, self.numAmps)
# "Save to fits" and test again
storedTable = storedLinearizer.toTable()
storedLinearizer = Linearizer().fromTable(storedTable)
measImage = inImage.Factory(inImage, True)
storedLinearizer = self.makeLinearizer(linearityType)
storedResult = storedLinearizer.applyLinearity(measImage,
log=self.log)
self.compareResults(measImage, storedResult.numOutOfRange,
storedResult.numLinearized,
storedResult.numAmps, refImage,
refNumOutOfRange,
self.numAmps - zeroLinearity, self.numAmps)