def setUp(self):
self.defaultConfig = cpPipe.ptc.MeasurePhotonTransferCurveTask.ConfigClass()
self.defaultTask = cpPipe.ptc.MeasurePhotonTransferCurveTask(config=self.defaultConfig)
self.defaultConfigExtract = cpPipe.ptc.PhotonTransferCurveExtractTask.ConfigClass()
self.defaultTaskExtract = cpPipe.ptc.PhotonTransferCurveExtractTask(config=self.defaultConfigExtract)
self.defaultConfigSolve = cpPipe.ptc.PhotonTransferCurveSolveTask.ConfigClass()
self.defaultTaskSolve = cpPipe.ptc.PhotonTransferCurveSolveTask(config=self.defaultConfigSolve)
self.flatMean = 2000
self.readNoiseAdu = 10
mockImageConfig = isrMock.IsrMock.ConfigClass()
# flatDrop is not really relevant as we replace the data
# but good to note it in case we change how this image is made
mockImageConfig.flatDrop = 0.99999
mockImageConfig.isTrimmed = True
self.flatExp1 = isrMock.FlatMock(config=mockImageConfig).run()
self.flatExp2 = self.flatExp1.clone()
(shapeY, shapeX) = self.flatExp1.getDimensions()
self.flatWidth = np.sqrt(self.flatMean) + self.readNoiseAdu
self.rng1 = np.random.RandomState(1984)
flatData1 = self.rng1.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
self.rng2 = np.random.RandomState(666)
flatData2 = self.rng2.normal(self.flatMean, self.flatWidth, (shapeX, shapeY))
self.flatExp1.image.array[:] = flatData1
self.flatExp2.image.array[:] = flatData2
# create fake PTC data to see if fit works, for one amp ('amp')
self.flux = 1000. # ADU/sec
self.timeVec = np.arange(1., 101.)
self.k2NonLinearity = -5e-6
# quadratic signal-chain non-linearity
muVec = self.flux*self.timeVec + self.k2NonLinearity*self.timeVec**2
self.gain = 1.5 # e-/ADU
self.c1 = 1./self.gain
self.noiseSq = 5*self.gain # 7.5 (e-)^2
self.a00 = -1.2e-6
self.c2 = -1.5e-6
self.c3 = -4.7e-12 # tuned so that it turns over for 200k mean
self.ampNames = [amp.getName() for amp in self.flatExp1.getDetector().getAmplifiers()]
self.dataset = PhotonTransferCurveDataset(self.ampNames, " ") # pack raw data for fitting
for ampName in self.ampNames: # just the expTimes and means here - vars vary per function
self.dataset.rawExpTimes[ampName] = self.timeVec
self.dataset.rawMeans[ampName] = muVec
def test_ptcDatset(self):
# Fill the set up with made up data.
nSignalPoints = 5
nSideCovMatrix = 2
for fitType in ['POLYNOMIAL', 'EXPAPPROXIMATION', 'FULLCOVARIANCE']:
localDataset = PhotonTransferCurveDataset(self.ampNames, " ",
nSideCovMatrix)
localDataset.ptcFitType = fitType
localDataset.badAmps = [
localDataset.ampNames[0], localDataset.ampNames[1]
]
for ampName in localDataset.ampNames:
localDataset.inputExpIdPairs[ampName] = np.repeat(
1, nSignalPoints).tolist()
localDataset.expIdMask[ampName] = [
True, False, True, True, False, True, False, True, True,
True, True, False, True, False, True
]
localDataset.rawExpTimes[ampName] = np.arange(
nSignalPoints).tolist()
localDataset.rawMeans[ampName] = np.array(
self.flux * np.arange(nSignalPoints)).tolist()
localDataset.rawVars[ampName] = np.array(
self.c1 * self.flux * np.arange(nSignalPoints)).tolist()
localDataset.photoCharge[ampName] = np.repeat(
np.nan, nSignalPoints).tolist()
localDataset.gain[ampName] = self.gain
localDataset.gainErr[ampName] = 0.1
localDataset.noise[ampName] = self.noiseSq
localDataset.noiseErr[ampName] = 2.0
localDataset.finalVars[ampName] = np.array(
self.c1 * self.flux * np.arange(nSignalPoints)).tolist()
localDataset.finalModelVars[ampName] = np.repeat(
100.0, nSignalPoints).tolist()
localDataset.finalMeans[ampName] = np.array(
self.flux * np.arange(nSignalPoints)).tolist()
if fitType in [
'POLYNOMIAL',
'EXPAPPROXIMATION',
]:
localDataset.ptcFitPars[ampName] = np.array(
[10.0, 1.5, 1e-6]).tolist()
localDataset.ptcFitParsError[ampName] = np.array(
[1.0, 0.2, 1e-7]).tolist()
localDataset.ptcFitChiSq[ampName] = 1.0
localDataset.covariances[ampName] = np.full(
(nSignalPoints, nSideCovMatrix, nSideCovMatrix),
np.nan).tolist()
localDataset.covariancesModel[ampName] = np.full(
(nSignalPoints, nSideCovMatrix, nSideCovMatrix),
np.nan).tolist()
localDataset.covariancesSqrtWeights[ampName] = np.full(
(nSignalPoints, nSideCovMatrix, nSideCovMatrix),
np.nan).tolist()
localDataset.aMatrix[ampName] = np.full(
(nSideCovMatrix, nSideCovMatrix), np.nan).tolist()
localDataset.bMatrix[ampName] = np.full(
(nSideCovMatrix, nSideCovMatrix), np.nan).tolist()
localDataset.covariancesModelNoB[ampName] = np.full(
(nSignalPoints, nSideCovMatrix, nSideCovMatrix),
np.nan).tolist()
localDataset.aMatrixNoB[ampName] = np.full(
(nSideCovMatrix, nSideCovMatrix), np.nan).tolist()
if localDataset.ptcFitType in [
'FULLCOVARIANCE',
]:
localDataset.ptcFitPars[ampName] = np.array(
[np.nan, np.nan]).tolist()
localDataset.ptcFitParsError[ampName] = np.array(
[np.nan, np.nan]).tolist()
localDataset.ptcFitChiSq[ampName] = np.array(
[np.nan, np.nan]).tolist()
localDataset.covariances[ampName] = np.full(
(nSignalPoints, nSideCovMatrix, nSideCovMatrix),
105.0).tolist()
localDataset.covariancesModel[ampName] = np.full(
(nSignalPoints, nSideCovMatrix, nSideCovMatrix),
100.0).tolist()
localDataset.covariancesSqrtWeights[ampName] = np.full(
(nSignalPoints, nSideCovMatrix, nSideCovMatrix),
10.0).tolist()
localDataset.aMatrix[ampName] = np.full(
(nSideCovMatrix, nSideCovMatrix), 1e-6).tolist()
localDataset.bMatrix[ampName] = np.full(
(nSideCovMatrix, nSideCovMatrix), 1e-7).tolist()
localDataset.covariancesModelNoB[ampName] = np.full(
(nSignalPoints, nSideCovMatrix, nSideCovMatrix),
15.0).tolist()
localDataset.aMatrixNoB[ampName] = np.full(
(nSideCovMatrix, nSideCovMatrix), 2e-6).tolist()
filename = tempfile.mktemp()
usedFilename = localDataset.writeText(filename + ".yaml")
fromText = PhotonTransferCurveDataset.readText(usedFilename)
self.assertEqual(localDataset, fromText)
filename = tempfile.mktemp()
usedFilename = localDataset.writeFits(filename + ".fits")
fromFits = PhotonTransferCurveDataset.readFits(usedFilename)
self.assertEqual(localDataset, fromFits)
def run(self, inputExp, inputDims):
"""Measure covariances from difference of flat pairs
Parameters
----------
inputExp : `dict` [`float`,
(`~lsst.afw.image.exposure.exposure.ExposureF`,
`~lsst.afw.image.exposure.exposure.ExposureF`, ...,
`~lsst.afw.image.exposure.exposure.ExposureF`)]
Dictionary that groups flat-field exposures that have the same
exposure time (seconds).
inputDims : `list`
List of exposure IDs.
"""
# inputExp.values() returns a view, which we turn into a list. We then
# access the first exposure to get teh detector.
detector = list(inputExp.values())[0][0].getDetector()
detNum = detector.getId()
amps = detector.getAmplifiers()
ampNames = [amp.getName() for amp in amps]
maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames}
minMeanSignalDict = {ampName: 0.0 for ampName in ampNames}
for ampName in ampNames:
if 'ALL_AMPS' in self.config.maxMeanSignal:
maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS']
elif ampName in self.config.maxMeanSignal:
maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName]
if 'ALL_AMPS' in self.config.minMeanSignal:
minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS']
elif ampName in self.config.minMeanSignal:
minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName]
tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'),
('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')]
dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY',
self.config.maximumRangeCovariancesAstier)
covArray = [np.full((self.config.maximumRangeCovariancesAstier,
self.config.maximumRangeCovariancesAstier), np.nan)]
for ampName in ampNames:
dummyPtcDataset.rawExpTimes[ampName] = [np.nan]
dummyPtcDataset.rawMeans[ampName] = [np.nan]
dummyPtcDataset.rawVars[ampName] = [np.nan]
dummyPtcDataset.inputExpIdPairs[ampName] = [(np.nan, np.nan)]
dummyPtcDataset.expIdMask[ampName] = [np.nan]
dummyPtcDataset.covariances[ampName] = covArray
dummyPtcDataset.covariancesModel[ampName] = np.full_like(covArray, np.nan)
dummyPtcDataset.covariancesSqrtWeights[ampName] = np.full_like(covArray, np.nan)
dummyPtcDataset.covariancesModelNoB[ampName] = np.full_like(covArray, np.nan)
dummyPtcDataset.aMatrix[ampName] = np.full_like(covArray[0], np.nan)
dummyPtcDataset.bMatrix[ampName] = np.full_like(covArray[0], np.nan)
dummyPtcDataset.aMatrixNoB[ampName] = np.full_like(covArray[0], np.nan)
dummyPtcDataset.ptcFitPars[ampName] = [np.nan]
dummyPtcDataset.ptcFitParsError[ampName] = [np.nan]
dummyPtcDataset.ptcFitChiSq[ampName] = np.nan
dummyPtcDataset.finalVars[ampName] = [np.nan]
dummyPtcDataset.finalModelVars[ampName] = [np.nan]
dummyPtcDataset.finalMeans[ampName] = [np.nan]
# Output list with PTC datasets.
partialDatasetPtcList = []
# The number of output references needs to match that of input references:
# initialize outputlist with dummy PTC datasets.
for i in range(len(inputDims)):
partialDatasetPtcList.append(dummyPtcDataset)
for expTime in inputExp:
exposures = inputExp[expTime]
if len(exposures) == 1:
self.log.warn(f"Only one exposure found at expTime {expTime}. Dropping exposure "
f"{exposures[0].getInfo().getVisitInfo().getExposureId()}.")
continue
else:
# Only use the first two exposures at expTime
exp1, exp2 = exposures[0], exposures[1]
if len(exposures) > 2:
self.log.warn(f"Already found 2 exposures at expTime {expTime}. "
"Ignoring exposures: "
f"{i.getInfo().getVisitInfo().getExposureId() for i in exposures[2:]}")
expId1 = exp1.getInfo().getVisitInfo().getExposureId()
expId2 = exp2.getInfo().getVisitInfo().getExposureId()
nAmpsNan = 0
partialDatasetPtc = PhotonTransferCurveDataset(ampNames, '',
self.config.maximumRangeCovariancesAstier)
for ampNumber, amp in enumerate(detector):
ampName = amp.getName()
# covAstier: [(i, j, var (cov[0,0]), cov, npix) for (i,j) in {maxLag, maxLag}^2]
doRealSpace = self.config.covAstierRealSpace
if self.config.detectorMeasurementRegion == 'AMP':
region = amp.getBBox()
elif self.config.detectorMeasurementRegion == 'FULL':
region = None
# The variable `covAstier` is of the form: [(i, j, var (cov[0,0]), cov, npix) for (i,j)
# in {maxLag, maxLag}^2]
muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=region,
covAstierRealSpace=doRealSpace)
expIdMask = True
if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None):
msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1},"
f" {expId2} of detector {detNum}.")
self.log.warn(msg)
nAmpsNan += 1
expIdMask = False
covArray = np.full((1, self.config.maximumRangeCovariancesAstier,
self.config.maximumRangeCovariancesAstier), np.nan)
covSqrtWeights = np.full_like(covArray, np.nan)
if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]):
expIdMask = False
partialDatasetPtc.rawExpTimes[ampName] = [expTime]
partialDatasetPtc.rawMeans[ampName] = [muDiff]
partialDatasetPtc.rawVars[ampName] = [varDiff]
if covAstier is not None:
tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime,
ampName) for covRow in covAstier]
tempStructArray = np.array(tupleRows, dtype=tags)
covArray, vcov, _ = makeCovArray(tempStructArray,
self.config.maximumRangeCovariancesAstier)
covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov))
partialDatasetPtc.inputExpIdPairs[ampName] = [(expId1, expId2)]
partialDatasetPtc.expIdMask[ampName] = [expIdMask]
partialDatasetPtc.covariances[ampName] = covArray
partialDatasetPtc.covariancesSqrtWeights[ampName] = covSqrtWeights
partialDatasetPtc.covariancesModel[ampName] = np.full_like(covArray, np.nan)
partialDatasetPtc.covariancesModelNoB[ampName] = np.full_like(covArray, np.nan)
partialDatasetPtc.aMatrix[ampName] = np.full_like(covArray[0], np.nan)
partialDatasetPtc.bMatrix[ampName] = np.full_like(covArray[0], np.nan)
partialDatasetPtc.aMatrixNoB[ampName] = np.full_like(covArray[0], np.nan)
partialDatasetPtc.ptcFitPars[ampName] = [np.nan]
partialDatasetPtc.ptcFitParsError[ampName] = [np.nan]
partialDatasetPtc.ptcFitChiSq[ampName] = np.nan
partialDatasetPtc.finalVars[ampName] = [np.nan]
partialDatasetPtc.finalModelVars[ampName] = [np.nan]
partialDatasetPtc.finalMeans[ampName] = [np.nan]
# Use location of exp1 to save PTC dataset from (exp1, exp2) pair.
# expId1 and expId2, as returned by getInfo().getVisitInfo().getExposureId(),
# and the exposure IDs stured in inoutDims,
# may have the zero-padded detector number appended at
# the end (in gen3). A temporary fix is to consider expId//1000 and/or
# inputDims//1000.
# Below, np.where(expId1 == np.array(inputDims)) (and the other analogous
# comparisons) returns a tuple with a single-element array, so [0][0]
# is necessary to extract the required index.
try:
datasetIndex = np.where(expId1 == np.array(inputDims))[0][0]
except IndexError:
try:
datasetIndex = np.where(expId1//1000 == np.array(inputDims))[0][0]
except IndexError:
datasetIndex = np.where(expId1//1000 == np.array(inputDims)//1000)[0][0]
partialDatasetPtcList[datasetIndex] = partialDatasetPtc
if nAmpsNan == len(ampNames):
msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}."
self.log.warn(msg)
return pipeBase.Struct(
outputCovariances=partialDatasetPtcList,
)
def run(self, inputCovariances, camera=None, inputExpList=None):
"""Fit measure covariances to different models.
Parameters
----------
inputCovariances : `list` [`lsst.ip.isr.PhotonTransferCurveDataset`]
List of lsst.ip.isr.PhotonTransferCurveDataset datasets.
camera : `lsst.afw.cameraGeom.Camera`, optional
Input camera.
inputExpList : `list` [`~lsst.afw.image.exposure.exposure.ExposureF`], optional
List of exposures.
Returns
-------
results : `lsst.pipe.base.Struct`
The results struct containing:
``outputPtcDatset`` : `lsst.ip.isr.PhotonTransferCurveDataset`
Final PTC dataset, containing information such as the means, variances,
and exposure times.
"""
# Assemble partial PTC datasets into a single dataset.
ampNames = np.unique(inputCovariances[0].ampNames)
datasetPtc = PhotonTransferCurveDataset(
ampNames, self.config.ptcFitType,
self.config.maximumRangeCovariancesAstier)
for partialPtcDataset in inputCovariances:
if partialPtcDataset.ptcFitType == 'DUMMY':
continue
for ampName in ampNames:
datasetPtc.inputExpIdPairs[ampName].append(
partialPtcDataset.inputExpIdPairs[ampName])
if type(partialPtcDataset.rawExpTimes[ampName]) is list:
datasetPtc.rawExpTimes[ampName].append(
partialPtcDataset.rawExpTimes[ampName][0])
else:
datasetPtc.rawExpTimes[ampName].append(
partialPtcDataset.rawExpTimes[ampName])
if type(partialPtcDataset.rawMeans[ampName]) is list:
datasetPtc.rawMeans[ampName].append(
partialPtcDataset.rawMeans[ampName][0])
else:
datasetPtc.rawMeans[ampName].append(
partialPtcDataset.rawMeans[ampName])
if type(partialPtcDataset.rawVars[ampName]) is list:
datasetPtc.rawVars[ampName].append(
partialPtcDataset.rawVars[ampName][0])
else:
datasetPtc.rawVars[ampName].append(
partialPtcDataset.rawVars[ampName])
datasetPtc.covariances[ampName].append(
np.array(partialPtcDataset.covariances[ampName][0]))
datasetPtc.covariancesSqrtWeights[ampName].append(
np.array(
partialPtcDataset.covariancesSqrtWeights[ampName][0]))
# Sort arrays that are filled so far in the final dataset by rawMeans index
for ampName in ampNames:
index = np.argsort(np.ravel(np.array(
datasetPtc.rawMeans[ampName])))
datasetPtc.inputExpIdPairs[ampName] = np.array(
datasetPtc.inputExpIdPairs[ampName])[index]
datasetPtc.rawExpTimes[ampName] = np.array(
datasetPtc.rawExpTimes[ampName])[index]
datasetPtc.rawMeans[ampName] = np.array(
datasetPtc.rawMeans[ampName])[index]
datasetPtc.rawVars[ampName] = np.array(
datasetPtc.rawVars[ampName])[index]
datasetPtc.covariances[ampName] = np.array(
datasetPtc.covariances[ampName])[index]
datasetPtc.covariancesSqrtWeights[ampName] = np.array(
datasetPtc.covariancesSqrtWeights[ampName])[index]
if self.config.ptcFitType == "FULLCOVARIANCE":
# Calculate covariances and fit them, including the PTC, to Astier+19 full model (Eq. 20)
# First, fit get the flat pairs that are masked, fitting C_00 vs mu to
# the EXPAPPROXIMATION model (Eq. 16 in Astier+19).
# The points at these fluxes will also be masked when calculating the other covariances, C_ij)
tempDatasetPtc = copy.copy(datasetPtc)
tempDatasetPtc.ptcFitType = "EXPAPPROXIMATION"
tempDatasetPtc = self.fitPtc(tempDatasetPtc)
for ampName in datasetPtc.ampNames:
datasetPtc.expIdMask[ampName] = tempDatasetPtc.expIdMask[
ampName]
datasetPtc.fitType = "FULLCOVARIANCE"
datasetPtc = self.fitCovariancesAstier(datasetPtc)
# The other options are: self.config.ptcFitType in ("EXPAPPROXIMATION", "POLYNOMIAL")
else:
# Fit the PTC to a polynomial or to Astier+19 exponential approximation (Eq. 16).
# Fill up PhotonTransferCurveDataset object.
datasetPtc = self.fitPtc(datasetPtc)
if inputExpList is not None:
# It should be a list of exposures, to get the detector.
detector = inputExpList[0].getDetector()
else:
detector = None
datasetPtc.updateMetadata(setDate=True,
camera=camera,
detector=detector)
return pipeBase.Struct(outputPtcDataset=datasetPtc, )
def test_generalBehaviour(self):
test = PhotonTransferCurveDataset(['C00', 'C01'], " ")
test.inputExpIdPairs = {'C00': [(123, 234), (345, 456), (567, 678)],
'C01': [(123, 234), (345, 456), (567, 678)]}
def setUp(self):
self.ptcData = PhotonTransferCurveDataset(['C00', 'C01'], " ")
self.ptcData.inputExpIdPairs = {'C00': [(123, 234), (345, 456), (567, 678)],
'C01': [(123, 234), (345, 456), (567, 678)]}