def makeMinimalIsrConfig(self):
"""Return an IsrConfig with all boolean flags disabled"""
isrConfig = IsrTask.ConfigClass()
for name in isrConfig:
if name.startswith("do"):
setattr(isrConfig, name, False)
return isrConfig
def runIsr():
'''Run the task to do ISR on a ccd'''
#Create the isr task with modified config
isrConfig = IsrTask.ConfigClass()
isrConfig.doBias = False #We didn't make a zero frame
isrConfig.doDark = True
isrConfig.doFlat = True
isrConfig.doFringe = False #There is no fringe frame for this example
isrConfig.assembleCcd.setGain = False
isrTask = IsrTask(config=isrConfig)
#Make raw, flat and dark exposures
DARKVAL = 2. #e-/sec
OSCAN = 1000. #DN
GRADIENT = .10
EXPTIME = 15 #seconds
DARKEXPTIME = 40. #seconds
darkExposure = exampleUtils.makeDark(DARKVAL, DARKEXPTIME)
flatExposure = exampleUtils.makeFlat(GRADIENT)
rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME)
output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure)
return output.exposure
def testTaskAPI(self):
"""Test that the Tasks work
Checks both MeasureCrosstalkTask and the CrosstalkTask.
"""
# make exposure available to NullIsrTask
# without NullIsrTask's `self` hiding this test class's `self`
exposure = self.exposure
class NullIsrTask(IsrTask):
def runDataRef(self, dataRef):
return Struct(exposure=exposure)
config = MeasureCrosstalkTask.ConfigClass()
config.isr.retarget(NullIsrTask)
config.threshold = self.value - 1
measure = MeasureCrosstalkTask(config=config)
fakeDataRef = Struct(dataId={'fake': 1})
coeff, coeffErr, coeffNum = measure.reduce(
[measure.runDataRef(fakeDataRef)])
self.checkCoefficients(coeff, coeffErr, coeffNum)
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
isr.crosstalk.run(self.exposure)
self.checkSubtracted(self.exposure)
def processExposure(exposure,
bias=None,
defects=None,
repair=False,
repo=None):
from lsst.ip.isr import IsrTask
import lsst.daf.persistence as dafPersist
import time
config = IsrTask.ConfigClass()
if (bias == None or defects == None) and repo == None:
raise (AttributeError(
'Repo keyword not set and is required to find bias/defect frames'))
else:
butler = dafPersist.Butler(repo)
if False:
config.overscanFitType = "AKIMA_SPLINE"
config.overscanOrder = 5
else:
config.overscanFitType = "POLY"
config.overscanOrder = 3
config.doBias = True
if config.doBias == True and bias == None:
bias = butler.get(
'bias',
visit=exposure.getInfo().getVisitInfo().getExposureId(),
)
config.doDark = False
config.doFringe = False
config.doFlat = False
config.doLinearize = False
# TODO: should be able to run this but there are issues with defects in the stack at the moment
config.doDefect = True if repair == True else False
if (config.doDefect == True and bias == None):
butler = dafPersist.Butler(repo)
defects = butler.get(
'defects',
visit=exposure.getInfo().getVisitInfo().getExposureId(),
)
config.doAddDistortionModel = False
config.doSaturationInterpolation = False
config.overscanNumLeadingColumnsToSkip = 20
isrTask = IsrTask(config=config)
# Run ISR
start = time.time()
isr_corrected_exposure = isrTask.run(exposure, bias=bias,
defects=defects).exposure
end = time.time()
print('Time to perform image ISR was {0:2f} [s]'.format(end - start))
return (isr_corrected_exposure)
def testTaskAPI(self):
"""Test that the Tasks work
Checks both MeasureCrosstalkTask and the CrosstalkTask.
"""
# make exposure available to NullIsrTask
# without NullIsrTask's `self` hiding this test class's `self`
exposure = self.exposure
class NullIsrTask(IsrTask):
def runDataRef(self, dataRef):
return Struct(exposure=exposure)
coeff = np.array(self.crosstalk).transpose()
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
calib = CrosstalkCalib().fromDetector(self.exposure.getDetector(),
coeffVector=coeff)
isr.crosstalk.run(self.exposure, crosstalk=calib)
self.checkSubtracted(self.exposure)
def test_interChip(self):
"""Test that passing an external exposure as the crosstalk source
works.
"""
exposure = self.exposure
ctSources = [self.ctSource]
coeff = np.array(self.crosstalk).transpose()
calib = CrosstalkCalib().fromDetector(exposure.getDetector(),
coeffVector=coeff)
# Now convert this into zero intra-chip, full inter-chip:
calib.interChip['detector 2'] = coeff
calib.coeffs = np.zeros_like(coeff)
# Process and check as above
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
isr.crosstalk.run(exposure,
crosstalk=calib,
crosstalkSources=ctSources)
self.checkSubtracted(exposure)
def __init__(self, *, config=None):
# programatically clone all of the connections from isrTask
# settin minimum values to zero for everything except the ccdExposure
super().__init__(config=IsrTask.ConfigClass(
)) # need a dummy config, isn't used other than for ctor
for name, connection in self.allConnections.items():
if hasattr(connection, 'minimum'):
args = _getArgs(connection)
if name != "ccdExposure": # need one input image always
args['minimum'] = 0
newConnection = type(connection)(**args)
self.allConnections[name] = newConnection
setattr(self, name, newConnection)
exposure = cT.Output( # called just "exposure" to mimic isrTask's return struct
name="quickLookExp",
doc="The quickLook output exposure.",
storageClass="ExposureF",
dimensions=("instrument", "exposure", "detector"),
)
# set like this to make it explicit that the outputExposure
# and the exposure are identical. The only reason there are two is for
# API compatibility.
self.outputExposure = exposure
def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, **kwargs):
"""Get the postIsr and cosmic-repaired image for this dataId.
Parameters
----------
expIdOrDataId : `dict`
The dataId
extraIsrOptions : `dict`, optional
extraIsrOptions is a dict of extra isr options applied to this
image only.
skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK!
Skip doing cosmic ray repair for this image?
Returns
-------
exp : `lsst.afw.image.Exposure`
The postIsr exposure
"""
dataId = self._parseExpIdOrDataId(expIdOrDataId, **kwargs)
try:
exp = self.butler.get(self._datasetName, dataId=dataId)
self.log.info("Found a ready-made quickLookExp in the repo. Returning that.")
return exp
except LookupError:
pass
try:
raw = self.butler.get('raw', dataId=dataId)
except LookupError:
raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None
# default options that are probably good for most engineering time
isrConfig = IsrTask.ConfigClass()
isrConfig.doWrite = False # this task writes separately, no need for this
isrConfig.doSaturation = True # saturation very important for roundness measurement in qfm
isrConfig.doSaturationInterpolation = True
isrConfig.overscanNumLeadingColumnsToSkip = 5
isrConfig.overscan.fitType = 'MEDIAN_PER_ROW'
# apply general overrides
self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions)
# apply per-image overrides
self._applyConfigOverrides(isrConfig, extraIsrOptions)
isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel',
'bfGains', 'ptc']
isrDict = {}
# we build a cache of all the isr components which will be used to save
# the IO time on subsequent calls. This assumes people will not update
# calibration products while this object lives, but this is a fringe
# use case, and if they do, all they would need to do would be call
# .clearCache() and this will rebuild with the new products.
for component in isrParts:
if component in self._cache and component != 'flat':
self.log.info(f"Using {component} from cache...")
isrDict[component] = self._cache[component]
continue
try:
# TODO: add caching for flats
item = self.butler.get(component, dataId=dataId)
self._cache[component] = item
isrDict[component] = self._cache[component]
except (RuntimeError, LookupError, OperationalError):
pass
quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure
if self.doWrite:
try:
self.butler.put(quickLookExp, self._datasetName, dataId)
self.log.info(f'Put {self._datasetName} for {dataId}')
except ConflictingDefinitionError:
# TODO: DM-34302 fix this message so that it's less scary for
# users. Do this by having daemons know they're daemons.
self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)')
pass
return quickLookExp
def run(self,
ccdExposure,
*,
camera=None,
bias=None,
dark=None,
flat=None,
defects=None,
linearizer=None,
crosstalk=None,
bfKernel=None,
bfGains=None,
ptc=None,
crosstalkSources=None,
isrBaseConfig=None):
"""Run isr and cosmic ray repair using, doing as much isr as possible.
Retrieves as many calibration products as are available, and runs isr
with those settings enabled, but always returns an assembled image at
a minimum. Then performs cosmic ray repair if configured to.
Parameters
----------
ccdExposure : `lsst.afw.image.Exposure`
The raw exposure that is to be run through ISR. The
exposure is modified by this method.
camera : `lsst.afw.cameraGeom.Camera`, optional
The camera geometry for this exposure. Required if
one or more of ``ccdExposure``, ``bias``, ``dark``, or
``flat`` does not have an associated detector.
bias : `lsst.afw.image.Exposure`, optional
Bias calibration frame.
linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
Functor for linearization.
crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
Calibration for crosstalk.
crosstalkSources : `list`, optional
List of possible crosstalk sources.
dark : `lsst.afw.image.Exposure`, optional
Dark calibration frame.
flat : `lsst.afw.image.Exposure`, optional
Flat calibration frame.
ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
Photon transfer curve dataset, with, e.g., gains
and read noise.
bfKernel : `numpy.ndarray`, optional
Brighter-fatter kernel.
bfGains : `dict` of `float`, optional
Gains used to override the detector's nominal gains for the
brighter-fatter correction. A dict keyed by amplifier name for
the detector in question.
defects : `lsst.ip.isr.Defects`, optional
List of defects.
fringes : `lsst.pipe.base.Struct`, optional
Struct containing the fringe correction data, with
elements:
- ``fringes``: fringe calibration frame (`afw.image.Exposure`)
- ``seed``: random seed derived from the ccdExposureId for random
number generator (`uint32`)
opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
A ``TransmissionCurve`` that represents the throughput of the,
optics, to be evaluated in focal-plane coordinates.
filterTransmission : `lsst.afw.image.TransmissionCurve`
A ``TransmissionCurve`` that represents the throughput of the
filter itself, to be evaluated in focal-plane coordinates.
sensorTransmission : `lsst.afw.image.TransmissionCurve`
A ``TransmissionCurve`` that represents the throughput of the
sensor itself, to be evaluated in post-assembly trimmed detector
coordinates.
atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
A ``TransmissionCurve`` that represents the throughput of the
atmosphere, assumed to be spatially constant.
detectorNum : `int`, optional
The integer number for the detector to process.
strayLightData : `object`, optional
Opaque object containing calibration information for stray-light
correction. If `None`, no correction will be performed.
illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
Illumination correction image.
isrBaseConfig : `lsst.ip.isr.IsrTaskConfig`, optional
An isrTask config to act as the base configuration. Options which
involve applying a calibration product are ignored, but this allows
for the configuration of e.g. the number of overscan columns.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with component:
- ``exposure`` : `afw.image.Exposure`
The ISRed and cosmic-ray-repaired exposure.
"""
isrConfig = isrBaseConfig if isrBaseConfig else IsrTask.ConfigClass()
isrConfig.doBias = False
isrConfig.doDark = False
isrConfig.doFlat = False
isrConfig.doFringe = False
isrConfig.doDefect = False
isrConfig.doLinearize = False
isrConfig.doCrosstalk = False
isrConfig.doBrighterFatter = False
isrConfig.usePtcGains = False
if bias:
isrConfig.doBias = True
self.log.info("Running with bias correction")
if dark:
isrConfig.doDark = True
self.log.info("Running with dark correction")
if flat:
isrConfig.doFlat = True
self.log.info("Running with flat correction")
# TODO: deal with fringes here
if defects:
isrConfig.doDefect = True
self.log.info("Running with defect correction")
if linearizer:
isrConfig.doLinearize = True
self.log.info("Running with linearity correction")
if crosstalk:
isrConfig.doCrosstalk = True
self.log.info("Running with crosstalk correction")
if bfKernel:
isrConfig.doBrighterFatter = True
self.log.info("Running with brighter-fatter correction")
if ptc:
isrConfig.usePtcGains = True
self.log.info("Running with ptc correction")
isrConfig.doWrite = False
isrTask = IsrTask(config=isrConfig)
result = isrTask.run(
ccdExposure,
camera=camera,
bias=bias,
dark=dark,
flat=flat,
# fringes=pipeBase.Struct(fringes=None),
defects=defects,
linearizer=linearizer,
crosstalk=crosstalk,
bfKernel=bfKernel,
bfGains=bfGains,
ptc=ptc,
crosstalkSources=crosstalkSources,
isGen3=True,
)
postIsr = result.exposure
if self.config.doRepairCosmics:
try: # can fail due to too many CRs detected, and we always want an exposure back
self.log.info("Repairing cosmics...")
if postIsr.getPsf() is None:
installPsfTask = InstallGaussianPsfTask()
installPsfTask.run(postIsr)
# TODO: try adding a reasonably wide Gaussian as a temp PSF
# and then just running repairTask on its own without any
# imChar. It should work, and be faster.
repairConfig = CharacterizeImageTask.ConfigClass()
repairConfig.doMeasurePsf = False
repairConfig.doApCorr = False
repairConfig.doDeblend = False
repairConfig.doWrite = False
repairConfig.repair.cosmicray.nCrPixelMax = 200000
repairTask = CharacterizeImageTask(config=repairConfig)
repairTask.repair.run(postIsr)
except Exception as e:
self.log.warning(f"During CR repair caught: {e}")
# exposure is returned for convenience to mimic isrTask's API.
return pipeBase.Struct(exposure=postIsr, outputExposure=postIsr)
