def testTaskAPI(self):
"""Test that the Tasks work
Checks both MeasureCrosstalkTask and the CrosstalkTask.
"""
# make exposure available to NullIsrTask
# without NullIsrTask's `self` hiding this test class's `self`
exposure = self.exposure
class NullIsrTask(IsrTask):
def runDataRef(self, dataRef):
return Struct(exposure=exposure)
config = MeasureCrosstalkTask.ConfigClass()
config.isr.retarget(NullIsrTask)
config.threshold = self.value - 1
measure = MeasureCrosstalkTask(config=config)
fakeDataRef = Struct(dataId={'fake': 1})
coeff, coeffErr, coeffNum = measure.reduce(
[measure.runDataRef(fakeDataRef)])
self.checkCoefficients(coeff, coeffErr, coeffNum)
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
isr.crosstalk.run(self.exposure)
self.checkSubtracted(self.exposure)
def testGainAndReadnoise(self):
isrTask = IsrTask()
detector = DetectorWrapper().detector
raw = afwImage.ExposureF(detector.getBBox())
level = 10
readNoise = 1.5
raw.image.set(level)
amp = detector[0]
for gain in [-1, 0, 0.1, 1, np.NaN]:
# Because amplifiers are immutable, we can't change the gain or
# read noise in-place. Instead, we clone, and update the clone.
testAmp = Amplifier.Builder()
testAmp.assign(amp)
testAmp.setReadNoise(readNoise)
testAmp.setGain(gain)
testAmp.finish()
isrTask.updateVariance(raw, testAmp)
if gain <= 0: # behave the same way as amp.setGain
gain = 1
if math.isnan(gain):
gain = 1
self.assertEqual(raw.variance[0, 0, afwImage.LOCAL],
level / gain + readNoise**2)
def maskAndInterpDefect(self, ccdExposure, defectBaseList):
"""Mask defects and edges, interpolate over defects in place
Mask defect pixels using mask plane BAD and interpolate over them.
Mask the potentially problematic glowing edges as SUSPECT.
Parameters
----------
ccdExposure : `lsst.afw.image.Exposure`
exposure to process
defectBaseList : `list`
a list of defects to mask and interpolate
Returns
-------
ccdExposure : `lsst.afw.image.Exposure`
exposure corrected in place
"""
IsrTask.maskAndInterpDefect(self, ccdExposure, defectBaseList)
maskedImage = ccdExposure.getMaskedImage()
goodBBox = maskedImage.getBBox()
# This makes a bbox numEdgeSuspect pixels smaller than the image on each side
goodBBox.grow(-self.config.numEdgeSuspect)
# Mask pixels outside goodBBox as SUSPECT
SourceDetectionTask.setEdgeBits(
maskedImage, goodBBox,
maskedImage.getMask().getPlaneBitMask("SUSPECT"))
def runIsr():
'''Run the task to do ISR on a ccd'''
#Create the isr task with modified config
isrConfig = IsrTask.ConfigClass()
isrConfig.doBias = False #We didn't make a zero frame
isrConfig.doDark = True
isrConfig.doFlat = True
isrConfig.doFringe = False #There is no fringe frame for this example
isrConfig.assembleCcd.setGain = False
isrTask = IsrTask(config=isrConfig)
#Make raw, flat and dark exposures
DARKVAL = 2. #e-/sec
OSCAN = 1000. #DN
GRADIENT = .10
EXPTIME = 15 #seconds
DARKEXPTIME = 40. #seconds
darkExposure = exampleUtils.makeDark(DARKVAL, DARKEXPTIME)
flatExposure = exampleUtils.makeFlat(GRADIENT)
rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME)
output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure)
return output.exposure
def testGainAndReadnoise(self):
import lsst.afw.image as afwImage
from lsst.afw.cameraGeom.testUtils import DetectorWrapper
from lsst.ip.isr import IsrTask
isrTask = IsrTask()
detector = DetectorWrapper().detector
raw = afwImage.ExposureF(detector.getBBox())
level = 10
readNoise = 1
raw.image.set(level)
amp = detector[0]
amp.setReadNoise(readNoise)
for gain in [-1, 0, 0.1, 1]:
amp.setGain(gain)
isrTask.updateVariance(raw, amp)
if gain <= 0:
gain = 1
self.assertEqual(raw.variance[0, 0, afwImage.LOCAL],
level / gain + readNoise**2)
def runIsr():
'''Run the task to do ISR on a ccd'''
#Create the isr task with modified config
isrConfig = IsrTask.ConfigClass()
isrConfig.doBias = False #We didn't make a zero frame
isrConfig.doDark = True
isrConfig.doFlat = True
isrConfig.doFringe = False #There is no fringe frame for this example
isrConfig.assembleCcd.doRenorm = False #We'll take care of gain in the flats
isrConfig.assembleCcd.setGain = False
isrTask = IsrTask(config=isrConfig)
#Make raw, flat and dark exposures
DARKVAL = 2. #e-/sec
OSCAN = 1000. #DN
GRADIENT = .10
EXPTIME = 15 #seconds
DARKEXPTIME = 40. #seconds
darkExposure = exampleUtils.makeDark(DARKVAL, DARKEXPTIME)
flatExposure = exampleUtils.makeFlat(GRADIENT)
rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME)
output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure)
return output.exposure
def execute(self, dataRef):
"""!Apply common instrument signature correction algorithms to a raw frame
@param dataRef: butler data reference
@return a pipeBase Struct containing:
- exposure
similar to IsrTask.runDataRef()
"""
self.log.info("Performing Super ISR on sensor data ID %s" %
(dataRef.dataId, ))
# IsrTask.runDataRef includes these three steps
self.log.info("Reading input data using dataRef")
inputData = self.read_input_data(dataRef)
self.log.info(
"Running operations. The run() method should not take anything Butler"
)
result = IsrTask.run(IsrTask(self.config), **inputData.getDict())
self.log.info("Writing output data using dataRef")
self.write_output_data(dataRef, result)
return result
def processExposure(exposure,
bias=None,
defects=None,
repair=False,
repo=None):
from lsst.ip.isr import IsrTask
import lsst.daf.persistence as dafPersist
import time
config = IsrTask.ConfigClass()
if (bias == None or defects == None) and repo == None:
raise (AttributeError(
'Repo keyword not set and is required to find bias/defect frames'))
else:
butler = dafPersist.Butler(repo)
if False:
config.overscanFitType = "AKIMA_SPLINE"
config.overscanOrder = 5
else:
config.overscanFitType = "POLY"
config.overscanOrder = 3
config.doBias = True
if config.doBias == True and bias == None:
bias = butler.get(
'bias',
visit=exposure.getInfo().getVisitInfo().getExposureId(),
)
config.doDark = False
config.doFringe = False
config.doFlat = False
config.doLinearize = False
# TODO: should be able to run this but there are issues with defects in the stack at the moment
config.doDefect = True if repair == True else False
if (config.doDefect == True and bias == None):
butler = dafPersist.Butler(repo)
defects = butler.get(
'defects',
visit=exposure.getInfo().getVisitInfo().getExposureId(),
)
config.doAddDistortionModel = False
config.doSaturationInterpolation = False
config.overscanNumLeadingColumnsToSkip = 20
isrTask = IsrTask(config=config)
# Run ISR
start = time.time()
isr_corrected_exposure = isrTask.run(exposure, bias=bias,
defects=defects).exposure
end = time.time()
print('Time to perform image ISR was {0:2f} [s]'.format(end - start))
return (isr_corrected_exposure)
def overscanCorrection(self, exposure, amp):
"""Apply overscan correction in place
If the input exposure is on the readout backplanes listed in
config.overscanBiasJumpBKP, cut the amplifier in two vertically
and correct each piece separately.
@param[in,out] exposure: exposure to process; must include both
DataSec and BiasSec pixels
@param[in] amp: amplifier device data
"""
if not (
exposure.getMetadata().exists("FPA")
and exposure.getMetadata().get("FPA") in self.config.overscanBiasJumpBKP
):
IsrTask.overscanCorrection(self, exposure, amp)
return
dataBox = amp.getRawDataBBox()
overscanBox = amp.getRawHorizontalOverscanBBox()
if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
yLower = self.config.overscanBiasJumpLocation
yUpper = dataBox.getHeight() - yLower
else:
yUpper = self.config.overscanBiasJumpLocation
yLower = dataBox.getHeight() - yUpper
lowerDataBBox = afwGeom.Box2I(dataBox.getBegin(), afwGeom.Extent2I(dataBox.getWidth(), yLower))
upperDataBBox = afwGeom.Box2I(
dataBox.getBegin() + afwGeom.Extent2I(0, yLower), afwGeom.Extent2I(dataBox.getWidth(), yUpper)
)
lowerOverscanBBox = afwGeom.Box2I(overscanBox.getBegin(), afwGeom.Extent2I(overscanBox.getWidth(), yLower))
upperOverscanBBox = afwGeom.Box2I(
overscanBox.getBegin() + afwGeom.Extent2I(0, yLower), afwGeom.Extent2I(overscanBox.getWidth(), yUpper)
)
maskedImage = exposure.getMaskedImage()
lowerDataView = maskedImage.Factory(maskedImage, lowerDataBBox)
upperDataView = maskedImage.Factory(maskedImage, upperDataBBox)
expImage = exposure.getMaskedImage().getImage()
lowerOverscanImage = expImage.Factory(expImage, lowerOverscanBBox)
upperOverscanImage = expImage.Factory(expImage, upperOverscanBBox)
overscanCorrection(
ampMaskedImage=lowerDataView,
overscanImage=lowerOverscanImage,
fitType=self.config.overscanFitType,
order=self.config.overscanOrder,
collapseRej=self.config.overscanRej,
)
overscanCorrection(
ampMaskedImage=upperDataView,
overscanImage=upperOverscanImage,
fitType=self.config.overscanFitType,
order=self.config.overscanOrder,
collapseRej=self.config.overscanRej,
)
def __init__(self, **kwargs):
IsrTask.__init__(self, **kwargs)
self.transposeForInterpolation = True # temporary hack until LSST data is in proper order
self.statsCtrl = afwMath.StatisticsControl()
self.statsCtrl.setNumSigmaClip(self.config.sigmaClip)
self.statsCtrl.setNumIter(self.config.clipIter)
# Not sure how to do this.
# self.statsCtrl.setAndMask('BAD')
self.isr = isr
def __init__(self, **kwargs):
IsrTask.__init__(self, **kwargs)
self.transposeForInterpolation = True # temporary hack until LSST data is in proper order
self.statsCtrl = afwMath.StatisticsControl()
self.statsCtrl.setNumSigmaClip(self.config.sigmaClip)
self.statsCtrl.setNumIter(self.config.clipIter)
# Not sure how to do this.
# self.statsCtrl.setAndMask('BAD')
self.isr = isr
def testRunWithAddDistortionModel(self):
"""Test IsrTask.run with config.doAddDistortionModel true"""
isrConfig = self.makeMinimalIsrConfig()
isrConfig.doAddDistortionModel = True
isrTask = IsrTask(config=isrConfig)
with self.assertRaises(RuntimeError):
# the camera argument is required
isrTask.run(ccdExposure=self.exposure)
exposure = isrTask.run(ccdExposure=self.exposure, camera=self.camera).exposure
desiredWcs = self.makeDesiredDistortedWcs()
self.assertWcsAlmostEqualOverBBox(desiredWcs, exposure.getWcs(), self.bbox)
def testAddDistortionMethod(self):
"""Call IsrTask.addDistortionModel directly"""
isrTask = IsrTask()
isrTask.addDistortionModel(self.exposure, self.camera)
self.assertFalse(
wcsAlmostEqualOverBBox(self.wcs, self.exposure.getWcs(),
self.bbox))
desiredWcs = self.makeDesiredDistortedWcs()
self.assertWcsAlmostEqualOverBBox(desiredWcs, self.exposure.getWcs(),
self.bbox)
def testRunWithoutAddDistortionModel(self):
"""Test IsrTask.run with config.doAddDistortionModel false"""
isrConfig = self.makeMinimalIsrConfig()
isrTask = IsrTask(config=isrConfig)
# the camera argument is not needed
exposure = isrTask.run(ccdExposure=self.exposure).exposure
self.assertEqual(self.wcs, exposure.getWcs())
# and the camera argument is ignored if provided
exposure2 = isrTask.run(ccdExposure=self.exposure, camera=self.camera).exposure
self.assertEqual(self.wcs, exposure2.getWcs())
def read_input_data(self, dataRef):
"""Read needed data thru Butler in a pipeBase.Struct based on config"""
ccdExp = dataRef.get('raw', immediate=True)
isrData = IsrTask.readIsrData(IsrTask(self.config), dataRef, ccdExp)
isrDataDict = isrData.getDict()
return pipeBase.Struct(ccdExposure=ccdExp,
bias=isrDataDict['bias'],
linearizer=isrDataDict['linearizer'],
dark=isrDataDict['dark'],
flat=isrDataDict['flat'],
defects=isrDataDict['defects'],
fringes=isrDataDict['fringes'],
bfKernel=isrDataDict['bfKernel'])
def testRunWithAddDistortionModel(self):
"""Test IsrTask.run with config.doAddDistortionModel true"""
isrConfig = self.makeMinimalIsrConfig()
isrConfig.doAddDistortionModel = True
isrTask = IsrTask(config=isrConfig)
with self.assertRaises(RuntimeError):
# the camera argument is required
isrTask.run(ccdExposure=self.exposure)
exposure = isrTask.run(ccdExposure=self.exposure,
camera=self.camera).exposure
desiredWcs = self.makeDesiredDistortedWcs()
self.assertWcsAlmostEqualOverBBox(desiredWcs, exposure.getWcs(),
self.bbox)
def testRunWithoutAddDistortionModel(self):
"""Test IsrTask.run with config.doAddDistortionModel false"""
isrConfig = self.makeMinimalIsrConfig()
isrTask = IsrTask(config=isrConfig)
# the camera argument is not needed
exposure = isrTask.run(ccdExposure=self.exposure).exposure
self.assertEqual(self.wcs, exposure.getWcs())
# and the camera argument is ignored if provided
exposure2 = isrTask.run(ccdExposure=self.exposure,
camera=self.camera).exposure
self.assertEqual(self.wcs, exposure2.getWcs())
def maskAndInterpDefect(self, ccdExposure, defectBaseList):
"""Mask defects and edges, interpolate over defects in place
Mask defect pixels using mask plane BAD and interpolate over them.
Mask the potentially problematic glowing edges as SUSPECT.
@param[in,out] ccdExposure: exposure to process
@param[in] defectBaseList: a list of defects to mask and interpolate
"""
IsrTask.maskAndInterpDefect(self, ccdExposure, defectBaseList)
maskedImage = ccdExposure.getMaskedImage()
goodBBox = maskedImage.getBBox()
# This makes a bbox numEdgeSuspect pixels smaller than the image on each side
goodBBox.grow(-self.config.numEdgeSuspect)
# Mask pixels outside goodBBox as SUSPECT
SourceDetectionTask.setEdgeBits(maskedImage, goodBBox, maskedImage.getMask().getPlaneBitMask("SUSPECT"))
def makeMinimalIsrConfig(self):
"""Return an IsrConfig with all boolean flags disabled"""
isrConfig = IsrTask.ConfigClass()
for name in isrConfig:
if name.startswith("do"):
setattr(isrConfig, name, False)
return isrConfig
def testTaskAPI(self):
"""Test that the Tasks work
Checks both MeasureCrosstalkTask and the CrosstalkTask.
"""
# make exposure available to NullIsrTask
# without NullIsrTask's `self` hiding this test class's `self`
exposure = self.exposure
class NullIsrTask(IsrTask):
def runDataRef(self, dataRef):
return Struct(exposure=exposure)
coeff = np.array(self.crosstalk).transpose()
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
calib = CrosstalkCalib().fromDetector(self.exposure.getDetector(),
coeffVector=coeff)
isr.crosstalk.run(self.exposure, crosstalk=calib)
self.checkSubtracted(self.exposure)
def run(self, ccdExposure, bias=None, dark=None, flat=None, defects=None, fringes=None):
"""Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
- Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
@param[in] ccdExposure -- lsst.afw.image.exposure of detector data
@param[in] bias -- exposure of bias frame
@param[in] dark -- exposure of dark frame
@param[in] flat -- exposure of flatfield
@param[in] defects -- list of detects
@param[in] fringes -- exposure of fringe frame or list of fringe exposure
@return a pipeBase.Struct with fields:
- exposure: the exposure after application of ISR
"""
ccd = ccdExposure.getDetector()
floatExposure = self.convertIntToFloat(ccdExposure)
metadata = floatExposure.getMetadata()
# Detect saturation
# Saturation values recorded in the fits hader is not reliable, try to estimate it from the pixel vales
# Find the peak location in the high end part the pixel values' histogram and set the saturation level at
# safe * (peak location) where safe is a configurable parameter (typically 0.95)
image = floatExposure.getMaskedImage().getImage()
imageArray = image.getArray()
maxValue = np.max(imageArray)
if maxValue > 60000.0:
hist, bin_edges = np.histogram(imageArray.ravel(), bins=100, range=(60000.0, maxValue + 1.0))
saturate = int(self.config.safe * bin_edges[np.argmax(hist)])
else:
saturate = metadata.get("SATURATE")
self.log.info("Saturation set to %d" % saturate)
for amp in ccd:
amp.setSaturation(saturate)
if amp.getName() == "A":
amp.setGain(metadata.get("GAINA"))
amp.setReadNoise(metadata.get("RDNOISEA"))
elif amp.getName() == "B":
amp.setGain(metadata.get("GAINB"))
amp.setReadNoise(metadata.get("RDNOISEB"))
else:
raise ValueError("Unexpected amplifier name : %s" % (amp.getName()))
return IsrTask.run(
self, ccdExposure=ccdExposure, bias=bias, dark=dark, flat=flat, defects=defects, fringes=fringes
)
def test_interChip(self):
"""Test that passing an external exposure as the crosstalk source
works.
"""
exposure = self.exposure
ctSources = [self.ctSource]
coeff = np.array(self.crosstalk).transpose()
calib = CrosstalkCalib().fromDetector(exposure.getDetector(),
coeffVector=coeff)
# Now convert this into zero intra-chip, full inter-chip:
calib.interChip['detector 2'] = coeff
calib.coeffs = np.zeros_like(coeff)
# Process and check as above
config = IsrTask.ConfigClass()
config.crosstalk.minPixelToMask = self.value - 1
config.crosstalk.crosstalkMaskPlane = self.crosstalkStr
isr = IsrTask(config=config)
isr.crosstalk.run(exposure,
crosstalk=calib,
crosstalkSources=ctSources)
self.checkSubtracted(exposure)
def testGainAndReadnoise(self):
import lsst.afw.image as afwImage
from lsst.afw.cameraGeom.testUtils import DetectorWrapper
from lsst.ip.isr import IsrTask
isrTask = IsrTask()
detector = DetectorWrapper().detector
raw = afwImage.ExposureF(detector.getBBox())
level = 10
readNoise = 1
raw.image.set(level)
amp = detector[0]
amp.setReadNoise(readNoise)
for gain in [-1, 0, 0.1, 1]:
amp.setGain(gain)
isrTask.updateVariance(raw, amp)
if gain <= 0:
gain = 1
self.assertEqual(raw.variance[0, 0, afwImage.LOCAL], level/gain + readNoise**2)
def __init__(self, *, config=None):
# programatically clone all of the connections from isrTask
# settin minimum values to zero for everything except the ccdExposure
super().__init__(config=IsrTask.ConfigClass(
)) # need a dummy config, isn't used other than for ctor
for name, connection in self.allConnections.items():
if hasattr(connection, 'minimum'):
args = _getArgs(connection)
if name != "ccdExposure": # need one input image always
args['minimum'] = 0
newConnection = type(connection)(**args)
self.allConnections[name] = newConnection
setattr(self, name, newConnection)
exposure = cT.Output( # called just "exposure" to mimic isrTask's return struct
name="quickLookExp",
doc="The quickLook output exposure.",
storageClass="ExposureF",
dimensions=("instrument", "exposure", "detector"),
)
# set like this to make it explicit that the outputExposure
# and the exposure are identical. The only reason there are two is for
# API compatibility.
self.outputExposure = exposure
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
fringes=None, bfKernel=None, camera=None, **kwds):
"""Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
- Persist the ISR-corrected exposure as "postISRCCD" if
config.doWrite is True
Parameters
----------
ccdExposure : `lsst.afw.image.Exposure`
Detector data.
bias : `lsst.afw.image.exposure`
Exposure of bias frame.
linearizer : `lsst.ip.isr.LinearizeBase` callable
Linearizing functor; a subclass of lsst.ip.isr.LinearizeBase.
dark : `lsst.afw.image.exposure`
Exposure of dark frame.
flat : `lsst.afw.image.exposure`
Exposure of flatfield.
defects : `list`
list of detects
fringes : `lsst.afw.image.exposure` or `list` of `lsst.afw.image.exposure`
exposure of fringe frame or list of fringe exposure
bfKernel : None
kernel used for brighter-fatter correction; currently unsupported
camera : `lsst.afw.cameraGeom.Camera`
Camera geometry, used by addDistortionModel.
**kwds : `dict`
additional kwargs forwarded to IsrTask.run.
Returns
-------
struct : `lsst.pipe.base.Struct` with fields:
- exposure: the exposure after application of ISR
"""
if bfKernel is not None:
raise ValueError("CFHT ISR does not currently support brighter-fatter correction.")
ccd = ccdExposure.getDetector()
floatExposure = self.convertIntToFloat(ccdExposure)
metadata = floatExposure.getMetadata()
# Detect saturation
# Saturation values recorded in the fits hader is not reliable, try to
# estimate it from the pixel values.
# Find the peak location in the high end part the pixel values'
# histogram and set the saturation level at safe * (peak location)
# where safe is a configurable parameter (typically 0.95)
image = floatExposure.getMaskedImage().getImage()
imageArray = image.getArray()
maxValue = np.max(imageArray)
if maxValue > 60000.0:
hist, bin_edges = np.histogram(imageArray.ravel(), bins=100, range=(60000.0, maxValue+1.0))
saturate = int(self.config.safe*bin_edges[np.argmax(hist)])
else:
saturate = metadata.getScalar("SATURATE")
self.log.info("Saturation set to %d" % saturate)
for amp in ccd:
amp.setSaturation(saturate)
if amp.getName() == "A":
amp.setGain(metadata.getScalar("GAINA"))
rdnA = metadata.getScalar("RDNOISEA")
# Check if the noise value is making sense for this amp. If
# not, replace with value stored in RDNOISE slot. This change
# is necessary to process some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnA > 60000.0:
rdnA = metadata.getScalar("RDNOISE")
amp.setReadNoise(rdnA)
elif amp.getName() == "B":
amp.setGain(metadata.getScalar("GAINB"))
rdnB = metadata.getScalar("RDNOISEB")
# Check if the noise value is making sense for this amp.
# If not, replace with value
# stored in RDNOISE slot. This change is necessary to process
# some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnB > 60000.0:
rdnB = metadata.getScalar("RDNOISE")
amp.setReadNoise(rdnB)
else:
raise ValueError("Unexpected amplifier name : %s"%(amp.getName()))
return IsrTask.run(self,
ccdExposure=ccdExposure,
bias=bias,
linearizer=linearizer,
dark=dark,
flat=flat,
defects=defects,
fringes=fringes,
camera=camera,
**kwds
)
def __init__(self, **kwargs):
IsrTask.__init__(self, **kwargs)
self.makeSubtask("snapCombine")
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp(
) if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2")
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask), name="charImage2")
calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema)
try:
dataId = dict(visit=1)
dataIdStrList = [
"%s=%s" % (key, val) for key, val in dataId.items()
]
isrResult1 = IsrTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get(
"postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
icResult1 = CharacterizeImageTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure,
exposureIdInfo=exposureIdInfo)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get(
"icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId,
immediate=True),
icResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground",
dataId,
immediate=True),
icResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background)
calResult1 = CalibrateTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat,
exposureIdInfo=exposureIdInfo,
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get(
"calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent"))
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground",
dataId,
immediate=True),
calResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" %
(OutputName, ))
def run(self, exposure, crosstalkSources=None):
"""Perform crosstalk correction on a DECam exposure and its corresponding dataRef.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure to correct.
dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
DataRef of exposure to correct which must include a dataId
with at least one visit and ccdnum.
crosstalkSources : `defaultdict`
Must contain image data and corresponding crosstalk coefficients for
each crosstalk source of the given dataRef victim. This is returned
by prepCrosstalk.
Returns
-------
`lsst.pipe.base.Struct`
Struct with components:
- ``exposure``: The exposure after crosstalk correction has been
applied (`lsst.afw.image.Exposure`).
"""
self.log.info('Applying crosstalk correction')
assert crosstalkSources is not None, "Sources are required for DECam crosstalk correction; \
you must run CrosstalkTask via IsrTask which will \
call prepCrosstalk to get the sources."
# Load data from crosstalk 'victim' exposure we want to correct
det = exposure.getDetector()
for amp in det:
ccdnum = det.getId()
ccdnum_str = '%02d' % ccdnum
victim = ccdnum_str + amp.getName()
# If doCrosstalkBeforeAssemble=True, then use getRawBBox(). Otherwise, getBBox().
dataBBox = amp.getRawBBox()
# Check to see if victim overscan has been corrected, and if not, correct it first
if not exposure.getMetadata().exists('OVERSCAN'):
decamisr = IsrTask()
decamisr.overscanCorrection(exposure, amp)
self.log.warn(
'Overscan correction did not happen prior to crosstalk correction'
)
self.log.info(
'Correcting victim %s overscan before crosstalk' % victim)
image = exposure.getMaskedImage().getImage()
victim_data = image.Factory(image, dataBBox)
# Load data from crosstalk 'source' exposures
for source in crosstalkSources[victim]:
source_data, source_coeff, source_idx = source
victim_idx = 0
if 'A' in victim:
victim_idx = 0
elif 'B' in victim:
victim_idx = 1
else:
self.log.fatal(
'DECam victim amp name does not contain A or B, cannot proceed'
)
if source_idx != victim_idx:
# Occurs with amp A and B mismatch; need to flip horizontally
source_data = afwMath.flipImage(source_data,
flipLR=True,
flipTB=False)
# Perform the linear crosstalk correction
try:
source_data *= source_coeff
victim_data -= source_data
except RuntimeError:
self.log.fatal(
'Crosstalk correction failed for victim %s from source %s'
% (victim, source))
return pipeBase.Struct(exposure=exposure, )
def prepCrosstalk(self, dataRef):
"""Retrieve source image data and coefficients to prepare for crosstalk correction.
This is called by readIsrData. The crosstalkSources land in isrData.
Parameters
----------
dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef`
DataRef of exposure to correct which must include a dataId with
at least one visit and a ccdnum corresponding to the detector id.
Returns
-------
crosstalkSources : `defaultdict`
Contains image data and corresponding crosstalk coefficients for
each crosstalk source of the given dataRef victim.
"""
# Retrieve crosstalk sources and coefficients for the whole focal plane
sources, coeffs = self.config.getSourcesAndCoeffs()
# Define a butler to prepare for accessing crosstalk 'source' image data
try:
butler = dataRef.getButler()
except RuntimeError:
self.log.fatal('Cannot get a Butler from the dataRef provided')
# Retrieve visit and ccdnum from 'victim' so we can look up 'sources'
victim_exposure = dataRef.get()
det = victim_exposure.getDetector()
visit = dataRef.dataId['visit']
ccdnum = det.getId()
ccdnum_str = '%02d' % ccdnum
crosstalkSources = defaultdict(list)
decamisr = IsrTask() # needed for source_exposure overscan correction
for amp in det:
victim = ccdnum_str + amp.getName()
for source in sources[victim]:
source_ccdnum = int(source[:2])
try:
source_exposure = butler.get('raw',
dataId={
'visit': visit,
'ccdnum': source_ccdnum
})
except RuntimeError:
self.log.warn('Cannot access source %d, SKIPPING IT' %
source_ccdnum)
else:
self.log.info(
'Correcting victim %s from crosstalk source %s' %
(victim, source))
if 'A' in source:
amp_idx = 0
elif 'B' in source:
amp_idx = 1
else:
self.log.fatal(
'DECam source amp name does not contain A or B, cannot proceed'
)
source_amp = source_exposure.getDetector()[amp_idx]
# If doCrosstalkBeforeAssemble=True, then use getRawBBox(). Otherwise, getRawDataBBox().
source_dataBBox = source_amp.getRawBBox()
# Check to see if source overscan has been corrected, and if not, correct it first
if not source_exposure.getMetadata().exists('OVERSCAN'):
decamisr.overscanCorrection(source_exposure,
source_amp)
self.log.info(
'Correcting source %s overscan before using to correct crosstalk'
% source)
source_image = source_exposure.getMaskedImage().getImage()
source_data = source_image.Factory(source_image,
source_dataBBox,
deep=True)
crosstalkSources[victim].append(
(source_data, coeffs[(victim, source)], amp_idx))
return crosstalkSources
def overscanCorrection(self, exposure, amp):
"""Apply overscan correction in place
If the input exposure is on the readout backplanes listed in
config.overscanBiasJumpBKP, cut the amplifier in two vertically
and correct each piece separately.
@param[in,out] exposure: exposure to process; must include both
DataSec and BiasSec pixels
@param[in] amp: amplifier device data
"""
if not (exposure.getMetadata().exists('FPA')
and exposure.getMetadata().get('FPA')
in self.config.overscanBiasJumpBKP):
IsrTask.overscanCorrection(self, exposure, amp)
return
dataBox = amp.getRawDataBBox()
overscanBox = amp.getRawHorizontalOverscanBBox()
if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
yLower = self.config.overscanBiasJumpLocation
yUpper = dataBox.getHeight() - yLower
else:
yUpper = self.config.overscanBiasJumpLocation
yLower = dataBox.getHeight() - yUpper
lowerDataBBox = afwGeom.Box2I(
dataBox.getBegin(), afwGeom.Extent2I(dataBox.getWidth(), yLower))
upperDataBBox = afwGeom.Box2I(
dataBox.getBegin() + afwGeom.Extent2I(0, yLower),
afwGeom.Extent2I(dataBox.getWidth(), yUpper))
lowerOverscanBBox = afwGeom.Box2I(
overscanBox.getBegin(),
afwGeom.Extent2I(overscanBox.getWidth(), yLower))
upperOverscanBBox = afwGeom.Box2I(
overscanBox.getBegin() + afwGeom.Extent2I(0, yLower),
afwGeom.Extent2I(overscanBox.getWidth(), yUpper))
maskedImage = exposure.getMaskedImage()
lowerDataView = maskedImage.Factory(maskedImage, lowerDataBBox)
upperDataView = maskedImage.Factory(maskedImage, upperDataBBox)
expImage = exposure.getMaskedImage().getImage()
lowerOverscanImage = expImage.Factory(expImage, lowerOverscanBBox)
upperOverscanImage = expImage.Factory(expImage, upperOverscanBBox)
overscanCorrection(
ampMaskedImage=lowerDataView,
overscanImage=lowerOverscanImage,
fitType=self.config.overscanFitType,
order=self.config.overscanOrder,
collapseRej=self.config.overscanRej,
)
overscanCorrection(
ampMaskedImage=upperDataView,
overscanImage=upperOverscanImage,
fitType=self.config.overscanFitType,
order=self.config.overscanOrder,
collapseRej=self.config.overscanRej,
)
def overscanCorrection(self, exposure, amp):
"""Apply overscan correction in place
If the input exposure is on the readout backplanes listed in
config.overscanBiasJumpBKP, cut the amplifier in two vertically
and correct each piece separately.
Parameters
----------
exposure: `lsst.afw.image.Exposure`
exposure to process; must include both DataSec and BiasSec pixels
amp: `lsst.afw.table.AmpInfoRecord`
amplifier device data
Returns
-------
exposure: `lsst.afw.image.Exposure`
exposure corrected in place with updated metadata
"""
if not (exposure.getMetadata().exists('FPA')
and exposure.getMetadata().get('FPA')
in self.config.overscanBiasJumpBKP):
IsrTask.overscanCorrection(self, exposure, amp)
return
dataBox = amp.getRawDataBBox()
overscanBox = amp.getRawHorizontalOverscanBBox()
if amp.getReadoutCorner() in (afwTable.LL, afwTable.LR):
yLower = self.config.overscanBiasJumpLocation
yUpper = dataBox.getHeight() - yLower
else:
yUpper = self.config.overscanBiasJumpLocation
yLower = dataBox.getHeight() - yUpper
lowerDataBBox = afwGeom.Box2I(
dataBox.getBegin(), afwGeom.Extent2I(dataBox.getWidth(), yLower))
upperDataBBox = afwGeom.Box2I(
dataBox.getBegin() + afwGeom.Extent2I(0, yLower),
afwGeom.Extent2I(dataBox.getWidth(), yUpper))
lowerOverscanBBox = afwGeom.Box2I(
overscanBox.getBegin(),
afwGeom.Extent2I(overscanBox.getWidth(), yLower))
upperOverscanBBox = afwGeom.Box2I(
overscanBox.getBegin() + afwGeom.Extent2I(0, yLower),
afwGeom.Extent2I(overscanBox.getWidth(), yUpper))
maskedImage = exposure.getMaskedImage()
lowerDataView = maskedImage.Factory(maskedImage, lowerDataBBox)
upperDataView = maskedImage.Factory(maskedImage, upperDataBBox)
expImage = exposure.getMaskedImage().getImage()
lowerOverscanImage = expImage.Factory(expImage, lowerOverscanBBox)
upperOverscanImage = expImage.Factory(expImage, upperOverscanBBox)
overscanCorrection(
ampMaskedImage=lowerDataView,
overscanImage=lowerOverscanImage,
fitType=self.config.overscanFitType,
order=self.config.overscanOrder,
collapseRej=self.config.overscanRej,
)
overscanCorrection(
ampMaskedImage=upperDataView,
overscanImage=upperOverscanImage,
fitType=self.config.overscanFitType,
order=self.config.overscanOrder,
collapseRej=self.config.overscanRej,
)
# Note that overscan correction has been done in exposure metadata
metadata = exposure.getMetadata()
metadata.set(
'OVERSCAN', 'Overscan corrected on {0}'.format(
dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")))
exposure.setMetadata(metadata)
def __init__(self, **kwargs):
IsrTask.__init__(self, **kwargs)
self.makeSubtask("snapCombine")
def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, **kwargs):
"""Get the postIsr and cosmic-repaired image for this dataId.
Parameters
----------
expIdOrDataId : `dict`
The dataId
extraIsrOptions : `dict`, optional
extraIsrOptions is a dict of extra isr options applied to this
image only.
skipCosmics : `bool`, optional # XXX THIS CURRENTLY DOESN'T WORK!
Skip doing cosmic ray repair for this image?
Returns
-------
exp : `lsst.afw.image.Exposure`
The postIsr exposure
"""
dataId = self._parseExpIdOrDataId(expIdOrDataId, **kwargs)
try:
exp = self.butler.get(self._datasetName, dataId=dataId)
self.log.info("Found a ready-made quickLookExp in the repo. Returning that.")
return exp
except LookupError:
pass
try:
raw = self.butler.get('raw', dataId=dataId)
except LookupError:
raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None
# default options that are probably good for most engineering time
isrConfig = IsrTask.ConfigClass()
isrConfig.doWrite = False # this task writes separately, no need for this
isrConfig.doSaturation = True # saturation very important for roundness measurement in qfm
isrConfig.doSaturationInterpolation = True
isrConfig.overscanNumLeadingColumnsToSkip = 5
isrConfig.overscan.fitType = 'MEDIAN_PER_ROW'
# apply general overrides
self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions)
# apply per-image overrides
self._applyConfigOverrides(isrConfig, extraIsrOptions)
isrParts = ['camera', 'bias', 'dark', 'flat', 'defects', 'linearizer', 'crosstalk', 'bfKernel',
'bfGains', 'ptc']
isrDict = {}
# we build a cache of all the isr components which will be used to save
# the IO time on subsequent calls. This assumes people will not update
# calibration products while this object lives, but this is a fringe
# use case, and if they do, all they would need to do would be call
# .clearCache() and this will rebuild with the new products.
for component in isrParts:
if component in self._cache and component != 'flat':
self.log.info(f"Using {component} from cache...")
isrDict[component] = self._cache[component]
continue
try:
# TODO: add caching for flats
item = self.butler.get(component, dataId=dataId)
self._cache[component] = item
isrDict[component] = self._cache[component]
except (RuntimeError, LookupError, OperationalError):
pass
quickLookExp = self.quickLookIsrTask.run(raw, **isrDict, isrBaseConfig=isrConfig).outputExposure
if self.doWrite:
try:
self.butler.put(quickLookExp, self._datasetName, dataId)
self.log.info(f'Put {self._datasetName} for {dataId}')
except ConflictingDefinitionError:
# TODO: DM-34302 fix this message so that it's less scary for
# users. Do this by having daemons know they're daemons.
self.log.warning('Skipped putting existing exp into collection! (ignore if there was a race)')
pass
return quickLookExp
def run(config,
inputFiles,
weightFiles=None,
varianceFiles=None,
returnCalibSources=False,
displayResults=[],
verbose=False):
#
# Create the tasks
#
schema = afwTable.SourceTable.makeMinimalSchema()
algMetadata = dafBase.PropertyList()
isrTask = IsrTask(config=config.isr)
charImageTask = CharacterizeImageTask(None, config=config.charImage)
sourceDetectionTask = SourceDetectionTask(config=config.detection,
schema=schema)
if config.doDeblend:
if SourceDeblendTask:
sourceDeblendTask = SourceDeblendTask(config=config.deblend,
schema=schema)
else:
print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False"
config.doDeblend = False
sourceMeasurementTask = SingleFrameMeasurementTask(
schema=schema, config=config.measurement, algMetadata=algMetadata)
keysToCopy = []
for key in [
charImageTask.measurePsf.reservedKey,
charImageTask.measurePsf.usedKey,
]:
keysToCopy.append(
(schema.addField(charImageTask.schema.find(key).field), key))
exposureDict = {}
calibSourcesDict = {}
sourcesDict = {}
for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles,
varianceFiles):
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# read the data
#
if verbose:
print "Reading %s" % inputFile
exposure = makeExposure(inputFile, weightFile, varianceFile,
config.badPixelValue, config.variance)
#
if config.interpPlanes:
import lsst.ip.isr as ipIsr
defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(),
config.interpPlanes,
growFootprints=0)
isrTask.run(exposure, defects=defects)
#
# process the data
#
if config.doCalibrate:
result = charImageTask.characterize(exposure)
exposure, calibSources = result.exposure, result.sourceCat
else:
calibSources = None
if not exposure.getPsf():
charImageTask.installSimplePsf.run(exposure)
exposureDict[inputFile] = exposure
calibSourcesDict[
inputFile] = calibSources if returnCalibSources else None
result = sourceDetectionTask.run(tab, exposure)
sources = result.sources
sourcesDict[inputFile] = sources
if config.doDeblend:
sourceDeblendTask.run(exposure, sources)
sourceMeasurementTask.measure(exposure, sources)
if verbose:
print "Detected %d objects" % len(sources)
propagatePsfFlags(keysToCopy, calibSources, sources)
if displayResults: # display results of processing (see also --debug argparse option)
showApertures = "showApertures".upper() in displayResults
showPSFs = "showPSFs".upper() in displayResults
showShapes = "showShapes".upper() in displayResults
display = afwDisplay.getDisplay(frame=1)
if algMetadata.exists("base_CircularApertureFlux_radii"):
radii = algMetadata.get("base_CircularApertureFlux_radii")
else:
radii = []
display.mtv(exposure, title=os.path.split(inputFile)[1])
with display.Buffering():
for s in sources:
xy = s.getCentroid()
display.dot('+',
*xy,
ctype=afwDisplay.CYAN if
s.get("flags_negative") else afwDisplay.GREEN)
if showPSFs and (s.get("calib_psfUsed")
or s.get("calib_psfReserved")):
display.dot(
'o',
*xy,
size=10,
ctype=afwDisplay.GREEN
if s.get("calib_psfUsed") else afwDisplay.YELLOW)
if showShapes:
display.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
if showApertures:
for radius in radii:
display.dot('o',
*xy,
size=radius,
ctype=afwDisplay.YELLOW)
return exposureDict, calibSourcesDict, sourcesDict
def run(self,
ccdExposure,
bias=None,
linearizer=None,
dark=None,
flat=None,
defects=None,
fringes=None,
bfKernel=None,
**kwds):
"""Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
- Persist the ISR-corrected exposure as "postISRCCD" if
config.doWrite is True
Parameters
----------
ccdExposure : `lsst.afw.image.Exposure`
Detector data.
bias : `lsst.afw.image.exposure`
Exposure of bias frame.
linearizer : `lsst.ip.isr.LinearizeBase` callable
Linearizing functor; a subclass of lsst.ip.isr.LinearizeBase.
dark : `lsst.afw.image.exposure`
Exposure of dark frame.
flat : `lsst.afw.image.exposure`
Exposure of flatfield.
defects : `list`
list of detects
fringes : `lsst.afw.image.Exposure` or list `lsst.afw.image.Exposure`
exposure of fringe frame or list of fringe exposure
bfKernel : None
kernel used for brighter-fatter correction; currently unsupported
**kwds : `dict`
additional kwargs forwarded to IsrTask.run.
Returns
-------
struct : `lsst.pipe.base.Struct` with fields:
- exposure: the exposure after application of ISR
"""
if bfKernel is not None:
raise ValueError(
"CFHT ISR does not currently support brighter-fatter correction."
)
ccd = ccdExposure.getDetector()
floatExposure = self.convertIntToFloat(ccdExposure)
metadata = floatExposure.getMetadata()
# Detect saturation
# Saturation values recorded in the fits hader is not reliable, try to
# estimate it from the pixel values.
# Find the peak location in the high end part the pixel values'
# histogram and set the saturation level at safe * (peak location)
# where safe is a configurable parameter (typically 0.95)
image = floatExposure.getMaskedImage().getImage()
imageArray = image.getArray()
maxValue = np.max(imageArray)
if maxValue > 60000.0:
hist, bin_edges = np.histogram(imageArray.ravel(),
bins=100,
range=(60000.0, maxValue + 1.0))
saturate = int(self.config.safe * bin_edges[np.argmax(hist)])
else:
saturate = metadata.getScalar("SATURATE")
self.log.info("Saturation set to %d" % saturate)
tempCcd = ccd.rebuild()
tempCcd.clear()
for amp in ccd:
tempAmp = amp.rebuild()
tempAmp.setSaturation(saturate)
if tempAmp.getName() == "A":
tempAmp.setGain(metadata.getScalar("GAINA"))
rdnA = metadata.getScalar("RDNOISEA")
# Check if the noise value is making sense for this amp. If
# not, replace with value stored in RDNOISE slot. This change
# is necessary to process some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnA > 60000.0:
rdnA = metadata.getScalar("RDNOISE")
tempAmp.setReadNoise(rdnA)
elif tempAmp.getName() == "B":
tempAmp.setGain(metadata.getScalar("GAINB"))
rdnB = metadata.getScalar("RDNOISEB")
# Check if the noise value is making sense for this amp.
# If not, replace with value
# stored in RDNOISE slot. This change is necessary to process
# some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnB > 60000.0:
rdnB = metadata.getScalar("RDNOISE")
tempAmp.setReadNoise(rdnB)
else:
raise ValueError("Unexpected amplifier name : %s" %
(amp.getName()))
tempCcd.append(tempAmp)
ccd = tempCcd.finish()
ccdExposure.setDetector(ccd)
return IsrTask.run(self,
ccdExposure=ccdExposure,
bias=bias,
linearizer=linearizer,
dark=dark,
flat=flat,
defects=defects,
fringes=fringes,
**kwds)
def run(self,
ccdExposure,
bias=None,
linearizer=None,
dark=None,
flat=None,
defects=None,
fringes=None,
bfKernel=None):
"""Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
- Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
@param[in] ccdExposure -- lsst.afw.image.exposure of detector data
@param[in] bias -- exposure of bias frame
@param[in] linearizer -- linearizing functor; a subclass of lsst.ip.isr.LinearizeBase
@param[in] dark -- exposure of dark frame
@param[in] flat -- exposure of flatfield
@param[in] defects -- list of detects
@param[in] fringes -- exposure of fringe frame or list of fringe exposure
@param[in] bfKernel - kernel used for brighter-fatter correction; currently unsupported
@return a pipeBase.Struct with fields:
- exposure: the exposure after application of ISR
"""
if bfKernel is not None:
raise ValueError(
"CFHT ISR does not currently support brighter-fatter correction."
)
ccd = ccdExposure.getDetector()
floatExposure = self.convertIntToFloat(ccdExposure)
metadata = floatExposure.getMetadata()
# Detect saturation
# Saturation values recorded in the fits hader is not reliable, try to estimate it from
# the pixel vales
# Find the peak location in the high end part the pixel values' histogram and set the saturation
# level at safe * (peak location) where safe is a configurable parameter (typically 0.95)
image = floatExposure.getMaskedImage().getImage()
imageArray = image.getArray()
maxValue = np.max(imageArray)
if maxValue > 60000.0:
hist, bin_edges = np.histogram(imageArray.ravel(),
bins=100,
range=(60000.0, maxValue + 1.0))
saturate = int(self.config.safe * bin_edges[np.argmax(hist)])
else:
saturate = metadata.get("SATURATE")
self.log.info("Saturation set to %d" % saturate)
for amp in ccd:
amp.setSaturation(saturate)
if amp.getName() == "A":
amp.setGain(metadata.get("GAINA"))
rdnA = metadata.get("RDNOISEA")
# Check if the noise value is making sense for this amp. If not, replace with value
# stored in RDNOISE slot. This change is necessary to process some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnA > 60000.0:
rdnA = metadata.get("RDNOISE")
amp.setReadNoise(rdnA)
elif amp.getName() == "B":
amp.setGain(metadata.get("GAINB"))
rdnB = metadata.get("RDNOISEB")
# Check if the noise value is making sense for this amp. If not, replace with value
# stored in RDNOISE slot. This change is necessary to process some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnB > 60000.0:
rdnB = metadata.get("RDNOISE")
amp.setReadNoise(rdnB)
else:
raise ValueError("Unexpected amplifier name : %s" %
(amp.getName()))
return IsrTask.run(
self,
ccdExposure=ccdExposure,
bias=bias,
linearizer=linearizer,
dark=dark,
flat=flat,
defects=defects,
fringes=fringes,
)
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(
config=getObsTestConfig(IsrTask),
name="isr2"
)
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask),
name="charImage2"
)
calibrateTask = CalibrateTask(
config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema
)
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()]
isrResult1 = IsrTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
icResult1 = CharacterizeImageTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True),
icResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True),
icResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background
)
calResult1 = CalibrateTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent")
)
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True),
calResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background
)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def run(config, inputFiles, weightFiles=None, varianceFiles=None,
returnCalibSources=False, displayResults=[], verbose=False):
#
# Create the tasks
#
schema = afwTable.SourceTable.makeMinimalSchema()
algMetadata = dafBase.PropertyList()
isrTask = IsrTask(config=config.isr)
calibrateTask = CalibrateTask(config=config.calibrate)
sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema)
if config.doDeblend:
if SourceDeblendTask:
sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema)
else:
print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False"
config.doDeblend = False
sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement,
schema=schema, algMetadata=algMetadata)
sourceMeasurementTask.config.doApplyApCorr = 'yes'
#
# Add fields needed to identify stars while calibrating
#
keysToCopy = [(schema.addField(afwTable.Field["Flag"]("calib_detected",
"Source was detected by calibrate")), None)]
for key in calibrateTask.getCalibKeys():
keysToCopy.append((schema.addField(calibrateTask.schema.find(key).field), key))
exposureDict = {}; calibSourcesDict = {}; sourcesDict = {}
for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles):
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# read the data
#
if verbose:
print "Reading %s" % inputFile
exposure = makeExposure(inputFile, weightFile, varianceFile,
config.badPixelValue, config.variance)
#
if config.interpPlanes:
import lsst.ip.isr as ipIsr
defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes,
growFootprints=0)
isrTask.run(exposure, defects=defects)
#
# process the data
#
if config.doCalibrate:
result = calibrateTask.run(exposure)
exposure, calibSources = result.exposure, result.sources
else:
calibSources = None
if not exposure.getPsf():
calibrateTask.installInitialPsf(exposure)
exposureDict[inputFile] = exposure
calibSourcesDict[inputFile] = calibSources if returnCalibSources else None
result = sourceDetectionTask.run(tab, exposure)
sources = result.sources
sourcesDict[inputFile] = sources
if config.doDeblend:
sourceDeblendTask.run(exposure, sources, exposure.getPsf())
sourceMeasurementTask.measure(exposure, sources)
if verbose:
print "Detected %d objects" % len(sources)
propagateCalibFlags(keysToCopy, calibSources, sources)
if displayResults: # display results of processing (see also --debug argparse option)
showApertures = "showApertures".upper() in displayResults
showShapes = "showShapes".upper() in displayResults
display = afwDisplay.getDisplay(frame=1)
if algMetadata.exists("base_CircularApertureFlux_radii"):
radii = algMetadata.get("base_CircularApertureFlux_radii")
else:
radii = []
display.mtv(exposure, title=os.path.split(inputFile)[1])
with display.Buffering():
for s in sources:
xy = s.getCentroid()
display.dot('+', *xy,
ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN)
if showShapes:
display.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
if showApertures:
for radius in radii:
display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
return exposureDict, calibSourcesDict, sourcesDict
#!/usr/bin/env python # # LSST Data Management System # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. # # This product includes software developed by the # LSST Project (http://www.lsst.org/). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # from lsst.ip.isr import IsrTask IsrTask.parseAndRun()
def run(self,
ccdExposure,
*,
camera=None,
bias=None,
dark=None,
flat=None,
defects=None,
linearizer=None,
crosstalk=None,
bfKernel=None,
bfGains=None,
ptc=None,
crosstalkSources=None,
isrBaseConfig=None):
"""Run isr and cosmic ray repair using, doing as much isr as possible.
Retrieves as many calibration products as are available, and runs isr
with those settings enabled, but always returns an assembled image at
a minimum. Then performs cosmic ray repair if configured to.
Parameters
----------
ccdExposure : `lsst.afw.image.Exposure`
The raw exposure that is to be run through ISR. The
exposure is modified by this method.
camera : `lsst.afw.cameraGeom.Camera`, optional
The camera geometry for this exposure. Required if
one or more of ``ccdExposure``, ``bias``, ``dark``, or
``flat`` does not have an associated detector.
bias : `lsst.afw.image.Exposure`, optional
Bias calibration frame.
linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
Functor for linearization.
crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
Calibration for crosstalk.
crosstalkSources : `list`, optional
List of possible crosstalk sources.
dark : `lsst.afw.image.Exposure`, optional
Dark calibration frame.
flat : `lsst.afw.image.Exposure`, optional
Flat calibration frame.
ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
Photon transfer curve dataset, with, e.g., gains
and read noise.
bfKernel : `numpy.ndarray`, optional
Brighter-fatter kernel.
bfGains : `dict` of `float`, optional
Gains used to override the detector's nominal gains for the
brighter-fatter correction. A dict keyed by amplifier name for
the detector in question.
defects : `lsst.ip.isr.Defects`, optional
List of defects.
fringes : `lsst.pipe.base.Struct`, optional
Struct containing the fringe correction data, with
elements:
- ``fringes``: fringe calibration frame (`afw.image.Exposure`)
- ``seed``: random seed derived from the ccdExposureId for random
number generator (`uint32`)
opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
A ``TransmissionCurve`` that represents the throughput of the,
optics, to be evaluated in focal-plane coordinates.
filterTransmission : `lsst.afw.image.TransmissionCurve`
A ``TransmissionCurve`` that represents the throughput of the
filter itself, to be evaluated in focal-plane coordinates.
sensorTransmission : `lsst.afw.image.TransmissionCurve`
A ``TransmissionCurve`` that represents the throughput of the
sensor itself, to be evaluated in post-assembly trimmed detector
coordinates.
atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
A ``TransmissionCurve`` that represents the throughput of the
atmosphere, assumed to be spatially constant.
detectorNum : `int`, optional
The integer number for the detector to process.
strayLightData : `object`, optional
Opaque object containing calibration information for stray-light
correction. If `None`, no correction will be performed.
illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
Illumination correction image.
isrBaseConfig : `lsst.ip.isr.IsrTaskConfig`, optional
An isrTask config to act as the base configuration. Options which
involve applying a calibration product are ignored, but this allows
for the configuration of e.g. the number of overscan columns.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with component:
- ``exposure`` : `afw.image.Exposure`
The ISRed and cosmic-ray-repaired exposure.
"""
isrConfig = isrBaseConfig if isrBaseConfig else IsrTask.ConfigClass()
isrConfig.doBias = False
isrConfig.doDark = False
isrConfig.doFlat = False
isrConfig.doFringe = False
isrConfig.doDefect = False
isrConfig.doLinearize = False
isrConfig.doCrosstalk = False
isrConfig.doBrighterFatter = False
isrConfig.usePtcGains = False
if bias:
isrConfig.doBias = True
self.log.info("Running with bias correction")
if dark:
isrConfig.doDark = True
self.log.info("Running with dark correction")
if flat:
isrConfig.doFlat = True
self.log.info("Running with flat correction")
# TODO: deal with fringes here
if defects:
isrConfig.doDefect = True
self.log.info("Running with defect correction")
if linearizer:
isrConfig.doLinearize = True
self.log.info("Running with linearity correction")
if crosstalk:
isrConfig.doCrosstalk = True
self.log.info("Running with crosstalk correction")
if bfKernel:
isrConfig.doBrighterFatter = True
self.log.info("Running with brighter-fatter correction")
if ptc:
isrConfig.usePtcGains = True
self.log.info("Running with ptc correction")
isrConfig.doWrite = False
isrTask = IsrTask(config=isrConfig)
result = isrTask.run(
ccdExposure,
camera=camera,
bias=bias,
dark=dark,
flat=flat,
# fringes=pipeBase.Struct(fringes=None),
defects=defects,
linearizer=linearizer,
crosstalk=crosstalk,
bfKernel=bfKernel,
bfGains=bfGains,
ptc=ptc,
crosstalkSources=crosstalkSources,
isGen3=True,
)
postIsr = result.exposure
if self.config.doRepairCosmics:
try: # can fail due to too many CRs detected, and we always want an exposure back
self.log.info("Repairing cosmics...")
if postIsr.getPsf() is None:
installPsfTask = InstallGaussianPsfTask()
installPsfTask.run(postIsr)
# TODO: try adding a reasonably wide Gaussian as a temp PSF
# and then just running repairTask on its own without any
# imChar. It should work, and be faster.
repairConfig = CharacterizeImageTask.ConfigClass()
repairConfig.doMeasurePsf = False
repairConfig.doApCorr = False
repairConfig.doDeblend = False
repairConfig.doWrite = False
repairConfig.repair.cosmicray.nCrPixelMax = 200000
repairTask = CharacterizeImageTask(config=repairConfig)
repairTask.repair.run(postIsr)
except Exception as e:
self.log.warning(f"During CR repair caught: {e}")
# exposure is returned for convenience to mimic isrTask's API.
return pipeBase.Struct(exposure=postIsr, outputExposure=postIsr)