def testRunWithAddDistortionModel(self):
"""Test IsrTask.run with config.doAddDistortionModel true"""
isrConfig = self.makeMinimalIsrConfig()
isrConfig.doAddDistortionModel = True
isrTask = IsrTask(config=isrConfig)
with self.assertRaises(RuntimeError):
# the camera argument is required
isrTask.run(ccdExposure=self.exposure)
exposure = isrTask.run(ccdExposure=self.exposure, camera=self.camera).exposure
desiredWcs = self.makeDesiredDistortedWcs()
self.assertWcsAlmostEqualOverBBox(desiredWcs, exposure.getWcs(), self.bbox)
def testRunWithoutAddDistortionModel(self):
"""Test IsrTask.run with config.doAddDistortionModel false"""
isrConfig = self.makeMinimalIsrConfig()
isrTask = IsrTask(config=isrConfig)
# the camera argument is not needed
exposure = isrTask.run(ccdExposure=self.exposure).exposure
self.assertEqual(self.wcs, exposure.getWcs())
# and the camera argument is ignored if provided
exposure2 = isrTask.run(ccdExposure=self.exposure, camera=self.camera).exposure
self.assertEqual(self.wcs, exposure2.getWcs())
def testRunWithoutAddDistortionModel(self):
"""Test IsrTask.run with config.doAddDistortionModel false"""
isrConfig = self.makeMinimalIsrConfig()
isrTask = IsrTask(config=isrConfig)
# the camera argument is not needed
exposure = isrTask.run(ccdExposure=self.exposure).exposure
self.assertEqual(self.wcs, exposure.getWcs())
# and the camera argument is ignored if provided
exposure2 = isrTask.run(ccdExposure=self.exposure,
camera=self.camera).exposure
self.assertEqual(self.wcs, exposure2.getWcs())
def testRunWithAddDistortionModel(self):
"""Test IsrTask.run with config.doAddDistortionModel true"""
isrConfig = self.makeMinimalIsrConfig()
isrConfig.doAddDistortionModel = True
isrTask = IsrTask(config=isrConfig)
with self.assertRaises(RuntimeError):
# the camera argument is required
isrTask.run(ccdExposure=self.exposure)
exposure = isrTask.run(ccdExposure=self.exposure,
camera=self.camera).exposure
desiredWcs = self.makeDesiredDistortedWcs()
self.assertWcsAlmostEqualOverBBox(desiredWcs, exposure.getWcs(),
self.bbox)
def execute(self, dataRef):
"""!Apply common instrument signature correction algorithms to a raw frame
@param dataRef: butler data reference
@return a pipeBase Struct containing:
- exposure
similar to IsrTask.runDataRef()
"""
self.log.info("Performing Super ISR on sensor data ID %s" %
(dataRef.dataId, ))
# IsrTask.runDataRef includes these three steps
self.log.info("Reading input data using dataRef")
inputData = self.read_input_data(dataRef)
self.log.info(
"Running operations. The run() method should not take anything Butler"
)
result = IsrTask.run(IsrTask(self.config), **inputData.getDict())
self.log.info("Writing output data using dataRef")
self.write_output_data(dataRef, result)
return result
def runIsr():
'''Run the task to do ISR on a ccd'''
#Create the isr task with modified config
isrConfig = IsrTask.ConfigClass()
isrConfig.doBias = False #We didn't make a zero frame
isrConfig.doDark = True
isrConfig.doFlat = True
isrConfig.doFringe = False #There is no fringe frame for this example
isrConfig.assembleCcd.setGain = False
isrTask = IsrTask(config=isrConfig)
#Make raw, flat and dark exposures
DARKVAL = 2. #e-/sec
OSCAN = 1000. #DN
GRADIENT = .10
EXPTIME = 15 #seconds
DARKEXPTIME = 40. #seconds
darkExposure = exampleUtils.makeDark(DARKVAL, DARKEXPTIME)
flatExposure = exampleUtils.makeFlat(GRADIENT)
rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME)
output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure)
return output.exposure
def runIsr():
'''Run the task to do ISR on a ccd'''
#Create the isr task with modified config
isrConfig = IsrTask.ConfigClass()
isrConfig.doBias = False #We didn't make a zero frame
isrConfig.doDark = True
isrConfig.doFlat = True
isrConfig.doFringe = False #There is no fringe frame for this example
isrConfig.assembleCcd.doRenorm = False #We'll take care of gain in the flats
isrConfig.assembleCcd.setGain = False
isrTask = IsrTask(config=isrConfig)
#Make raw, flat and dark exposures
DARKVAL = 2. #e-/sec
OSCAN = 1000. #DN
GRADIENT = .10
EXPTIME = 15 #seconds
DARKEXPTIME = 40. #seconds
darkExposure = exampleUtils.makeDark(DARKVAL, DARKEXPTIME)
flatExposure = exampleUtils.makeFlat(GRADIENT)
rawExposure = exampleUtils.makeRaw(DARKVAL, OSCAN, GRADIENT, EXPTIME)
output = isrTask.run(rawExposure, dark=darkExposure, flat=flatExposure)
return output.exposure
def processExposure(exposure,
bias=None,
defects=None,
repair=False,
repo=None):
from lsst.ip.isr import IsrTask
import lsst.daf.persistence as dafPersist
import time
config = IsrTask.ConfigClass()
if (bias == None or defects == None) and repo == None:
raise (AttributeError(
'Repo keyword not set and is required to find bias/defect frames'))
else:
butler = dafPersist.Butler(repo)
if False:
config.overscanFitType = "AKIMA_SPLINE"
config.overscanOrder = 5
else:
config.overscanFitType = "POLY"
config.overscanOrder = 3
config.doBias = True
if config.doBias == True and bias == None:
bias = butler.get(
'bias',
visit=exposure.getInfo().getVisitInfo().getExposureId(),
)
config.doDark = False
config.doFringe = False
config.doFlat = False
config.doLinearize = False
# TODO: should be able to run this but there are issues with defects in the stack at the moment
config.doDefect = True if repair == True else False
if (config.doDefect == True and bias == None):
butler = dafPersist.Butler(repo)
defects = butler.get(
'defects',
visit=exposure.getInfo().getVisitInfo().getExposureId(),
)
config.doAddDistortionModel = False
config.doSaturationInterpolation = False
config.overscanNumLeadingColumnsToSkip = 20
isrTask = IsrTask(config=config)
# Run ISR
start = time.time()
isr_corrected_exposure = isrTask.run(exposure, bias=bias,
defects=defects).exposure
end = time.time()
print('Time to perform image ISR was {0:2f} [s]'.format(end - start))
return (isr_corrected_exposure)
def run(self, ccdExposure, bias=None, dark=None, flat=None, defects=None, fringes=None):
"""Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
- Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
@param[in] ccdExposure -- lsst.afw.image.exposure of detector data
@param[in] bias -- exposure of bias frame
@param[in] dark -- exposure of dark frame
@param[in] flat -- exposure of flatfield
@param[in] defects -- list of detects
@param[in] fringes -- exposure of fringe frame or list of fringe exposure
@return a pipeBase.Struct with fields:
- exposure: the exposure after application of ISR
"""
ccd = ccdExposure.getDetector()
floatExposure = self.convertIntToFloat(ccdExposure)
metadata = floatExposure.getMetadata()
# Detect saturation
# Saturation values recorded in the fits hader is not reliable, try to estimate it from the pixel vales
# Find the peak location in the high end part the pixel values' histogram and set the saturation level at
# safe * (peak location) where safe is a configurable parameter (typically 0.95)
image = floatExposure.getMaskedImage().getImage()
imageArray = image.getArray()
maxValue = np.max(imageArray)
if maxValue > 60000.0:
hist, bin_edges = np.histogram(imageArray.ravel(), bins=100, range=(60000.0, maxValue + 1.0))
saturate = int(self.config.safe * bin_edges[np.argmax(hist)])
else:
saturate = metadata.get("SATURATE")
self.log.info("Saturation set to %d" % saturate)
for amp in ccd:
amp.setSaturation(saturate)
if amp.getName() == "A":
amp.setGain(metadata.get("GAINA"))
amp.setReadNoise(metadata.get("RDNOISEA"))
elif amp.getName() == "B":
amp.setGain(metadata.get("GAINB"))
amp.setReadNoise(metadata.get("RDNOISEB"))
else:
raise ValueError("Unexpected amplifier name : %s" % (amp.getName()))
return IsrTask.run(
self, ccdExposure=ccdExposure, bias=bias, dark=dark, flat=flat, defects=defects, fringes=fringes
)
def run(self,
ccdExposure,
*,
camera=None,
bias=None,
dark=None,
flat=None,
defects=None,
linearizer=None,
crosstalk=None,
bfKernel=None,
bfGains=None,
ptc=None,
crosstalkSources=None,
isrBaseConfig=None):
"""Run isr and cosmic ray repair using, doing as much isr as possible.
Retrieves as many calibration products as are available, and runs isr
with those settings enabled, but always returns an assembled image at
a minimum. Then performs cosmic ray repair if configured to.
Parameters
----------
ccdExposure : `lsst.afw.image.Exposure`
The raw exposure that is to be run through ISR. The
exposure is modified by this method.
camera : `lsst.afw.cameraGeom.Camera`, optional
The camera geometry for this exposure. Required if
one or more of ``ccdExposure``, ``bias``, ``dark``, or
``flat`` does not have an associated detector.
bias : `lsst.afw.image.Exposure`, optional
Bias calibration frame.
linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
Functor for linearization.
crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
Calibration for crosstalk.
crosstalkSources : `list`, optional
List of possible crosstalk sources.
dark : `lsst.afw.image.Exposure`, optional
Dark calibration frame.
flat : `lsst.afw.image.Exposure`, optional
Flat calibration frame.
ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
Photon transfer curve dataset, with, e.g., gains
and read noise.
bfKernel : `numpy.ndarray`, optional
Brighter-fatter kernel.
bfGains : `dict` of `float`, optional
Gains used to override the detector's nominal gains for the
brighter-fatter correction. A dict keyed by amplifier name for
the detector in question.
defects : `lsst.ip.isr.Defects`, optional
List of defects.
fringes : `lsst.pipe.base.Struct`, optional
Struct containing the fringe correction data, with
elements:
- ``fringes``: fringe calibration frame (`afw.image.Exposure`)
- ``seed``: random seed derived from the ccdExposureId for random
number generator (`uint32`)
opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
A ``TransmissionCurve`` that represents the throughput of the,
optics, to be evaluated in focal-plane coordinates.
filterTransmission : `lsst.afw.image.TransmissionCurve`
A ``TransmissionCurve`` that represents the throughput of the
filter itself, to be evaluated in focal-plane coordinates.
sensorTransmission : `lsst.afw.image.TransmissionCurve`
A ``TransmissionCurve`` that represents the throughput of the
sensor itself, to be evaluated in post-assembly trimmed detector
coordinates.
atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
A ``TransmissionCurve`` that represents the throughput of the
atmosphere, assumed to be spatially constant.
detectorNum : `int`, optional
The integer number for the detector to process.
strayLightData : `object`, optional
Opaque object containing calibration information for stray-light
correction. If `None`, no correction will be performed.
illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
Illumination correction image.
isrBaseConfig : `lsst.ip.isr.IsrTaskConfig`, optional
An isrTask config to act as the base configuration. Options which
involve applying a calibration product are ignored, but this allows
for the configuration of e.g. the number of overscan columns.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with component:
- ``exposure`` : `afw.image.Exposure`
The ISRed and cosmic-ray-repaired exposure.
"""
isrConfig = isrBaseConfig if isrBaseConfig else IsrTask.ConfigClass()
isrConfig.doBias = False
isrConfig.doDark = False
isrConfig.doFlat = False
isrConfig.doFringe = False
isrConfig.doDefect = False
isrConfig.doLinearize = False
isrConfig.doCrosstalk = False
isrConfig.doBrighterFatter = False
isrConfig.usePtcGains = False
if bias:
isrConfig.doBias = True
self.log.info("Running with bias correction")
if dark:
isrConfig.doDark = True
self.log.info("Running with dark correction")
if flat:
isrConfig.doFlat = True
self.log.info("Running with flat correction")
# TODO: deal with fringes here
if defects:
isrConfig.doDefect = True
self.log.info("Running with defect correction")
if linearizer:
isrConfig.doLinearize = True
self.log.info("Running with linearity correction")
if crosstalk:
isrConfig.doCrosstalk = True
self.log.info("Running with crosstalk correction")
if bfKernel:
isrConfig.doBrighterFatter = True
self.log.info("Running with brighter-fatter correction")
if ptc:
isrConfig.usePtcGains = True
self.log.info("Running with ptc correction")
isrConfig.doWrite = False
isrTask = IsrTask(config=isrConfig)
result = isrTask.run(
ccdExposure,
camera=camera,
bias=bias,
dark=dark,
flat=flat,
# fringes=pipeBase.Struct(fringes=None),
defects=defects,
linearizer=linearizer,
crosstalk=crosstalk,
bfKernel=bfKernel,
bfGains=bfGains,
ptc=ptc,
crosstalkSources=crosstalkSources,
isGen3=True,
)
postIsr = result.exposure
if self.config.doRepairCosmics:
try: # can fail due to too many CRs detected, and we always want an exposure back
self.log.info("Repairing cosmics...")
if postIsr.getPsf() is None:
installPsfTask = InstallGaussianPsfTask()
installPsfTask.run(postIsr)
# TODO: try adding a reasonably wide Gaussian as a temp PSF
# and then just running repairTask on its own without any
# imChar. It should work, and be faster.
repairConfig = CharacterizeImageTask.ConfigClass()
repairConfig.doMeasurePsf = False
repairConfig.doApCorr = False
repairConfig.doDeblend = False
repairConfig.doWrite = False
repairConfig.repair.cosmicray.nCrPixelMax = 200000
repairTask = CharacterizeImageTask(config=repairConfig)
repairTask.repair.run(postIsr)
except Exception as e:
self.log.warning(f"During CR repair caught: {e}")
# exposure is returned for convenience to mimic isrTask's API.
return pipeBase.Struct(exposure=postIsr, outputExposure=postIsr)
def run(config, inputFiles, weightFiles=None, varianceFiles=None,
returnCalibSources=False, displayResults=[], verbose=False):
#
# Create the tasks
#
schema = afwTable.SourceTable.makeMinimalSchema()
algMetadata = dafBase.PropertyList()
isrTask = IsrTask(config=config.isr)
calibrateTask = CalibrateTask(config=config.calibrate)
sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema)
if config.doDeblend:
if SourceDeblendTask:
sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema)
else:
print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False"
config.doDeblend = False
sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement,
schema=schema, algMetadata=algMetadata)
sourceMeasurementTask.config.doApplyApCorr = 'yes'
#
# Add fields needed to identify stars while calibrating
#
keysToCopy = [(schema.addField(afwTable.Field["Flag"]("calib_detected",
"Source was detected by calibrate")), None)]
for key in calibrateTask.getCalibKeys():
keysToCopy.append((schema.addField(calibrateTask.schema.find(key).field), key))
exposureDict = {}; calibSourcesDict = {}; sourcesDict = {}
for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles):
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# read the data
#
if verbose:
print "Reading %s" % inputFile
exposure = makeExposure(inputFile, weightFile, varianceFile,
config.badPixelValue, config.variance)
#
if config.interpPlanes:
import lsst.ip.isr as ipIsr
defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes,
growFootprints=0)
isrTask.run(exposure, defects=defects)
#
# process the data
#
if config.doCalibrate:
result = calibrateTask.run(exposure)
exposure, calibSources = result.exposure, result.sources
else:
calibSources = None
if not exposure.getPsf():
calibrateTask.installInitialPsf(exposure)
exposureDict[inputFile] = exposure
calibSourcesDict[inputFile] = calibSources if returnCalibSources else None
result = sourceDetectionTask.run(tab, exposure)
sources = result.sources
sourcesDict[inputFile] = sources
if config.doDeblend:
sourceDeblendTask.run(exposure, sources, exposure.getPsf())
sourceMeasurementTask.measure(exposure, sources)
if verbose:
print "Detected %d objects" % len(sources)
propagateCalibFlags(keysToCopy, calibSources, sources)
if displayResults: # display results of processing (see also --debug argparse option)
showApertures = "showApertures".upper() in displayResults
showShapes = "showShapes".upper() in displayResults
display = afwDisplay.getDisplay(frame=1)
if algMetadata.exists("base_CircularApertureFlux_radii"):
radii = algMetadata.get("base_CircularApertureFlux_radii")
else:
radii = []
display.mtv(exposure, title=os.path.split(inputFile)[1])
with display.Buffering():
for s in sources:
xy = s.getCentroid()
display.dot('+', *xy,
ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN)
if showShapes:
display.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
if showApertures:
for radius in radii:
display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
return exposureDict, calibSourcesDict, sourcesDict
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp(
) if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2")
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask), name="charImage2")
calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema)
try:
dataId = dict(visit=1)
dataIdStrList = [
"%s=%s" % (key, val) for key, val in dataId.items()
]
isrResult1 = IsrTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get(
"postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
icResult1 = CharacterizeImageTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure,
exposureIdInfo=exposureIdInfo)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get(
"icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId,
immediate=True),
icResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground",
dataId,
immediate=True),
icResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background)
calResult1 = CalibrateTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat,
exposureIdInfo=exposureIdInfo,
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get(
"calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent"))
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground",
dataId,
immediate=True),
calResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" %
(OutputName, ))
def run(config,
inputFiles,
weightFiles=None,
varianceFiles=None,
returnCalibSources=False,
displayResults=[],
verbose=False):
#
# Create the tasks
#
schema = afwTable.SourceTable.makeMinimalSchema()
algMetadata = dafBase.PropertyList()
isrTask = IsrTask(config=config.isr)
charImageTask = CharacterizeImageTask(None, config=config.charImage)
sourceDetectionTask = SourceDetectionTask(config=config.detection,
schema=schema)
if config.doDeblend:
if SourceDeblendTask:
sourceDeblendTask = SourceDeblendTask(config=config.deblend,
schema=schema)
else:
print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False"
config.doDeblend = False
sourceMeasurementTask = SingleFrameMeasurementTask(
schema=schema, config=config.measurement, algMetadata=algMetadata)
keysToCopy = []
for key in [
charImageTask.measurePsf.reservedKey,
charImageTask.measurePsf.usedKey,
]:
keysToCopy.append(
(schema.addField(charImageTask.schema.find(key).field), key))
exposureDict = {}
calibSourcesDict = {}
sourcesDict = {}
for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles,
varianceFiles):
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# read the data
#
if verbose:
print "Reading %s" % inputFile
exposure = makeExposure(inputFile, weightFile, varianceFile,
config.badPixelValue, config.variance)
#
if config.interpPlanes:
import lsst.ip.isr as ipIsr
defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(),
config.interpPlanes,
growFootprints=0)
isrTask.run(exposure, defects=defects)
#
# process the data
#
if config.doCalibrate:
result = charImageTask.characterize(exposure)
exposure, calibSources = result.exposure, result.sourceCat
else:
calibSources = None
if not exposure.getPsf():
charImageTask.installSimplePsf.run(exposure)
exposureDict[inputFile] = exposure
calibSourcesDict[
inputFile] = calibSources if returnCalibSources else None
result = sourceDetectionTask.run(tab, exposure)
sources = result.sources
sourcesDict[inputFile] = sources
if config.doDeblend:
sourceDeblendTask.run(exposure, sources)
sourceMeasurementTask.measure(exposure, sources)
if verbose:
print "Detected %d objects" % len(sources)
propagatePsfFlags(keysToCopy, calibSources, sources)
if displayResults: # display results of processing (see also --debug argparse option)
showApertures = "showApertures".upper() in displayResults
showPSFs = "showPSFs".upper() in displayResults
showShapes = "showShapes".upper() in displayResults
display = afwDisplay.getDisplay(frame=1)
if algMetadata.exists("base_CircularApertureFlux_radii"):
radii = algMetadata.get("base_CircularApertureFlux_radii")
else:
radii = []
display.mtv(exposure, title=os.path.split(inputFile)[1])
with display.Buffering():
for s in sources:
xy = s.getCentroid()
display.dot('+',
*xy,
ctype=afwDisplay.CYAN if
s.get("flags_negative") else afwDisplay.GREEN)
if showPSFs and (s.get("calib_psfUsed")
or s.get("calib_psfReserved")):
display.dot(
'o',
*xy,
size=10,
ctype=afwDisplay.GREEN
if s.get("calib_psfUsed") else afwDisplay.YELLOW)
if showShapes:
display.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
if showApertures:
for radius in radii:
display.dot('o',
*xy,
size=radius,
ctype=afwDisplay.YELLOW)
return exposureDict, calibSourcesDict, sourcesDict
def run(self,
ccdExposure,
bias=None,
linearizer=None,
dark=None,
flat=None,
defects=None,
fringes=None,
bfKernel=None,
**kwds):
"""Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
- Persist the ISR-corrected exposure as "postISRCCD" if
config.doWrite is True
Parameters
----------
ccdExposure : `lsst.afw.image.Exposure`
Detector data.
bias : `lsst.afw.image.exposure`
Exposure of bias frame.
linearizer : `lsst.ip.isr.LinearizeBase` callable
Linearizing functor; a subclass of lsst.ip.isr.LinearizeBase.
dark : `lsst.afw.image.exposure`
Exposure of dark frame.
flat : `lsst.afw.image.exposure`
Exposure of flatfield.
defects : `list`
list of detects
fringes : `lsst.afw.image.Exposure` or list `lsst.afw.image.Exposure`
exposure of fringe frame or list of fringe exposure
bfKernel : None
kernel used for brighter-fatter correction; currently unsupported
**kwds : `dict`
additional kwargs forwarded to IsrTask.run.
Returns
-------
struct : `lsst.pipe.base.Struct` with fields:
- exposure: the exposure after application of ISR
"""
if bfKernel is not None:
raise ValueError(
"CFHT ISR does not currently support brighter-fatter correction."
)
ccd = ccdExposure.getDetector()
floatExposure = self.convertIntToFloat(ccdExposure)
metadata = floatExposure.getMetadata()
# Detect saturation
# Saturation values recorded in the fits hader is not reliable, try to
# estimate it from the pixel values.
# Find the peak location in the high end part the pixel values'
# histogram and set the saturation level at safe * (peak location)
# where safe is a configurable parameter (typically 0.95)
image = floatExposure.getMaskedImage().getImage()
imageArray = image.getArray()
maxValue = np.max(imageArray)
if maxValue > 60000.0:
hist, bin_edges = np.histogram(imageArray.ravel(),
bins=100,
range=(60000.0, maxValue + 1.0))
saturate = int(self.config.safe * bin_edges[np.argmax(hist)])
else:
saturate = metadata.getScalar("SATURATE")
self.log.info("Saturation set to %d" % saturate)
tempCcd = ccd.rebuild()
tempCcd.clear()
for amp in ccd:
tempAmp = amp.rebuild()
tempAmp.setSaturation(saturate)
if tempAmp.getName() == "A":
tempAmp.setGain(metadata.getScalar("GAINA"))
rdnA = metadata.getScalar("RDNOISEA")
# Check if the noise value is making sense for this amp. If
# not, replace with value stored in RDNOISE slot. This change
# is necessary to process some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnA > 60000.0:
rdnA = metadata.getScalar("RDNOISE")
tempAmp.setReadNoise(rdnA)
elif tempAmp.getName() == "B":
tempAmp.setGain(metadata.getScalar("GAINB"))
rdnB = metadata.getScalar("RDNOISEB")
# Check if the noise value is making sense for this amp.
# If not, replace with value
# stored in RDNOISE slot. This change is necessary to process
# some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnB > 60000.0:
rdnB = metadata.getScalar("RDNOISE")
tempAmp.setReadNoise(rdnB)
else:
raise ValueError("Unexpected amplifier name : %s" %
(amp.getName()))
tempCcd.append(tempAmp)
ccd = tempCcd.finish()
ccdExposure.setDetector(ccd)
return IsrTask.run(self,
ccdExposure=ccdExposure,
bias=bias,
linearizer=linearizer,
dark=dark,
flat=flat,
defects=defects,
fringes=fringes,
**kwds)
def run(self,
ccdExposure,
bias=None,
linearizer=None,
dark=None,
flat=None,
defects=None,
fringes=None,
bfKernel=None):
"""Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
- Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True
@param[in] ccdExposure -- lsst.afw.image.exposure of detector data
@param[in] bias -- exposure of bias frame
@param[in] linearizer -- linearizing functor; a subclass of lsst.ip.isr.LinearizeBase
@param[in] dark -- exposure of dark frame
@param[in] flat -- exposure of flatfield
@param[in] defects -- list of detects
@param[in] fringes -- exposure of fringe frame or list of fringe exposure
@param[in] bfKernel - kernel used for brighter-fatter correction; currently unsupported
@return a pipeBase.Struct with fields:
- exposure: the exposure after application of ISR
"""
if bfKernel is not None:
raise ValueError(
"CFHT ISR does not currently support brighter-fatter correction."
)
ccd = ccdExposure.getDetector()
floatExposure = self.convertIntToFloat(ccdExposure)
metadata = floatExposure.getMetadata()
# Detect saturation
# Saturation values recorded in the fits hader is not reliable, try to estimate it from
# the pixel vales
# Find the peak location in the high end part the pixel values' histogram and set the saturation
# level at safe * (peak location) where safe is a configurable parameter (typically 0.95)
image = floatExposure.getMaskedImage().getImage()
imageArray = image.getArray()
maxValue = np.max(imageArray)
if maxValue > 60000.0:
hist, bin_edges = np.histogram(imageArray.ravel(),
bins=100,
range=(60000.0, maxValue + 1.0))
saturate = int(self.config.safe * bin_edges[np.argmax(hist)])
else:
saturate = metadata.get("SATURATE")
self.log.info("Saturation set to %d" % saturate)
for amp in ccd:
amp.setSaturation(saturate)
if amp.getName() == "A":
amp.setGain(metadata.get("GAINA"))
rdnA = metadata.get("RDNOISEA")
# Check if the noise value is making sense for this amp. If not, replace with value
# stored in RDNOISE slot. This change is necessary to process some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnA > 60000.0:
rdnA = metadata.get("RDNOISE")
amp.setReadNoise(rdnA)
elif amp.getName() == "B":
amp.setGain(metadata.get("GAINB"))
rdnB = metadata.get("RDNOISEB")
# Check if the noise value is making sense for this amp. If not, replace with value
# stored in RDNOISE slot. This change is necessary to process some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnB > 60000.0:
rdnB = metadata.get("RDNOISE")
amp.setReadNoise(rdnB)
else:
raise ValueError("Unexpected amplifier name : %s" %
(amp.getName()))
return IsrTask.run(
self,
ccdExposure=ccdExposure,
bias=bias,
linearizer=linearizer,
dark=dark,
flat=flat,
defects=defects,
fringes=fringes,
)
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(
config=getObsTestConfig(IsrTask),
name="isr2"
)
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask),
name="charImage2"
)
calibrateTask = CalibrateTask(
config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema
)
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()]
isrResult1 = IsrTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
icResult1 = CharacterizeImageTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True),
icResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True),
icResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background
)
calResult1 = CalibrateTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent")
)
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True),
calResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background
)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None,
fringes=None, bfKernel=None, camera=None, **kwds):
"""Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
- Persist the ISR-corrected exposure as "postISRCCD" if
config.doWrite is True
Parameters
----------
ccdExposure : `lsst.afw.image.Exposure`
Detector data.
bias : `lsst.afw.image.exposure`
Exposure of bias frame.
linearizer : `lsst.ip.isr.LinearizeBase` callable
Linearizing functor; a subclass of lsst.ip.isr.LinearizeBase.
dark : `lsst.afw.image.exposure`
Exposure of dark frame.
flat : `lsst.afw.image.exposure`
Exposure of flatfield.
defects : `list`
list of detects
fringes : `lsst.afw.image.exposure` or `list` of `lsst.afw.image.exposure`
exposure of fringe frame or list of fringe exposure
bfKernel : None
kernel used for brighter-fatter correction; currently unsupported
camera : `lsst.afw.cameraGeom.Camera`
Camera geometry, used by addDistortionModel.
**kwds : `dict`
additional kwargs forwarded to IsrTask.run.
Returns
-------
struct : `lsst.pipe.base.Struct` with fields:
- exposure: the exposure after application of ISR
"""
if bfKernel is not None:
raise ValueError("CFHT ISR does not currently support brighter-fatter correction.")
ccd = ccdExposure.getDetector()
floatExposure = self.convertIntToFloat(ccdExposure)
metadata = floatExposure.getMetadata()
# Detect saturation
# Saturation values recorded in the fits hader is not reliable, try to
# estimate it from the pixel values.
# Find the peak location in the high end part the pixel values'
# histogram and set the saturation level at safe * (peak location)
# where safe is a configurable parameter (typically 0.95)
image = floatExposure.getMaskedImage().getImage()
imageArray = image.getArray()
maxValue = np.max(imageArray)
if maxValue > 60000.0:
hist, bin_edges = np.histogram(imageArray.ravel(), bins=100, range=(60000.0, maxValue+1.0))
saturate = int(self.config.safe*bin_edges[np.argmax(hist)])
else:
saturate = metadata.getScalar("SATURATE")
self.log.info("Saturation set to %d" % saturate)
for amp in ccd:
amp.setSaturation(saturate)
if amp.getName() == "A":
amp.setGain(metadata.getScalar("GAINA"))
rdnA = metadata.getScalar("RDNOISEA")
# Check if the noise value is making sense for this amp. If
# not, replace with value stored in RDNOISE slot. This change
# is necessary to process some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnA > 60000.0:
rdnA = metadata.getScalar("RDNOISE")
amp.setReadNoise(rdnA)
elif amp.getName() == "B":
amp.setGain(metadata.getScalar("GAINB"))
rdnB = metadata.getScalar("RDNOISEB")
# Check if the noise value is making sense for this amp.
# If not, replace with value
# stored in RDNOISE slot. This change is necessary to process
# some old CFHT images
# (visit : 7xxxxx) where RDNOISEA/B = 65535
if rdnB > 60000.0:
rdnB = metadata.getScalar("RDNOISE")
amp.setReadNoise(rdnB)
else:
raise ValueError("Unexpected amplifier name : %s"%(amp.getName()))
return IsrTask.run(self,
ccdExposure=ccdExposure,
bias=bias,
linearizer=linearizer,
dark=dark,
flat=flat,
defects=defects,
fringes=fringes,
camera=camera,
**kwds
)
