def read_input_data(self, dataRef):
"""Read needed data thru Butler in a pipeBase.Struct based on config"""
ccdExp = dataRef.get('raw', immediate=True)
isrData = IsrTask.readIsrData(IsrTask(self.config), dataRef, ccdExp)
isrDataDict = isrData.getDict()
return pipeBase.Struct(ccdExposure=ccdExp,
bias=isrDataDict['bias'],
linearizer=isrDataDict['linearizer'],
dark=isrDataDict['dark'],
flat=isrDataDict['flat'],
defects=isrDataDict['defects'],
fringes=isrDataDict['fringes'],
bfKernel=isrDataDict['bfKernel'])
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp(
) if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2")
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask), name="charImage2")
calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema)
try:
dataId = dict(visit=1)
dataIdStrList = [
"%s=%s" % (key, val) for key, val in dataId.items()
]
isrResult1 = IsrTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get(
"postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
icResult1 = CharacterizeImageTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure,
exposureIdInfo=exposureIdInfo)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get(
"icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId,
immediate=True),
icResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground",
dataId,
immediate=True),
icResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background)
calResult1 = CalibrateTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat,
exposureIdInfo=exposureIdInfo,
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get(
"calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent"))
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground",
dataId,
immediate=True),
calResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" %
(OutputName, ))
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(
config=getObsTestConfig(IsrTask),
name="isr2"
)
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask),
name="charImage2"
)
calibrateTask = CalibrateTask(
config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema
)
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()]
isrResult1 = IsrTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
icResult1 = CharacterizeImageTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True),
icResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True),
icResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background
)
calResult1 = CalibrateTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent")
)
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True),
calResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background
)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
