def testChain2(self):
"""Test for two-task chain.
"""
butler = ButlerMock()
config1 = AddConfig()
connections1 = config1.connections.ConnectionsClass(config=config1)
task1 = AddTask(config=config1)
config2 = AddConfig()
config2.addend = 200
config2.connections.input = task1.config.connections.output
config2.connections.output = "add_output_2"
task2 = AddTask2(config=config2)
connections2 = config2.connections.ConnectionsClass(config=config2)
# make all quanta
quanta1 = self._makeQuanta(task1.config)
quanta2 = self._makeQuanta(task2.config)
# add input data to butler
task1Connections = task1.config.connections.ConnectionsClass(
config=task1.config)
dstype0 = task1Connections.input.makeDatasetType(
butler.registry.dimensions)
for i, quantum in enumerate(quanta1):
ref = quantum.predictedInputs[dstype0.name][0]
butler.put(
100 + i,
pipeBase.Struct(datasetType=dstype0.name, dataId=ref.dataId))
# run task on each quanta
for quantum in quanta1:
butlerQC = pipeBase.ButlerQuantumContext(butler, quantum)
inputRefs, outputRefs = connections1.buildDatasetRefs(quantum)
task1.runQuantum(butlerQC, inputRefs, outputRefs)
for quantum in quanta2:
butlerQC = pipeBase.ButlerQuantumContext(butler, quantum)
inputRefs, outputRefs = connections2.buildDatasetRefs(quantum)
task2.runQuantum(butlerQC, inputRefs, outputRefs)
# look at the output produced by the task
outputName = task1.config.connections.output
dsdata = butler.datasets[outputName]
self.assertEqual(len(dsdata), len(quanta1))
for i, quantum in enumerate(quanta1):
ref = quantum.outputs[outputName][0]
self.assertEqual(dsdata[ref.dataId], 100 + i + 3)
outputName = task2.config.connections.output
dsdata = butler.datasets[outputName]
self.assertEqual(len(dsdata), len(quanta2))
for i, quantum in enumerate(quanta2):
ref = quantum.outputs[outputName][0]
self.assertEqual(dsdata[ref.dataId], 100 + i + 3 + 200)
def testRunQuantum(self):
"""Test for AddTask.runQuantum() implementation.
"""
butler = ButlerMock()
task = AddTask(config=AddConfig())
connections = task.config.connections.ConnectionsClass(
config=task.config)
# make all quanta
quanta = self._makeQuanta(task.config)
# add input data to butler
dstype0 = connections.input.makeDatasetType(butler.registry.dimensions)
for i, quantum in enumerate(quanta):
ref = quantum.predictedInputs[dstype0.name][0]
butler.put(
100 + i,
pipeBase.Struct(datasetType=dstype0.name, dataId=ref.dataId))
# run task on each quanta
for quantum in quanta:
butlerQC = pipeBase.ButlerQuantumContext(butler, quantum)
inputRefs, outputRefs = connections.buildDatasetRefs(quantum)
task.runQuantum(butlerQC, inputRefs, outputRefs)
# look at the output produced by the task
outputName = connections.output.name
dsdata = butler.datasets[outputName]
self.assertEqual(len(dsdata), len(quanta))
for i, quantum in enumerate(quanta):
ref = quantum.outputs[outputName][0]
self.assertEqual(dsdata[ref.dataId], 100 + i + 3)
def testRunQuantum(self):
# Set up test quantum from butler data
inputRefs = pipeBase.InputQuantizedConnection()
badInstrument = "LSSTComCam"
inputRefs.exposures = [
dafButler.DatasetRef(
self.registry.getDatasetType("postISRCCD"),
{
"instrument": badInstrument,
"detector": 191,
"exposure": 4021123106000,
},
id="3104de33-c107-4678-b07e-1fc62407a52e",
run="run2",
)
]
inputRefs.camera = self.butler.getDeferred(
"camera",
instrument="LSSTComCam",
collections="LSSTComCam/calib/unbounded").ref
outputRefs = pipeBase.OutputQuantizedConnection()
quantum = dafButler.Quantum(
inputs={
inputRefs.exposures[0].datasetType: inputRefs.exposures,
inputRefs.camera.datasetType: [inputRefs.camera],
})
butlerQC = pipeBase.ButlerQuantumContext(self.butler, quantum)
# Test that we will get an error if we try to use an
# unsupported instrument.
errMsg = f"{badInstrument} is not a valid camera name."
with self.assertRaises(ValueError, msg=errMsg) as context:
self.task.runQuantum(butlerQC, inputRefs, outputRefs)
self.assertEqual(
f"{badInstrument} is not a valid camera name.",
str(context.exception),
)
# Test error raised when list sizes are unequal
inputRefs.exposures = [
self.butler.getDeferred("postISRCCD",
dataId=self.dataIdExtra,
collections=[self.runName]).ref,
]
inputRefs.camera = self.butler.getDeferred(
"camera",
instrument="LSSTCam",
collections="LSSTCam/calib/unbounded").ref
inputRefs.donutCatalog = [
self.butler.getDeferred("donutCatalog",
dataId=self.dataIdExtra,
collections=[self.runName]).ref,
self.butler.getDeferred("donutCatalog",
dataId=self.dataIdIntra,
collections=[self.runName]).ref,
]
outputRefs = pipeBase.OutputQuantizedConnection()
quantum = dafButler.Quantum(
inputs={
inputRefs.exposures[0].datasetType: inputRefs.exposures,
inputRefs.donutCatalog[0].datasetType: inputRefs.donutCatalog,
inputRefs.camera.datasetType: [inputRefs.camera],
})
butlerQC = pipeBase.ButlerQuantumContext(self.butler, quantum)
unequalMsg = "Unequal number of intra and extra focal detectors."
with self.assertRaises(ValueError) as context:
self.task.runQuantum(butlerQC, inputRefs, outputRefs)
self.assertEqual(str(context.exception), unequalMsg)
# Test error raised when extra and intra focal do not
# have correct partner
self.mismatchDataId = copy(self.dataIdIntra)
self.mismatchDataId["detector"] = 196
# Test errors raised
inputRefs.exposures.append(
self.butler.getDeferred("postISRCCD",
dataId=self.mismatchDataId,
collections=[self.runName]).ref)
butlerQC = pipeBase.ButlerQuantumContext(self.butler, quantum)
mismatchMsg = "Intra and extra focal detectors not adjacent."
with self.assertRaises(ValueError) as context:
self.task.runQuantum(butlerQC, inputRefs, outputRefs)
self.assertEqual(str(context.exception), mismatchMsg)
def testRunQuantum(self):
"""Test for AddTask.runQuantum() implementation."""
butler = ButlerMock()
task = AddTask(config=AddConfig())
connections = task.config.connections.ConnectionsClass(config=task.config)
# make all quanta
quanta = self._makeQuanta(task.config)
# add input data to butler
dstype0 = connections.input.makeDatasetType(butler.registry.dimensions)
for i, quantum in enumerate(quanta):
ref = quantum.inputs[dstype0.name][0]
butler.put(100 + i, pipeBase.Struct(datasetType=dstype0.name, dataId=ref.dataId))
# run task on each quanta
checked_get = False
for quantum in quanta:
butlerQC = pipeBase.ButlerQuantumContext(butler, quantum)
inputRefs, outputRefs = connections.buildDatasetRefs(quantum)
task.runQuantum(butlerQC, inputRefs, outputRefs)
# Test getting of datasets in different ways.
# (only need to do this one time)
if not checked_get:
# Force the periodic logger to issue messages.
lsst.utils.logging.PeriodicLogger.LOGGING_INTERVAL = 0.0
checked_get = True
with self.assertLogs("lsst.pipe.base", level=lsst.utils.logging.VERBOSE) as cm:
input_data = butlerQC.get(inputRefs)
self.assertIn("Completed", cm.output[-1])
self.assertEqual(len(input_data), len(inputRefs))
# In this test there are no multiples returned.
refs = [ref for _, ref in inputRefs]
with self.assertLogs("lsst.pipe.base", level=lsst.utils.logging.VERBOSE) as cm:
list_get = butlerQC.get(refs)
lsst.utils.logging.PeriodicLogger.LOGGING_INTERVAL = 600.0
self.assertIn("Completed", cm.output[-1])
self.assertEqual(len(list_get), len(input_data))
self.assertIsInstance(list_get[0], int)
scalar_get = butlerQC.get(refs[0])
self.assertEqual(scalar_get, list_get[0])
with self.assertRaises(TypeError):
butlerQC.get({})
# Output ref won't be known to this quantum.
outputs = [ref for _, ref in outputRefs]
with self.assertRaises(ValueError):
butlerQC.get(outputs[0])
# look at the output produced by the task
outputName = connections.output.name
dsdata = butler.datasets[outputName]
self.assertEqual(len(dsdata), len(quanta))
for i, quantum in enumerate(quanta):
ref = quantum.outputs[outputName][0]
self.assertEqual(dsdata[ref.dataId], 100 + i + 3)