def testAppend(self):
"""Testing append() method
"""
pipeline = Pipeline()
self.assertEqual(len(pipeline), 0)
pipeline.append(TaskDef("lsst.pipe.base.tests.Add", AddConfig()))
pipeline.append(TaskDef("lsst.pipe.base.tests.Mult", MultConfig()))
self.assertEqual(len(pipeline), 2)
self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add")
self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testInitial(self):
"""Testing constructor with initial data
"""
pipeline = Pipeline([
TaskDef("lsst.pipe.base.tests.Add", AddConfig()),
TaskDef("lsst.pipe.base.tests.Mult", MultConfig())
])
self.assertEqual(len(pipeline), 2)
self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add")
self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testPickle(self):
"""Test pickling/unpickling.
"""
pipeline = Pipeline([
TaskDef("lsst.pipe.base.tests.Add", AddConfig()),
TaskDef("lsst.pipe.base.tests.Mult", MultConfig())
])
blob = pickle.dumps(pipeline)
pipeline = pickle.loads(blob)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 2)
self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add")
self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testTaskDef(self):
"""Tests for TaskDef structure
"""
task1 = TaskDef("lsst.pipe.base.tests.Add", AddConfig())
self.assertEqual(task1.taskName, "lsst.pipe.base.tests.Add")
self.assertIsInstance(task1.config, AddConfig)
self.assertIsNone(task1.taskClass)
self.assertEqual(task1.label, "")
task2 = TaskDef("lsst.pipe.base.tests.Mult", MultConfig(), MultTask,
"mult_task")
self.assertEqual(task2.taskName, "lsst.pipe.base.tests.Mult")
self.assertIsInstance(task2.config, MultConfig)
self.assertIs(task2.taskClass, MultTask)
self.assertEqual(task2.label, "mult_task")
def _makePipeline(tasks):
"""Generate Pipeline instance.
Parameters
----------
tasks : list of tuples
Each tuple in the list has 3 or 4 items:
- input DatasetType name(s), string or tuple of strings
- output DatasetType name(s), string or tuple of strings
- task label, string
- optional task class object, can be None
Returns
-------
Pipeline instance
"""
pipe = Pipeline()
for task in tasks:
inputs = task[0]
outputs = task[1]
label = task[2]
klass = task[3] if len(task) > 3 else ExamplePipelineTask
config = _makeConfig(inputs, outputs)
pipe.append(TaskDef("ExamplePipelineTask", config, klass, label))
return pipe
def testTaskDef(self):
"""Tests for TaskDef structure"""
task1 = TaskDef(taskClass=AddTask, config=AddTask.ConfigClass())
self.assertIn("Add", task1.taskName)
self.assertIsInstance(task1.config, AddTask.ConfigClass)
self.assertIsNotNone(task1.taskClass)
self.assertEqual(task1.label, "add_task")
task1a = pickle.loads(pickle.dumps(task1))
self.assertEqual(task1, task1a)
def _makeQGraph():
"""Make a trivial QuantumGraph with one quantum.
The only thing that we need to do with this quantum graph is to pickle
it, the quanta in this graph are not usable for anything else.
Returns
-------
qgraph : `~lsst.pipe.base.QuantumGraph`
"""
# The task name in TaskDef needs to be a real importable name, use one that is sure to exist
taskDef = TaskDef(taskName="lsst.pipe.base.Struct", config=SimpleConfig())
quanta = [
Quantum(taskName="lsst.pipe.base.Struct",
inputs={FakeTaskDef("A"): FakeDSRef("A", (1, 2))})
] # type: ignore
qgraph = QuantumGraph({taskDef: set(quanta)})
return qgraph
def setUp(self):
config = Config(
{
"version": 1,
"namespace": "pipe_base_test",
"skypix": {
"common": "htm7",
"htm": {
"class": "lsst.sphgeom.HtmPixelization",
"max_level": 24,
},
},
"elements": {
"A": {
"keys": [
{
"name": "id",
"type": "int",
}
],
"storage": {
"cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage",
},
},
"B": {
"keys": [
{
"name": "id",
"type": "int",
}
],
"storage": {
"cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage",
},
},
},
"packers": {},
}
)
universe = DimensionUniverse(config=config)
# need to make a mapping of TaskDef to set of quantum
quantumMap = {}
tasks = []
for task, label in (
(Dummy1PipelineTask, "R"),
(Dummy2PipelineTask, "S"),
(Dummy3PipelineTask, "T"),
(Dummy4PipelineTask, "U"),
):
config = task.ConfigClass()
taskDef = TaskDef(get_full_type_name(task), config, task, label)
tasks.append(taskDef)
quantumSet = set()
connections = taskDef.connections
for a, b in ((1, 2), (3, 4)):
if connections.initInputs:
initInputDSType = DatasetType(
connections.initInput.name,
tuple(),
storageClass=connections.initInput.storageClass,
universe=universe,
)
initRefs = [DatasetRef(initInputDSType, DataCoordinate.makeEmpty(universe))]
else:
initRefs = None
inputDSType = DatasetType(
connections.input.name,
connections.input.dimensions,
storageClass=connections.input.storageClass,
universe=universe,
)
inputRefs = [
DatasetRef(inputDSType, DataCoordinate.standardize({"A": a, "B": b}, universe=universe))
]
outputDSType = DatasetType(
connections.output.name,
connections.output.dimensions,
storageClass=connections.output.storageClass,
universe=universe,
)
outputRefs = [
DatasetRef(outputDSType, DataCoordinate.standardize({"A": a, "B": b}, universe=universe))
]
quantumSet.add(
Quantum(
taskName=task.__qualname__,
dataId=DataCoordinate.standardize({"A": a, "B": b}, universe=universe),
taskClass=task,
initInputs=initRefs,
inputs={inputDSType: inputRefs},
outputs={outputDSType: outputRefs},
)
)
quantumMap[taskDef] = quantumSet
self.tasks = tasks
self.quantumMap = quantumMap
self.qGraph = QuantumGraph(quantumMap, metadata=METADATA)
self.universe = universe