def _makePipeline(tasks):
"""Generate Pipeline instance.
Parameters
----------
tasks : list of tuples
Each tuple in the list has 3 or 4 items:
- input DatasetType name(s), string or tuple of strings
- output DatasetType name(s), string or tuple of strings
- task label, string
- optional task class object, can be None
Returns
-------
Pipeline instance
"""
pipe = Pipeline()
for task in tasks:
inputs = task[0]
outputs = task[1]
label = task[2]
klass = task[3] if len(task) > 3 else ExamplePipelineTask
config = _makeConfig(inputs, outputs)
pipe.append(TaskDef("ExamplePipelineTask", config, klass, label))
return pipe
def _makePipeline(tasks):
"""Generate Pipeline instance.
Parameters
----------
tasks : list of tuples
Each tuple in the list has 3 or 4 items:
- input DatasetType name(s), string or tuple of strings
- output DatasetType name(s), string or tuple of strings
- task label, string
- optional task class object, can be None
Returns
-------
Pipeline instance
"""
pipe = Pipeline()
for task in tasks:
inputs = task[0]
outputs = task[1]
label = task[2]
klass = task[3] if len(task) > 3 else ExamplePipelineTask
config = _makeConfig(inputs, outputs)
pipe.append(TaskDef("ExamplePipelineTask", config, klass, label))
return pipe
def _makePipeline(tasks):
"""Generate Pipeline instance.
Parameters
----------
tasks : list of tuples
Each tuple in the list has 3 or 4 items:
- input DatasetType name(s), string or tuple of strings
- output DatasetType name(s), string or tuple of strings
- task label, string
- optional task class object, can be None
Returns
-------
Pipeline instance
"""
pipe = Pipeline("test pipeline")
for task in tasks:
inputs = task[0]
outputs = task[1]
label = task[2]
klass = task[3] if len(task) > 3 else ExamplePipelineTask
pipe.addTask(klass, label)
_makeConfig(inputs, outputs, pipe, label)
return list(pipe.toExpandedPipeline())
def testAppend(self):
"""Testing append() method
"""
pipeline = Pipeline()
self.assertEqual(len(pipeline), 0)
pipeline.append(TaskDef("lsst.pipe.base.tests.Add", AddConfig()))
pipeline.append(TaskDef("lsst.pipe.base.tests.Mult", MultConfig()))
self.assertEqual(len(pipeline), 2)
self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add")
self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testParameters(self):
"""Test that parameters can be set and used to format"""
pipeline_str = textwrap.dedent(
"""
description: Test Pipeline
parameters:
testValue: 5
tasks:
add:
class: test_pipeline.AddTask
config:
addend: parameters.testValue
"""
)
# verify that parameters are used in expanding a pipeline
pipeline = Pipeline.fromString(pipeline_str)
expandedPipeline = list(pipeline.toExpandedPipeline())
self.assertEqual(expandedPipeline[0].config.addend, 5)
# verify that a parameter can be overridden on the "command line"
pipeline.addConfigOverride("parameters", "testValue", 14)
expandedPipeline = list(pipeline.toExpandedPipeline())
self.assertEqual(expandedPipeline[0].config.addend, 14)
# verify that a non existing parameter cant be overridden
with self.assertRaises(ValueError):
pipeline.addConfigOverride("parameters", "missingValue", 17)
# verify that parameters does not support files or python overrides
with self.assertRaises(ValueError):
pipeline.addConfigFile("parameters", "fakeFile")
with self.assertRaises(ValueError):
pipeline.addConfigPython("parameters", "fakePythonString")
def testSerialization(self):
pipeline = Pipeline("test")
pipeline.addTask(AddTask, "add")
pipeline.addTask(MultTask, "mult")
dump = str(pipeline)
load = Pipeline.fromString(dump)
self.assertEqual(pipeline, load)
def testInitial(self):
"""Testing constructor with initial data
"""
pipeline = Pipeline([
TaskDef("lsst.pipe.base.tests.Add", AddConfig()),
TaskDef("lsst.pipe.base.tests.Mult", MultConfig())
])
self.assertEqual(len(pipeline), 2)
self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add")
self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testInitial(self):
"""Testing constructor with initial data
"""
pipeline = Pipeline("test")
pipeline.addTask(AddTask, "add")
pipeline.addTask(MultTask, "mult")
self.assertEqual(len(pipeline), 2)
expandedPipeline = list(pipeline.toExpandedPipeline())
self.assertEqual(expandedPipeline[0].taskName, "AddTask")
self.assertEqual(expandedPipeline[1].taskName, "MultTask")
def testPickle(self):
"""Test pickling/unpickling.
"""
pipeline = Pipeline([
TaskDef("lsst.pipe.base.tests.Add", AddConfig()),
TaskDef("lsst.pipe.base.tests.Mult", MultConfig())
])
blob = pickle.dumps(pipeline)
pipeline = pickle.loads(blob)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 2)
self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add")
self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testAppend(self):
"""Testing append() method
"""
pipeline = Pipeline()
self.assertEqual(len(pipeline), 0)
pipeline.append(TaskDef("lsst.pipe.base.tests.Add", AddConfig()))
pipeline.append(TaskDef("lsst.pipe.base.tests.Mult", MultConfig()))
self.assertEqual(len(pipeline), 2)
self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add")
self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testEmpty(self):
"""Creating empty pipeline
"""
pipeline = Pipeline("test")
self.assertEqual(len(pipeline), 0)
def makePipeline(self, args):
"""Build a pipeline from command line arguments.
Parameters
----------
args : `argparse.Namespace`
Parsed command line
Returns
-------
pipeline : `~lsst.pipe.base.Pipeline`
"""
if args.pipeline:
pipeline = Pipeline.fromFile(args.pipeline)
else:
pipeline = Pipeline("anonymous")
# loop over all pipeline actions and apply them in order
for action in args.pipeline_actions:
if action.action == "add_instrument":
pipeline.addInstrument(action.value)
elif action.action == "new_task":
pipeline.addTask(action.value, action.label)
elif action.action == "delete_task":
pipeline.removeTask(action.label)
elif action.action == "config":
# action value string is "field=value", split it at '='
field, _, value = action.value.partition("=")
pipeline.addConfigOverride(action.label, field, value)
elif action.action == "configfile":
pipeline.addConfigFile(action.label, action.value)
else:
raise ValueError(f"Unexpected pipeline action: {action.action}")
if args.save_pipeline:
pipeline.toFile(args.save_pipeline)
if args.pipeline_dot:
pipeline2dot(pipeline, args.pipeline_dot)
return pipeline
def _runPipeline(self,
repo,
pipelineFile,
queryString='',
inputCollections=None,
outputCollection=None,
configFiles={},
configOptions={},
registerDatasetTypes=False):
"""Run a pipeline via pipetask.
Parameters
----------
repo : `str`
Gen3 repository yaml file.
pipelineFile : `str`
Pipeline definition file.
queryString : `str`, optional
Where query that defines the data to use.
inputCollections : `list` [`str`], optional
Input collections list.
outputCollection : `str`, optional
Output collection name.
configFiles : `dict` [`list`], optional
Dictionary of config files. The key of the ``configFiles``
dict is the relevant task label. The value of ``configFiles``
is a list of config files to apply (in order) to that task.
configOptions : `dict` [`dict`], optional
Dictionary of individual config options. The key of the
``configOptions`` dict is the relevant task label. The value
of ``configOptions`` is another dict that contains config
key/value overrides to apply.
configOptions : `list` [`str`], optional
List of individual config options to use. Each string will
be of the form ``taskName:configField=value``.
registerDatasetTypes : `bool`, optional
Register new dataset types?
Returns
-------
exit_code : `int`
Exit code for pipetask run.
Raises
------
RuntimeError : Raised if the "pipetask" call fails.
"""
butler = SimplePipelineExecutor.prep_butler(repo,
inputs=inputCollections,
output=outputCollection)
pipeline = Pipeline.fromFile(pipelineFile)
for taskName, fileList in configFiles.items():
for fileName in fileList:
pipeline.addConfigFile(taskName, fileName)
for taskName, configDict in configOptions.items():
for option, value in configDict.items():
pipeline.addConfigOverride(taskName, option, value)
executor = SimplePipelineExecutor.from_pipeline(pipeline,
where=queryString,
root=repo,
butler=butler)
quanta = executor.run(register_dataset_types=registerDatasetTypes)
return len(quanta)
def testSerialization(self):
pipeline = makeSimplePipeline(2)
dump = str(pipeline)
load = Pipeline.fromString(dump)
self.assertEqual(pipeline, load)