def makePipeline(self, args):
"""Build a pipeline from command line arguments.
Parameters
----------
args : `argparse.Namespace`
Parsed command line
Returns
-------
pipeline : `~lsst.pipe.base.Pipeline`
"""
if args.pipeline:
pipeline = Pipeline.fromFile(args.pipeline)
else:
pipeline = Pipeline("anonymous")
# loop over all pipeline actions and apply them in order
for action in args.pipeline_actions:
if action.action == "add_instrument":
pipeline.addInstrument(action.value)
elif action.action == "new_task":
pipeline.addTask(action.value, action.label)
elif action.action == "delete_task":
pipeline.removeTask(action.label)
elif action.action == "config":
# action value string is "field=value", split it at '='
field, _, value = action.value.partition("=")
pipeline.addConfigOverride(action.label, field, value)
elif action.action == "configfile":
pipeline.addConfigFile(action.label, action.value)
else:
raise ValueError(f"Unexpected pipeline action: {action.action}")
if args.save_pipeline:
pipeline.toFile(args.save_pipeline)
if args.pipeline_dot:
pipeline2dot(pipeline, args.pipeline_dot)
return pipeline
def _runPipeline(self,
repo,
pipelineFile,
queryString='',
inputCollections=None,
outputCollection=None,
configFiles={},
configOptions={},
registerDatasetTypes=False):
"""Run a pipeline via pipetask.
Parameters
----------
repo : `str`
Gen3 repository yaml file.
pipelineFile : `str`
Pipeline definition file.
queryString : `str`, optional
Where query that defines the data to use.
inputCollections : `list` [`str`], optional
Input collections list.
outputCollection : `str`, optional
Output collection name.
configFiles : `dict` [`list`], optional
Dictionary of config files. The key of the ``configFiles``
dict is the relevant task label. The value of ``configFiles``
is a list of config files to apply (in order) to that task.
configOptions : `dict` [`dict`], optional
Dictionary of individual config options. The key of the
``configOptions`` dict is the relevant task label. The value
of ``configOptions`` is another dict that contains config
key/value overrides to apply.
configOptions : `list` [`str`], optional
List of individual config options to use. Each string will
be of the form ``taskName:configField=value``.
registerDatasetTypes : `bool`, optional
Register new dataset types?
Returns
-------
exit_code : `int`
Exit code for pipetask run.
Raises
------
RuntimeError : Raised if the "pipetask" call fails.
"""
butler = SimplePipelineExecutor.prep_butler(repo,
inputs=inputCollections,
output=outputCollection)
pipeline = Pipeline.fromFile(pipelineFile)
for taskName, fileList in configFiles.items():
for fileName in fileList:
pipeline.addConfigFile(taskName, fileName)
for taskName, configDict in configOptions.items():
for option, value in configDict.items():
pipeline.addConfigOverride(taskName, option, value)
executor = SimplePipelineExecutor.from_pipeline(pipeline,
where=queryString,
root=repo,
butler=butler)
quanta = executor.run(register_dataset_types=registerDatasetTypes)
return len(quanta)