def test_ConfigFileTask(self):
"""Simple test case for config overrides in file
"""
builder = PipelineBuilder(TaskFactoryMock())
# make simple pipeline
builder.addTask("TaskOne", "task")
builder.configOverride("task", "field=value")
pipeline = builder.pipeline()
self.assertEqual(pipeline[0].config.field, "value")
# save config to file for next test
overrides = NamedTemporaryFile(mode="wt", delete=False)
pipeline[0].config.saveToStream(overrides)
fname = overrides.name
del overrides
builder = PipelineBuilder(TaskFactoryMock())
builder.addTask("TaskOne", "task")
builder.configOverrideFile("task", fname)
pipeline = builder.pipeline()
self.assertEqual(pipeline[0].config.field, "value")
os.unlink(fname)
# unknown label should raise LookupError
with self.assertRaises(LookupError):
builder.configOverrideFile("label", "/dev/null")
def test_LabelTask(self):
"""Simple test case re-labeling tasks
"""
builder = PipelineBuilder(TaskFactoryMock())
# make short pipeline
tasks = [("TaskOne", ), ("TaskTwo", ), ("TaskOne", "label"),
("TaskTwo", "label2")]
for task in tasks:
builder.addTask(*task)
pipeline = builder.pipeline()
self.assertEqual([t.label for t in pipeline],
["TaskOne", "TaskTwo", "label", "label2"])
builder.labelTask("TaskOne", "label0")
pipeline = builder.pipeline()
self.assertEqual([t.label for t in pipeline],
["label0", "TaskTwo", "label", "label2"])
# unknown label should raise LookupError
with self.assertRaises(LookupError):
builder.labelTask("unlabel", "label3")
# duplicate label should raise LookupError
with self.assertRaises(LookupError):
builder.labelTask("label2", "label0")
def test_DeleteTask(self):
"""Simple test case removing tasks
"""
builder = PipelineBuilder(TaskFactoryMock())
# make short pipeline
tasks = [("TaskOne", ), ("TaskTwo", ), ("TaskOne", "label"),
("TaskTwo", "label2")]
for task in tasks:
builder.addTask(*task)
pipeline = builder.pipeline()
self.assertEqual(len(pipeline), 4)
builder.deleteTask("label")
pipeline = builder.pipeline()
self.assertEqual(len(pipeline), 3)
builder.deleteTask("TaskTwo")
pipeline = builder.pipeline()
self.assertEqual(len(pipeline), 2)
builder.deleteTask("TaskOne")
builder.deleteTask("label2")
pipeline = builder.pipeline()
self.assertEqual(len(pipeline), 0)
# unknown label should raise LookupError
builder.addTask("TaskOne")
builder.addTask("TaskTwo")
with self.assertRaises(LookupError):
builder.deleteTask("label2")
def test_substituteDatatypeNames(self):
"""Simple test case for substituteDatatypeNames method
"""
builder = PipelineBuilder(TaskFactoryMock())
builder.addTask("TaskOne", "task")
pipeline = builder.pipeline()
self.assertEqual(pipeline[0].config.schema.name, "schema")
builder = PipelineBuilder(TaskFactoryMock())
builder.addTask("TaskOne", "task")
builder.substituteDatatypeNames("task", {"template": "extra_"})
pipeline = builder.pipeline()
self.assertEqual(pipeline[0].config.schema.name, "extra_schema")
def test_ConfigTask(self):
"""Simple test case for config overrides
"""
builder = PipelineBuilder(TaskFactoryMock())
# make simple pipeline
builder.addTask("TaskOne", "task")
pipeline = builder.pipeline()
self.assertIsInstance(pipeline[0].config, SimpleConfig)
self.assertIsNone(pipeline[0].config.field)
builder.configOverride("task", "field=value")
pipeline = builder.pipeline()
self.assertEqual(pipeline[0].config.field, "value")
# unknown label should raise LookupError
with self.assertRaises(LookupError):
builder.configOverride("label", "field=value")
def test_LabelExceptions(self):
"""Simple test case adding tasks with identical labels
"""
builder = PipelineBuilder(TaskFactoryMock())
builder.addTask("TaskOne")
with self.assertRaises(LookupError):
builder.addTask("TaskOne")
builder = PipelineBuilder(TaskFactoryMock())
builder.addTask("TaskOne", "label")
with self.assertRaises(LookupError):
builder.addTask("TaskTwo", "label")
def test_AddTasks(self):
"""Simple test case adding tasks to a pipeline
"""
# create a task with default label
builder = PipelineBuilder(TaskFactoryMock())
builder.addTask("TaskOne")
pipeline = builder.pipeline()
self.assertEqual(len(pipeline), 1)
self.assertEqual(pipeline[0].taskName, "TaskOne")
self.assertEqual(pipeline[0].label, "TaskOne")
self.assertIs(pipeline[0].taskClass, TaskOne)
# create a task, give it a label
builder = PipelineBuilder(TaskFactoryMock())
builder.addTask("TaskOne", "label")
pipeline = builder.pipeline()
self.assertEqual(len(pipeline), 1)
self.assertEqual(pipeline[0].taskName, "TaskOne")
self.assertEqual(pipeline[0].label, "label")
self.assertIs(pipeline[0].taskClass, TaskOne)
# two different tasks
builder = PipelineBuilder(TaskFactoryMock())
builder.addTask("TaskOne")
builder.addTask("TaskTwo")
pipeline = builder.pipeline()
self.assertEqual(len(pipeline), 2)
self.assertEqual(pipeline[0].taskName, "TaskOne")
self.assertEqual(pipeline[0].label, "TaskOne")
self.assertIs(pipeline[0].taskClass, TaskOne)
self.assertEqual(pipeline[1].taskName, "TaskTwo")
self.assertEqual(pipeline[1].label, "TaskTwo")
self.assertIs(pipeline[1].taskClass, TaskTwo)
# more than one instance of each class
builder = PipelineBuilder(TaskFactoryMock())
tasks = [("TaskOne", ), ("TaskTwo", ), ("TaskOne", "label"),
("TaskTwo", "label2")]
for task in tasks:
builder.addTask(*task)
pipeline = builder.pipeline()
self.assertEqual(len(pipeline), 4)
self.assertEqual(pipeline[0].taskName, "TaskOne")
self.assertEqual(pipeline[0].label, "TaskOne")
self.assertEqual(pipeline[1].taskName, "TaskTwo")
self.assertEqual(pipeline[1].label, "TaskTwo")
self.assertEqual(pipeline[2].taskName, "TaskOne")
self.assertEqual(pipeline[2].label, "label")
self.assertEqual(pipeline[3].taskName, "TaskTwo")
self.assertEqual(pipeline[3].label, "label2")
def test_MoveTask(self):
"""Simple test case moving tasks
"""
builder = PipelineBuilder(TaskFactoryMock())
# make short pipeline
tasks = [("TaskOne", ), ("TaskTwo", ), ("TaskOne", "label"),
("TaskTwo", "label2")]
for task in tasks:
builder.addTask(*task)
pipeline = builder.pipeline()
self.assertEqual([t.label for t in pipeline],
["TaskOne", "TaskTwo", "label", "label2"])
builder.moveTask("label", 1)
pipeline = builder.pipeline()
self.assertEqual([t.label for t in pipeline],
["TaskOne", "label", "TaskTwo", "label2"])
# negatives are accepted but may be non-intuitive
builder.moveTask("TaskOne", -1)
pipeline = builder.pipeline()
self.assertEqual([t.label for t in pipeline],
["label", "TaskTwo", "TaskOne", "label2"])
# position larger than list size moves to end
builder.moveTask("label", 100)
pipeline = builder.pipeline()
self.assertEqual([t.label for t in pipeline],
["TaskTwo", "TaskOne", "label2", "label"])
builder.moveTask("label", 0)
pipeline = builder.pipeline()
self.assertEqual([t.label for t in pipeline],
["label", "TaskTwo", "TaskOne", "label2"])
# unknown label should raise LookupError
with self.assertRaises(LookupError):
builder.moveTask("unlabel", 0)
def makePipeline(self, taskFactory, args):
"""Build a pipeline from command line arguments.
Parameters
----------
taskFactory : `~lsst.pipe.base.TaskFactory`
Task factory.
args : `argparse.Namespace`
Parsed command line
Returns
-------
pipeline : `~lsst.pipe.base.Pipeline`
"""
# read existing pipeline from pickle file
pipeline = None
if args.pipeline:
with open(args.pipeline, 'rb') as pickleFile:
pipeline = pickle.load(pickleFile)
if not isinstance(pipeline, Pipeline):
raise TypeError(
"Pipeline pickle file has incorrect object type: {}".
format(type(pipeline)))
pipeBuilder = PipelineBuilder(taskFactory, pipeline)
# loop over all pipeline actions and apply them in order
for action in args.pipeline_actions:
if action.action == "new_task":
pipeBuilder.addTask(action.value, action.label)
elif action.action == "delete_task":
pipeBuilder.deleteTask(action.label)
elif action.action == "move_task":
pipeBuilder.moveTask(action.label, action.value)
elif action.action == "relabel":
pipeBuilder.labelTask(action.label, action.value)
elif action.action == "config":
pipeBuilder.configOverride(action.label, action.value)
elif action.action == "configfile":
pipeBuilder.configOverrideFile(action.label, action.value)
elif action.action == 'name_templates':
pipeBuilder.substituteDatatypeNames(action.label, action.value)
else:
raise ValueError(
f"Unexpected pipeline action: {action.action}")
pipeline = pipeBuilder.pipeline(args.order_pipeline)
if args.save_pipeline:
with open(args.save_pipeline, "wb") as pickleFile:
pickle.dump(pipeline, pickleFile)
if args.pipeline_dot:
pipeline2dot(pipeline, args.pipeline_dot, taskFactory)
return pipeline