def _makePipeline(tasks): """Generate Pipeline instance. Parameters ---------- tasks : list of tuples Each tuple in the list has 3 or 4 items: - input DatasetType name(s), string or tuple of strings - output DatasetType name(s), string or tuple of strings - task label, string - optional task class object, can be None Returns ------- Pipeline instance """ pipe = Pipeline() for task in tasks: inputs = task[0] outputs = task[1] label = task[2] klass = task[3] if len(task) > 3 else ExamplePipelineTask config = _makeConfig(inputs, outputs) pipe.append(TaskDef("ExamplePipelineTask", config, klass, label)) return pipe
def _makePipeline(tasks): """Generate Pipeline instance. Parameters ---------- tasks : list of tuples Each tuple in the list has 3 or 4 items: - input DatasetType name(s), string or tuple of strings - output DatasetType name(s), string or tuple of strings - task label, string - optional task class object, can be None Returns ------- Pipeline instance """ pipe = Pipeline() for task in tasks: inputs = task[0] outputs = task[1] label = task[2] klass = task[3] if len(task) > 3 else ExamplePipelineTask config = _makeConfig(inputs, outputs) pipe.append(TaskDef("ExamplePipelineTask", config, klass, label)) return pipe
def _makePipeline(tasks): """Generate Pipeline instance. Parameters ---------- tasks : list of tuples Each tuple in the list has 3 or 4 items: - input DatasetType name(s), string or tuple of strings - output DatasetType name(s), string or tuple of strings - task label, string - optional task class object, can be None Returns ------- Pipeline instance """ pipe = Pipeline("test pipeline") for task in tasks: inputs = task[0] outputs = task[1] label = task[2] klass = task[3] if len(task) > 3 else ExamplePipelineTask pipe.addTask(klass, label) _makeConfig(inputs, outputs, pipe, label) return list(pipe.toExpandedPipeline())
def testAppend(self): """Testing append() method """ pipeline = Pipeline() self.assertEqual(len(pipeline), 0) pipeline.append(TaskDef("lsst.pipe.base.tests.Add", AddConfig())) pipeline.append(TaskDef("lsst.pipe.base.tests.Mult", MultConfig())) self.assertEqual(len(pipeline), 2) self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add") self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testParameters(self): """Test that parameters can be set and used to format""" pipeline_str = textwrap.dedent( """ description: Test Pipeline parameters: testValue: 5 tasks: add: class: test_pipeline.AddTask config: addend: parameters.testValue """ ) # verify that parameters are used in expanding a pipeline pipeline = Pipeline.fromString(pipeline_str) expandedPipeline = list(pipeline.toExpandedPipeline()) self.assertEqual(expandedPipeline[0].config.addend, 5) # verify that a parameter can be overridden on the "command line" pipeline.addConfigOverride("parameters", "testValue", 14) expandedPipeline = list(pipeline.toExpandedPipeline()) self.assertEqual(expandedPipeline[0].config.addend, 14) # verify that a non existing parameter cant be overridden with self.assertRaises(ValueError): pipeline.addConfigOverride("parameters", "missingValue", 17) # verify that parameters does not support files or python overrides with self.assertRaises(ValueError): pipeline.addConfigFile("parameters", "fakeFile") with self.assertRaises(ValueError): pipeline.addConfigPython("parameters", "fakePythonString")
def testSerialization(self): pipeline = Pipeline("test") pipeline.addTask(AddTask, "add") pipeline.addTask(MultTask, "mult") dump = str(pipeline) load = Pipeline.fromString(dump) self.assertEqual(pipeline, load)
def testInitial(self): """Testing constructor with initial data """ pipeline = Pipeline([ TaskDef("lsst.pipe.base.tests.Add", AddConfig()), TaskDef("lsst.pipe.base.tests.Mult", MultConfig()) ]) self.assertEqual(len(pipeline), 2) self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add") self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testInitial(self): """Testing constructor with initial data """ pipeline = Pipeline("test") pipeline.addTask(AddTask, "add") pipeline.addTask(MultTask, "mult") self.assertEqual(len(pipeline), 2) expandedPipeline = list(pipeline.toExpandedPipeline()) self.assertEqual(expandedPipeline[0].taskName, "AddTask") self.assertEqual(expandedPipeline[1].taskName, "MultTask")
def testPickle(self): """Test pickling/unpickling. """ pipeline = Pipeline([ TaskDef("lsst.pipe.base.tests.Add", AddConfig()), TaskDef("lsst.pipe.base.tests.Mult", MultConfig()) ]) blob = pickle.dumps(pipeline) pipeline = pickle.loads(blob) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 2) self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add") self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testAppend(self): """Testing append() method """ pipeline = Pipeline() self.assertEqual(len(pipeline), 0) pipeline.append(TaskDef("lsst.pipe.base.tests.Add", AddConfig())) pipeline.append(TaskDef("lsst.pipe.base.tests.Mult", MultConfig())) self.assertEqual(len(pipeline), 2) self.assertEqual(pipeline[0].taskName, "lsst.pipe.base.tests.Add") self.assertEqual(pipeline[1].taskName, "lsst.pipe.base.tests.Mult")
def testEmpty(self): """Creating empty pipeline """ pipeline = Pipeline("test") self.assertEqual(len(pipeline), 0)
def makePipeline(self, args): """Build a pipeline from command line arguments. Parameters ---------- args : `argparse.Namespace` Parsed command line Returns ------- pipeline : `~lsst.pipe.base.Pipeline` """ if args.pipeline: pipeline = Pipeline.fromFile(args.pipeline) else: pipeline = Pipeline("anonymous") # loop over all pipeline actions and apply them in order for action in args.pipeline_actions: if action.action == "add_instrument": pipeline.addInstrument(action.value) elif action.action == "new_task": pipeline.addTask(action.value, action.label) elif action.action == "delete_task": pipeline.removeTask(action.label) elif action.action == "config": # action value string is "field=value", split it at '=' field, _, value = action.value.partition("=") pipeline.addConfigOverride(action.label, field, value) elif action.action == "configfile": pipeline.addConfigFile(action.label, action.value) else: raise ValueError(f"Unexpected pipeline action: {action.action}") if args.save_pipeline: pipeline.toFile(args.save_pipeline) if args.pipeline_dot: pipeline2dot(pipeline, args.pipeline_dot) return pipeline
def _runPipeline(self, repo, pipelineFile, queryString='', inputCollections=None, outputCollection=None, configFiles={}, configOptions={}, registerDatasetTypes=False): """Run a pipeline via pipetask. Parameters ---------- repo : `str` Gen3 repository yaml file. pipelineFile : `str` Pipeline definition file. queryString : `str`, optional Where query that defines the data to use. inputCollections : `list` [`str`], optional Input collections list. outputCollection : `str`, optional Output collection name. configFiles : `dict` [`list`], optional Dictionary of config files. The key of the ``configFiles`` dict is the relevant task label. The value of ``configFiles`` is a list of config files to apply (in order) to that task. configOptions : `dict` [`dict`], optional Dictionary of individual config options. The key of the ``configOptions`` dict is the relevant task label. The value of ``configOptions`` is another dict that contains config key/value overrides to apply. configOptions : `list` [`str`], optional List of individual config options to use. Each string will be of the form ``taskName:configField=value``. registerDatasetTypes : `bool`, optional Register new dataset types? Returns ------- exit_code : `int` Exit code for pipetask run. Raises ------ RuntimeError : Raised if the "pipetask" call fails. """ butler = SimplePipelineExecutor.prep_butler(repo, inputs=inputCollections, output=outputCollection) pipeline = Pipeline.fromFile(pipelineFile) for taskName, fileList in configFiles.items(): for fileName in fileList: pipeline.addConfigFile(taskName, fileName) for taskName, configDict in configOptions.items(): for option, value in configDict.items(): pipeline.addConfigOverride(taskName, option, value) executor = SimplePipelineExecutor.from_pipeline(pipeline, where=queryString, root=repo, butler=butler) quanta = executor.run(register_dataset_types=registerDatasetTypes) return len(quanta)
def testSerialization(self): pipeline = makeSimplePipeline(2) dump = str(pipeline) load = Pipeline.fromString(dump) self.assertEqual(pipeline, load)