def testMakeGraphFromPickle(self): """Tests for CmdLineFwk.makeGraph method. Only most trivial case is tested that does not do actual graph building. """ fwk = CmdLineFwk() taskFactory = TaskFactoryMock() with makeTmpFile() as tmpname: # make empty graph and store it in a file qgraph = QuantumGraph() with open(tmpname, "wb") as pickleFile: pickle.dump(qgraph, pickleFile) args = _makeArgs(qgraph=tmpname) with self.assertWarnsRegex(UserWarning, "QuantumGraph is empty"): # this also tests that warning is generated for empty graph qgraph = fwk.makeGraph(None, taskFactory, args) self.assertIsInstance(qgraph, QuantumGraph) self.assertEqual(len(qgraph), 0) # pickle with wrong object type with open(tmpname, "wb") as pickleFile: pickle.dump({}, pickleFile) args = _makeArgs(qgraph=tmpname) with self.assertRaises(TypeError): fwk.makeGraph(None, taskFactory, args)
def testSubgraph(self): """Test successfull execution of trivial quantum graph. """ nQuanta = 5 nNodes = 2 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) # Select first two nodes for execution. This depends on node ordering # which I assume is the same as execution order. nodeIds = [node.nodeId.number for node in qgraph] nodeIds = nodeIds[:nNodes] self.assertEqual(len(qgraph.taskGraph), 5) self.assertEqual(len(qgraph), nQuanta) with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry( ) as registryConfig: with open(tmpname, "wb") as saveFile: qgraph.save(saveFile) args = _makeArgs(qgraph=tmpname, qgraph_node_id=nodeIds, registryConfig=registryConfig) fwk = CmdLineFwk() # load graph, should only read a subset qgraph = fwk.makeGraph(pipeline=None, args=args) self.assertEqual(len(qgraph), nNodes)
def testShowGraphWorkflow(self): fwk = CmdLineFwk() nQuanta = 2 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) args = _makeArgs(show=["workflow"]) fwk.showInfo(args, pipeline=None, graph=qgraph)
def testShowGraph(self): """Test for --show options for quantum graph. """ fwk = CmdLineFwk() nQuanta = 2 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) args = _makeArgs(show=["graph"]) fwk.showInfo(args, pipeline=None, graph=qgraph)
def testMakePipeline(self): """Tests for CmdLineFwk.makePipeline method """ fwk = CmdLineFwk() taskFactory = TaskFactoryMock() # make empty pipeline args = _makeArgs() pipeline = fwk.makePipeline(taskFactory, args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 0) # few tests with pickle with makeTmpFile() as tmpname: # make empty pipeline and store it in a file args = _makeArgs(save_pipeline=tmpname) pipeline = fwk.makePipeline(taskFactory, args) self.assertIsInstance(pipeline, Pipeline) # read pipeline from a file args = _makeArgs(pipeline=tmpname) pipeline = fwk.makePipeline(taskFactory, args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 0) # pickle with wrong object type with open(tmpname, "wb") as pickleFile: pickle.dump({}, pickleFile) args = _makeArgs(pipeline=tmpname) with self.assertRaises(TypeError): fwk.makePipeline(taskFactory, args) # single task pipeline actions = [ _PipelineAction(action="new_task", label="task1", value="TaskOne") ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(taskFactory, args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 1) # many task pipeline actions = [ _PipelineAction(action="new_task", label="task1a", value="TaskOne"), _PipelineAction(action="new_task", label="task2", value="TaskTwo"), _PipelineAction(action="new_task", label="task1b", value="TaskOne") ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(taskFactory, args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 3)
def testSimpleQGraph(self): """Test successfull execution of trivial quantum graph. """ nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) self.assertEqual(len(qgraph.taskGraph), 5) self.assertEqual(len(qgraph), nQuanta) args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock() # run whole thing fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, nQuanta)
def testSimpleQGraphSkipExisting(self): """Test continuing execution of trivial quantum graph with --skip-existing. """ nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) self.assertEqual(len(qgraph.taskGraph), 5) self.assertEqual(len(qgraph), nQuanta) args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock(stopAt=3) # run first three quanta with self.assertRaises(RuntimeError): fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, 3) # run remaining ones taskFactory.stopAt = -1 args.skip_existing = True args.no_versions = True fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, nQuanta)
def testMakeGraphFromSave(self): """Tests for CmdLineFwk.makeGraph method. Only most trivial case is tested that does not do actual graph building. """ fwk = CmdLineFwk() with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry( ) as registryConfig: # make non-empty graph and store it in a file qgraph = _makeQGraph() with open(tmpname, "wb") as saveFile: qgraph.save(saveFile) args = _makeArgs(qgraph=tmpname, registryConfig=registryConfig) qgraph = fwk.makeGraph(None, args) self.assertIsInstance(qgraph, QuantumGraph) self.assertEqual(len(qgraph), 1) # will fail if graph id does not match args = _makeArgs(qgraph=tmpname, qgraph_id="R2-D2 is that you?", registryConfig=registryConfig) with self.assertRaisesRegex(ValueError, "graphID does not match"): fwk.makeGraph(None, args) # save with wrong object type with open(tmpname, "wb") as saveFile: pickle.dump({}, saveFile) args = _makeArgs(qgraph=tmpname, registryConfig=registryConfig) with self.assertRaises(ValueError): fwk.makeGraph(None, args) # reading empty graph from pickle should work but makeGraph() # will return None and make a warning qgraph = QuantumGraph(dict()) with open(tmpname, "wb") as saveFile: qgraph.save(saveFile) args = _makeArgs(qgraph=tmpname, registryConfig=registryConfig) with self.assertWarnsRegex(UserWarning, "QuantumGraph is empty"): # this also tests that warning is generated for empty graph qgraph = fwk.makeGraph(None, args) self.assertIs(qgraph, None)
def testSimpleQGraphClobberPartialOutputs(self): """Test continuing execution of trivial quantum graph with --clobber-partial-outputs. """ nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) # should have one task and number of quanta self.assertEqual(len(qgraph), nQuanta) args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock(stopAt=3) # run first three quanta with self.assertRaises(RuntimeError): fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, 3) # drop one of the two outputs from one task ref = butler._findDatasetRef("add2_dataset2", instrument="INSTR", detector=0) self.assertIsNotNone(ref) butler.pruneDatasets([ref], disassociate=True, unstore=True, purge=True) taskFactory.stopAt = -1 args.skip_existing = True args.clobber_partial_outputs = True args.no_versions = True fwk.runPipeline(qgraph, taskFactory, args, butler=butler) # number of executed quanta is incremented self.assertEqual(taskFactory.countExec, nQuanta + 1)
def testSimpleQGraphReplaceRun(self): """Test repeated execution of trivial quantum graph with --replace-run. """ # need non-memory registry in this case nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root, inMemory=False) # should have one task and number of quanta self.assertEqual(len(qgraph), nQuanta) fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock() # run whole thing args = _makeArgs(butler_config=self.root, input="test", output="output", output_run="output/run1") # deep copy is needed because quanta are updated in place fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) self.assertEqual(taskFactory.countExec, nQuanta) # need to refresh collections explicitly (or make new butler/registry) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) self.assertEqual(collections, {"test", "output", "output/run1"}) # number of datasets written by pipeline: # - nQuanta of init_outputs # - nQuanta of configs # - packages (single dataset) # - nQuanta * two output datasets # - nQuanta of metadata n_outputs = nQuanta * 5 + 1 refs = butler.registry.queryDatasets(..., collections="output/run1") self.assertEqual(len(list(refs)), n_outputs) # re-run with --replace-run (--inputs is not compatible) args.input = None args.replace_run = True args.output_run = "output/run2" fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) self.assertEqual(collections, {"test", "output", "output/run1", "output/run2"}) # new output collection refs = butler.registry.queryDatasets(..., collections="output/run2") self.assertEqual(len(list(refs)), n_outputs) # old output collection is still there refs = butler.registry.queryDatasets(..., collections="output/run1") self.assertEqual(len(list(refs)), n_outputs) # re-run with --replace-run and --prune-replaced=unstore args.input = None args.replace_run = True args.prune_replaced = "unstore" args.output_run = "output/run3" fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) self.assertEqual( collections, {"test", "output", "output/run1", "output/run2", "output/run3"}) # new output collection refs = butler.registry.queryDatasets(..., collections="output/run3") self.assertEqual(len(list(refs)), n_outputs) # old output collection is still there, and it has all datasets but # they are not in datastore refs = butler.registry.queryDatasets(..., collections="output/run2") refs = list(refs) self.assertEqual(len(refs), n_outputs) with self.assertRaises(FileNotFoundError): butler.get(refs[0], collections="output/run2") # re-run with --replace-run and --prune-replaced=purge args.input = None args.replace_run = True args.prune_replaced = "purge" args.output_run = "output/run4" fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) # output/run3 should disappear now self.assertEqual( collections, {"test", "output", "output/run1", "output/run2", "output/run4"}) # new output collection refs = butler.registry.queryDatasets(..., collections="output/run4") self.assertEqual(len(list(refs)), n_outputs)
def testShowPipeline(self): """Test for --show options for pipeline. """ fwk = CmdLineFwk() actions = [ _ACTION_ADD_TASK("lsst.pipe.base.tests.simpleQGraph.AddTask:task"), _ACTION_CONFIG("task:addend=100") ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args) args.show = ["pipeline"] fwk.showInfo(args, pipeline) args.show = ["config"] fwk.showInfo(args, pipeline) args.show = ["history=task::addend"] fwk.showInfo(args, pipeline) args.show = ["tasks"] fwk.showInfo(args, pipeline)
def testMakePipeline(self): """Tests for CmdLineFwk.makePipeline method """ fwk = CmdLineFwk() # make empty pipeline args = _makeArgs() pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 0) # few tests with serialization with makeTmpFile() as tmpname: # make empty pipeline and store it in a file args = _makeArgs(save_pipeline=tmpname) pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline) # read pipeline from a file args = _makeArgs(pipeline=tmpname) pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 0) # single task pipeline actions = [_ACTION_ADD_TASK("TaskOne:task1")] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 1) # many task pipeline actions = [ _ACTION_ADD_TASK("TaskOne:task1a"), _ACTION_ADD_TASK("TaskTwo:task2"), _ACTION_ADD_TASK("TaskOne:task1b") ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args) self.assertIsInstance(pipeline, Pipeline) self.assertEqual(len(pipeline), 3) # single task pipeline with config overrides, cannot use TaskOne, need # something that can be imported with `doImport()` actions = [ _ACTION_ADD_TASK("lsst.pipe.base.tests.simpleQGraph.AddTask:task"), _ACTION_CONFIG("task:addend=100") ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args) taskDefs = list(pipeline.toExpandedPipeline()) self.assertEqual(len(taskDefs), 1) self.assertEqual(taskDefs[0].config.addend, 100) overrides = b"config.addend = 1000\n" with makeTmpFile(overrides) as tmpname: actions = [ _ACTION_ADD_TASK( "lsst.pipe.base.tests.simpleQGraph.AddTask:task"), _ACTION_CONFIG_FILE("task:" + tmpname) ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args) taskDefs = list(pipeline.toExpandedPipeline()) self.assertEqual(len(taskDefs), 1) self.assertEqual(taskDefs[0].config.addend, 1000) # Check --instrument option, for now it only checks that it does not crash actions = [ _ACTION_ADD_TASK("lsst.pipe.base.tests.simpleQGraph.AddTask:task"), _ACTION_ADD_INSTRUMENT("Instrument") ] args = _makeArgs(pipeline_actions=actions) pipeline = fwk.makePipeline(args)