def testSimpleQGraphSkipExisting(self): """Test continuing execution of trivial quantum graph with --skip-existing. """ nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) self.assertEqual(len(qgraph.taskGraph), 5) self.assertEqual(len(qgraph), nQuanta) args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock(stopAt=3) # run first three quanta with self.assertRaises(RuntimeError): fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, 3) # run remaining ones taskFactory.stopAt = -1 args.skip_existing = True args.no_versions = True fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, nQuanta)
def testSimpleQGraph(self): """Test successfull execution of trivial quantum graph. """ nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) self.assertEqual(len(qgraph.taskGraph), 5) self.assertEqual(len(qgraph), nQuanta) args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock() # run whole thing fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, nQuanta)
def testSimpleQGraphClobberPartialOutputs(self): """Test continuing execution of trivial quantum graph with --clobber-partial-outputs. """ nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) # should have one task and number of quanta self.assertEqual(len(qgraph), nQuanta) args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock(stopAt=3) # run first three quanta with self.assertRaises(RuntimeError): fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, 3) # drop one of the two outputs from one task ref = butler._findDatasetRef("add2_dataset2", instrument="INSTR", detector=0) self.assertIsNotNone(ref) butler.pruneDatasets([ref], disassociate=True, unstore=True, purge=True) taskFactory.stopAt = -1 args.skip_existing = True args.clobber_partial_outputs = True args.no_versions = True fwk.runPipeline(qgraph, taskFactory, args, butler=butler) # number of executed quanta is incremented self.assertEqual(taskFactory.countExec, nQuanta + 1)
def testSimpleQGraphReplaceRun(self): """Test repeated execution of trivial quantum graph with --replace-run. """ # need non-memory registry in this case nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root, inMemory=False) # should have one task and number of quanta self.assertEqual(len(qgraph), nQuanta) fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock() # run whole thing args = _makeArgs(butler_config=self.root, input="test", output="output", output_run="output/run1") # deep copy is needed because quanta are updated in place fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) self.assertEqual(taskFactory.countExec, nQuanta) # need to refresh collections explicitly (or make new butler/registry) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) self.assertEqual(collections, {"test", "output", "output/run1"}) # number of datasets written by pipeline: # - nQuanta of init_outputs # - nQuanta of configs # - packages (single dataset) # - nQuanta * two output datasets # - nQuanta of metadata n_outputs = nQuanta * 5 + 1 refs = butler.registry.queryDatasets(..., collections="output/run1") self.assertEqual(len(list(refs)), n_outputs) # re-run with --replace-run (--inputs is not compatible) args.input = None args.replace_run = True args.output_run = "output/run2" fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) self.assertEqual(collections, {"test", "output", "output/run1", "output/run2"}) # new output collection refs = butler.registry.queryDatasets(..., collections="output/run2") self.assertEqual(len(list(refs)), n_outputs) # old output collection is still there refs = butler.registry.queryDatasets(..., collections="output/run1") self.assertEqual(len(list(refs)), n_outputs) # re-run with --replace-run and --prune-replaced=unstore args.input = None args.replace_run = True args.prune_replaced = "unstore" args.output_run = "output/run3" fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) self.assertEqual( collections, {"test", "output", "output/run1", "output/run2", "output/run3"}) # new output collection refs = butler.registry.queryDatasets(..., collections="output/run3") self.assertEqual(len(list(refs)), n_outputs) # old output collection is still there, and it has all datasets but # they are not in datastore refs = butler.registry.queryDatasets(..., collections="output/run2") refs = list(refs) self.assertEqual(len(refs), n_outputs) with self.assertRaises(FileNotFoundError): butler.get(refs[0], collections="output/run2") # re-run with --replace-run and --prune-replaced=purge args.input = None args.replace_run = True args.prune_replaced = "purge" args.output_run = "output/run4" fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) # output/run3 should disappear now self.assertEqual( collections, {"test", "output", "output/run1", "output/run2", "output/run4"}) # new output collection refs = butler.registry.queryDatasets(..., collections="output/run4") self.assertEqual(len(list(refs)), n_outputs)