def testSimpleQGraphSkipExisting(self):
"""Test continuing execution of trivial quantum graph with --skip-existing.
"""
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
self.assertEqual(len(qgraph.taskGraph), 5)
self.assertEqual(len(qgraph), nQuanta)
args = _makeArgs()
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock(stopAt=3)
# run first three quanta
with self.assertRaises(RuntimeError):
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, 3)
# run remaining ones
taskFactory.stopAt = -1
args.skip_existing = True
args.no_versions = True
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, nQuanta)
def testSimpleQGraph(self):
"""Test successfull execution of trivial quantum graph.
"""
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
self.assertEqual(len(qgraph.taskGraph), 5)
self.assertEqual(len(qgraph), nQuanta)
args = _makeArgs()
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock()
# run whole thing
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, nQuanta)
def testSimpleQGraphClobberPartialOutputs(self):
"""Test continuing execution of trivial quantum graph with
--clobber-partial-outputs.
"""
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
# should have one task and number of quanta
self.assertEqual(len(qgraph), nQuanta)
args = _makeArgs()
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock(stopAt=3)
# run first three quanta
with self.assertRaises(RuntimeError):
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, 3)
# drop one of the two outputs from one task
ref = butler._findDatasetRef("add2_dataset2",
instrument="INSTR",
detector=0)
self.assertIsNotNone(ref)
butler.pruneDatasets([ref],
disassociate=True,
unstore=True,
purge=True)
taskFactory.stopAt = -1
args.skip_existing = True
args.clobber_partial_outputs = True
args.no_versions = True
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
# number of executed quanta is incremented
self.assertEqual(taskFactory.countExec, nQuanta + 1)
def testSimpleQGraphReplaceRun(self):
"""Test repeated execution of trivial quantum graph with
--replace-run.
"""
# need non-memory registry in this case
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta,
root=self.root,
inMemory=False)
# should have one task and number of quanta
self.assertEqual(len(qgraph), nQuanta)
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock()
# run whole thing
args = _makeArgs(butler_config=self.root,
input="test",
output="output",
output_run="output/run1")
# deep copy is needed because quanta are updated in place
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
self.assertEqual(taskFactory.countExec, nQuanta)
# need to refresh collections explicitly (or make new butler/registry)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
self.assertEqual(collections, {"test", "output", "output/run1"})
# number of datasets written by pipeline:
# - nQuanta of init_outputs
# - nQuanta of configs
# - packages (single dataset)
# - nQuanta * two output datasets
# - nQuanta of metadata
n_outputs = nQuanta * 5 + 1
refs = butler.registry.queryDatasets(..., collections="output/run1")
self.assertEqual(len(list(refs)), n_outputs)
# re-run with --replace-run (--inputs is not compatible)
args.input = None
args.replace_run = True
args.output_run = "output/run2"
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
self.assertEqual(collections,
{"test", "output", "output/run1", "output/run2"})
# new output collection
refs = butler.registry.queryDatasets(..., collections="output/run2")
self.assertEqual(len(list(refs)), n_outputs)
# old output collection is still there
refs = butler.registry.queryDatasets(..., collections="output/run1")
self.assertEqual(len(list(refs)), n_outputs)
# re-run with --replace-run and --prune-replaced=unstore
args.input = None
args.replace_run = True
args.prune_replaced = "unstore"
args.output_run = "output/run3"
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
self.assertEqual(
collections,
{"test", "output", "output/run1", "output/run2", "output/run3"})
# new output collection
refs = butler.registry.queryDatasets(..., collections="output/run3")
self.assertEqual(len(list(refs)), n_outputs)
# old output collection is still there, and it has all datasets but
# they are not in datastore
refs = butler.registry.queryDatasets(..., collections="output/run2")
refs = list(refs)
self.assertEqual(len(refs), n_outputs)
with self.assertRaises(FileNotFoundError):
butler.get(refs[0], collections="output/run2")
# re-run with --replace-run and --prune-replaced=purge
args.input = None
args.replace_run = True
args.prune_replaced = "purge"
args.output_run = "output/run4"
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
# output/run3 should disappear now
self.assertEqual(
collections,
{"test", "output", "output/run1", "output/run2", "output/run4"})
# new output collection
refs = butler.registry.queryDatasets(..., collections="output/run4")
self.assertEqual(len(list(refs)), n_outputs)