def testMakeGraphFromPickle(self):
"""Tests for CmdLineFwk.makeGraph method.
Only most trivial case is tested that does not do actual graph
building.
"""
fwk = CmdLineFwk()
taskFactory = TaskFactoryMock()
with makeTmpFile() as tmpname:
# make empty graph and store it in a file
qgraph = QuantumGraph()
with open(tmpname, "wb") as pickleFile:
pickle.dump(qgraph, pickleFile)
args = _makeArgs(qgraph=tmpname)
with self.assertWarnsRegex(UserWarning, "QuantumGraph is empty"):
# this also tests that warning is generated for empty graph
qgraph = fwk.makeGraph(None, taskFactory, args)
self.assertIsInstance(qgraph, QuantumGraph)
self.assertEqual(len(qgraph), 0)
# pickle with wrong object type
with open(tmpname, "wb") as pickleFile:
pickle.dump({}, pickleFile)
args = _makeArgs(qgraph=tmpname)
with self.assertRaises(TypeError):
fwk.makeGraph(None, taskFactory, args)
def testSubgraph(self):
"""Test successfull execution of trivial quantum graph.
"""
nQuanta = 5
nNodes = 2
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
# Select first two nodes for execution. This depends on node ordering
# which I assume is the same as execution order.
nodeIds = [node.nodeId.number for node in qgraph]
nodeIds = nodeIds[:nNodes]
self.assertEqual(len(qgraph.taskGraph), 5)
self.assertEqual(len(qgraph), nQuanta)
with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry(
) as registryConfig:
with open(tmpname, "wb") as saveFile:
qgraph.save(saveFile)
args = _makeArgs(qgraph=tmpname,
qgraph_node_id=nodeIds,
registryConfig=registryConfig)
fwk = CmdLineFwk()
# load graph, should only read a subset
qgraph = fwk.makeGraph(pipeline=None, args=args)
self.assertEqual(len(qgraph), nNodes)
def testShowGraphWorkflow(self):
fwk = CmdLineFwk()
nQuanta = 2
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
args = _makeArgs(show=["workflow"])
fwk.showInfo(args, pipeline=None, graph=qgraph)
def testShowGraph(self):
"""Test for --show options for quantum graph.
"""
fwk = CmdLineFwk()
nQuanta = 2
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
args = _makeArgs(show=["graph"])
fwk.showInfo(args, pipeline=None, graph=qgraph)
def testMakePipeline(self):
"""Tests for CmdLineFwk.makePipeline method
"""
fwk = CmdLineFwk()
taskFactory = TaskFactoryMock()
# make empty pipeline
args = _makeArgs()
pipeline = fwk.makePipeline(taskFactory, args)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 0)
# few tests with pickle
with makeTmpFile() as tmpname:
# make empty pipeline and store it in a file
args = _makeArgs(save_pipeline=tmpname)
pipeline = fwk.makePipeline(taskFactory, args)
self.assertIsInstance(pipeline, Pipeline)
# read pipeline from a file
args = _makeArgs(pipeline=tmpname)
pipeline = fwk.makePipeline(taskFactory, args)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 0)
# pickle with wrong object type
with open(tmpname, "wb") as pickleFile:
pickle.dump({}, pickleFile)
args = _makeArgs(pipeline=tmpname)
with self.assertRaises(TypeError):
fwk.makePipeline(taskFactory, args)
# single task pipeline
actions = [
_PipelineAction(action="new_task", label="task1", value="TaskOne")
]
args = _makeArgs(pipeline_actions=actions)
pipeline = fwk.makePipeline(taskFactory, args)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 1)
# many task pipeline
actions = [
_PipelineAction(action="new_task", label="task1a",
value="TaskOne"),
_PipelineAction(action="new_task", label="task2", value="TaskTwo"),
_PipelineAction(action="new_task", label="task1b", value="TaskOne")
]
args = _makeArgs(pipeline_actions=actions)
pipeline = fwk.makePipeline(taskFactory, args)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 3)
def testSimpleQGraph(self):
"""Test successfull execution of trivial quantum graph.
"""
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
self.assertEqual(len(qgraph.taskGraph), 5)
self.assertEqual(len(qgraph), nQuanta)
args = _makeArgs()
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock()
# run whole thing
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, nQuanta)
def testSimpleQGraphSkipExisting(self):
"""Test continuing execution of trivial quantum graph with --skip-existing.
"""
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
self.assertEqual(len(qgraph.taskGraph), 5)
self.assertEqual(len(qgraph), nQuanta)
args = _makeArgs()
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock(stopAt=3)
# run first three quanta
with self.assertRaises(RuntimeError):
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, 3)
# run remaining ones
taskFactory.stopAt = -1
args.skip_existing = True
args.no_versions = True
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, nQuanta)
def testMakeGraphFromSave(self):
"""Tests for CmdLineFwk.makeGraph method.
Only most trivial case is tested that does not do actual graph
building.
"""
fwk = CmdLineFwk()
with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry(
) as registryConfig:
# make non-empty graph and store it in a file
qgraph = _makeQGraph()
with open(tmpname, "wb") as saveFile:
qgraph.save(saveFile)
args = _makeArgs(qgraph=tmpname, registryConfig=registryConfig)
qgraph = fwk.makeGraph(None, args)
self.assertIsInstance(qgraph, QuantumGraph)
self.assertEqual(len(qgraph), 1)
# will fail if graph id does not match
args = _makeArgs(qgraph=tmpname,
qgraph_id="R2-D2 is that you?",
registryConfig=registryConfig)
with self.assertRaisesRegex(ValueError, "graphID does not match"):
fwk.makeGraph(None, args)
# save with wrong object type
with open(tmpname, "wb") as saveFile:
pickle.dump({}, saveFile)
args = _makeArgs(qgraph=tmpname, registryConfig=registryConfig)
with self.assertRaises(ValueError):
fwk.makeGraph(None, args)
# reading empty graph from pickle should work but makeGraph()
# will return None and make a warning
qgraph = QuantumGraph(dict())
with open(tmpname, "wb") as saveFile:
qgraph.save(saveFile)
args = _makeArgs(qgraph=tmpname, registryConfig=registryConfig)
with self.assertWarnsRegex(UserWarning, "QuantumGraph is empty"):
# this also tests that warning is generated for empty graph
qgraph = fwk.makeGraph(None, args)
self.assertIs(qgraph, None)
def testSimpleQGraphClobberPartialOutputs(self):
"""Test continuing execution of trivial quantum graph with
--clobber-partial-outputs.
"""
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
# should have one task and number of quanta
self.assertEqual(len(qgraph), nQuanta)
args = _makeArgs()
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock(stopAt=3)
# run first three quanta
with self.assertRaises(RuntimeError):
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, 3)
# drop one of the two outputs from one task
ref = butler._findDatasetRef("add2_dataset2",
instrument="INSTR",
detector=0)
self.assertIsNotNone(ref)
butler.pruneDatasets([ref],
disassociate=True,
unstore=True,
purge=True)
taskFactory.stopAt = -1
args.skip_existing = True
args.clobber_partial_outputs = True
args.no_versions = True
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
# number of executed quanta is incremented
self.assertEqual(taskFactory.countExec, nQuanta + 1)
def testSimpleQGraphReplaceRun(self):
"""Test repeated execution of trivial quantum graph with
--replace-run.
"""
# need non-memory registry in this case
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta,
root=self.root,
inMemory=False)
# should have one task and number of quanta
self.assertEqual(len(qgraph), nQuanta)
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock()
# run whole thing
args = _makeArgs(butler_config=self.root,
input="test",
output="output",
output_run="output/run1")
# deep copy is needed because quanta are updated in place
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
self.assertEqual(taskFactory.countExec, nQuanta)
# need to refresh collections explicitly (or make new butler/registry)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
self.assertEqual(collections, {"test", "output", "output/run1"})
# number of datasets written by pipeline:
# - nQuanta of init_outputs
# - nQuanta of configs
# - packages (single dataset)
# - nQuanta * two output datasets
# - nQuanta of metadata
n_outputs = nQuanta * 5 + 1
refs = butler.registry.queryDatasets(..., collections="output/run1")
self.assertEqual(len(list(refs)), n_outputs)
# re-run with --replace-run (--inputs is not compatible)
args.input = None
args.replace_run = True
args.output_run = "output/run2"
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
self.assertEqual(collections,
{"test", "output", "output/run1", "output/run2"})
# new output collection
refs = butler.registry.queryDatasets(..., collections="output/run2")
self.assertEqual(len(list(refs)), n_outputs)
# old output collection is still there
refs = butler.registry.queryDatasets(..., collections="output/run1")
self.assertEqual(len(list(refs)), n_outputs)
# re-run with --replace-run and --prune-replaced=unstore
args.input = None
args.replace_run = True
args.prune_replaced = "unstore"
args.output_run = "output/run3"
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
self.assertEqual(
collections,
{"test", "output", "output/run1", "output/run2", "output/run3"})
# new output collection
refs = butler.registry.queryDatasets(..., collections="output/run3")
self.assertEqual(len(list(refs)), n_outputs)
# old output collection is still there, and it has all datasets but
# they are not in datastore
refs = butler.registry.queryDatasets(..., collections="output/run2")
refs = list(refs)
self.assertEqual(len(refs), n_outputs)
with self.assertRaises(FileNotFoundError):
butler.get(refs[0], collections="output/run2")
# re-run with --replace-run and --prune-replaced=purge
args.input = None
args.replace_run = True
args.prune_replaced = "purge"
args.output_run = "output/run4"
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
# output/run3 should disappear now
self.assertEqual(
collections,
{"test", "output", "output/run1", "output/run2", "output/run4"})
# new output collection
refs = butler.registry.queryDatasets(..., collections="output/run4")
self.assertEqual(len(list(refs)), n_outputs)
def testShowPipeline(self):
"""Test for --show options for pipeline.
"""
fwk = CmdLineFwk()
actions = [
_ACTION_ADD_TASK("lsst.pipe.base.tests.simpleQGraph.AddTask:task"),
_ACTION_CONFIG("task:addend=100")
]
args = _makeArgs(pipeline_actions=actions)
pipeline = fwk.makePipeline(args)
args.show = ["pipeline"]
fwk.showInfo(args, pipeline)
args.show = ["config"]
fwk.showInfo(args, pipeline)
args.show = ["history=task::addend"]
fwk.showInfo(args, pipeline)
args.show = ["tasks"]
fwk.showInfo(args, pipeline)
def testMakePipeline(self):
"""Tests for CmdLineFwk.makePipeline method
"""
fwk = CmdLineFwk()
# make empty pipeline
args = _makeArgs()
pipeline = fwk.makePipeline(args)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 0)
# few tests with serialization
with makeTmpFile() as tmpname:
# make empty pipeline and store it in a file
args = _makeArgs(save_pipeline=tmpname)
pipeline = fwk.makePipeline(args)
self.assertIsInstance(pipeline, Pipeline)
# read pipeline from a file
args = _makeArgs(pipeline=tmpname)
pipeline = fwk.makePipeline(args)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 0)
# single task pipeline
actions = [_ACTION_ADD_TASK("TaskOne:task1")]
args = _makeArgs(pipeline_actions=actions)
pipeline = fwk.makePipeline(args)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 1)
# many task pipeline
actions = [
_ACTION_ADD_TASK("TaskOne:task1a"),
_ACTION_ADD_TASK("TaskTwo:task2"),
_ACTION_ADD_TASK("TaskOne:task1b")
]
args = _makeArgs(pipeline_actions=actions)
pipeline = fwk.makePipeline(args)
self.assertIsInstance(pipeline, Pipeline)
self.assertEqual(len(pipeline), 3)
# single task pipeline with config overrides, cannot use TaskOne, need
# something that can be imported with `doImport()`
actions = [
_ACTION_ADD_TASK("lsst.pipe.base.tests.simpleQGraph.AddTask:task"),
_ACTION_CONFIG("task:addend=100")
]
args = _makeArgs(pipeline_actions=actions)
pipeline = fwk.makePipeline(args)
taskDefs = list(pipeline.toExpandedPipeline())
self.assertEqual(len(taskDefs), 1)
self.assertEqual(taskDefs[0].config.addend, 100)
overrides = b"config.addend = 1000\n"
with makeTmpFile(overrides) as tmpname:
actions = [
_ACTION_ADD_TASK(
"lsst.pipe.base.tests.simpleQGraph.AddTask:task"),
_ACTION_CONFIG_FILE("task:" + tmpname)
]
args = _makeArgs(pipeline_actions=actions)
pipeline = fwk.makePipeline(args)
taskDefs = list(pipeline.toExpandedPipeline())
self.assertEqual(len(taskDefs), 1)
self.assertEqual(taskDefs[0].config.addend, 1000)
# Check --instrument option, for now it only checks that it does not crash
actions = [
_ACTION_ADD_TASK("lsst.pipe.base.tests.simpleQGraph.AddTask:task"),
_ACTION_ADD_INSTRUMENT("Instrument")
]
args = _makeArgs(pipeline_actions=actions)
pipeline = fwk.makePipeline(args)