def testAddInstrumentMismatch(self):
"""Verify that a RuntimeError is raised if the instrument in the user
query does not match the instrument in the pipeline."""
with temporaryDirectory() as root:
pipeline = simpleQGraph.makeSimplePipeline(
nQuanta=5,
instrument="lsst.pipe.base.tests.simpleQGraph.SimpleInstrument")
with self.assertRaises(RuntimeError):
simpleQGraph.makeSimpleQGraph(root=root,
pipeline=pipeline,
userQuery="instrument = 'foo'")
def testSubgraph(self):
"""Test successfull execution of trivial quantum graph.
"""
nQuanta = 5
nNodes = 2
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
# Select first two nodes for execution. This depends on node ordering
# which I assume is the same as execution order.
nodeIds = [node.nodeId.number for node in qgraph]
nodeIds = nodeIds[:nNodes]
self.assertEqual(len(qgraph.taskGraph), 5)
self.assertEqual(len(qgraph), nQuanta)
with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry(
) as registryConfig:
with open(tmpname, "wb") as saveFile:
qgraph.save(saveFile)
args = _makeArgs(qgraph=tmpname,
qgraph_node_id=nodeIds,
registryConfig=registryConfig)
fwk = CmdLineFwk()
# load graph, should only read a subset
qgraph = fwk.makeGraph(pipeline=None, args=args)
self.assertEqual(len(qgraph), nNodes)
def testSimpleQGraphSkipExisting(self):
"""Test continuing execution of trivial quantum graph with --skip-existing.
"""
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
self.assertEqual(len(qgraph.taskGraph), 5)
self.assertEqual(len(qgraph), nQuanta)
args = _makeArgs()
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock(stopAt=3)
# run first three quanta
with self.assertRaises(RuntimeError):
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, 3)
# run remaining ones
taskFactory.stopAt = -1
args.skip_existing = True
args.no_versions = True
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, nQuanta)
def testDefault(self):
"""Simple test to verify makeSimpleQGraph can be used to make a Quantum
Graph."""
with temporaryDirectory() as root:
# makeSimpleQGraph calls GraphBuilder.
butler, qgraph = simpleQGraph.makeSimpleQGraph(root=root)
# by default makeSimpleQGraph makes a graph with 5 nodes
self.assertEqual(len(qgraph), 5)
def testShowGraphWorkflow(self):
fwk = CmdLineFwk()
nQuanta = 2
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
args = _makeArgs(show=["workflow"])
fwk.showInfo(args, pipeline=None, graph=qgraph)
def test_saveConfigs(self):
for skipExisting in (False, True):
with self.subTest(skipExisting=skipExisting):
with temporaryDirectory() as tmpdir:
butler, qgraph = makeSimpleQGraph(root=tmpdir)
preExecInit = PreExecInit(butler=butler,
taskFactory=None,
skipExisting=skipExisting)
preExecInit.saveConfigs(qgraph)
def test_saveInitOutputs(self):
taskFactory = AddTaskFactoryMock()
for skipExisting in (False, True):
with self.subTest(skipExisting=skipExisting):
with temporaryDirectory() as tmpdir:
butler, qgraph = makeSimpleQGraph(root=tmpdir)
preExecInit = PreExecInit(butler=butler,
taskFactory=taskFactory,
skipExisting=skipExisting)
preExecInit.saveInitOutputs(qgraph)
def testShowGraph(self):
"""Test for --show options for quantum graph.
"""
fwk = CmdLineFwk()
nQuanta = 2
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
args = _makeArgs(show=["graph"])
fwk.showInfo(args, pipeline=None, graph=qgraph)
def testDefault(self):
"""Simple test to verify makeSimpleQGraph can be used to make a Quantum
Graph."""
with temporaryDirectory() as root:
# makeSimpleQGraph calls GraphBuilder.
butler, qgraph = simpleQGraph.makeSimpleQGraph(root=root)
# by default makeSimpleQGraph makes a graph with 5 nodes
self.assertEqual(len(qgraph), 5)
constraint = DatasetQueryConstraintVariant.OFF
_, qgraph2 = simpleQGraph.makeSimpleQGraph(
butler=butler,
datasetQueryConstraint=constraint,
callPopulateButler=False)
self.assertEqual(len(qgraph2), 5)
self.assertEqual(qgraph, qgraph2)
constraint = DatasetQueryConstraintVariant.fromExpression(
"add_dataset0")
_, qgraph3 = simpleQGraph.makeSimpleQGraph(
butler=butler,
datasetQueryConstraint=constraint,
callPopulateButler=False)
self.assertEqual(qgraph2, qgraph3)
def test_saveConfigs_twice(self):
for skipExisting in (False, True):
with self.subTest(skipExisting=skipExisting):
with temporaryDirectory() as tmpdir:
butler, qgraph = makeSimpleQGraph(root=tmpdir)
preExecInit = PreExecInit(butler=butler,
taskFactory=None,
skipExisting=skipExisting)
preExecInit.saveConfigs(qgraph)
if skipExisting:
# will ignore this
preExecInit.saveConfigs(qgraph)
else:
# Second time it will fail
with self.assertRaises(Exception):
preExecInit.saveConfigs(qgraph)
def test_savePackageVersions_twice(self):
for skipExisting in (False, True):
with self.subTest(skipExisting=skipExisting):
with temporaryDirectory() as tmpdir:
butler, qgraph = makeSimpleQGraph(root=tmpdir)
preExecInit = PreExecInit(butler=butler,
taskFactory=None,
skipExisting=skipExisting)
preExecInit.savePackageVersions(qgraph)
if skipExisting:
# if this is the same packages then it should not attempt to save
preExecInit.savePackageVersions(qgraph)
else:
# second time it will fail
with self.assertRaises(Exception):
preExecInit.savePackageVersions(qgraph)
def testSimpleQGraph(self):
"""Test successfull execution of trivial quantum graph.
"""
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
self.assertEqual(len(qgraph.taskGraph), 5)
self.assertEqual(len(qgraph), nQuanta)
args = _makeArgs()
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock()
# run whole thing
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, nQuanta)
def test_datastore_records(self):
"""Test for generating datastore records."""
with temporaryDirectory() as root:
# need FileDatastore for this tests
butler, qgraph1 = simpleQGraph.makeSimpleQGraph(
root=root, inMemory=False, makeDatastoreRecords=True)
# save and reload
buffer = io.BytesIO()
qgraph1.save(buffer)
buffer.seek(0)
qgraph2 = QuantumGraph.load(buffer, universe=butler.dimensions)
del buffer
for qgraph in (qgraph1, qgraph2):
self.assertEqual(len(qgraph), 5)
for i, qnode in enumerate(qgraph):
quantum = qnode.quantum
self.assertIsNotNone(quantum.datastore_records)
# only the first quantum has a pre-existing input
if i == 0:
datastore_name = "FileDatastore@<butlerRoot>"
self.assertEqual(set(quantum.datastore_records.keys()),
{datastore_name})
records_data = quantum.datastore_records[
datastore_name]
records = dict(records_data.records)
self.assertEqual(len(records), 1)
_, records = records.popitem()
records = records["file_datastore_records"]
self.assertEqual(
[record.path for record in records],
[
"test/add_dataset0/add_dataset0_INSTR_det0_test.pickle"
],
)
else:
self.assertEqual(quantum.datastore_records, {})
def testSimpleQGraphClobberPartialOutputs(self):
"""Test continuing execution of trivial quantum graph with
--clobber-partial-outputs.
"""
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root)
# should have one task and number of quanta
self.assertEqual(len(qgraph), nQuanta)
args = _makeArgs()
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock(stopAt=3)
# run first three quanta
with self.assertRaises(RuntimeError):
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
self.assertEqual(taskFactory.countExec, 3)
# drop one of the two outputs from one task
ref = butler._findDatasetRef("add2_dataset2",
instrument="INSTR",
detector=0)
self.assertIsNotNone(ref)
butler.pruneDatasets([ref],
disassociate=True,
unstore=True,
purge=True)
taskFactory.stopAt = -1
args.skip_existing = True
args.clobber_partial_outputs = True
args.no_versions = True
fwk.runPipeline(qgraph, taskFactory, args, butler=butler)
# number of executed quanta is incremented
self.assertEqual(taskFactory.countExec, nQuanta + 1)
def testSimpleQGraphReplaceRun(self):
"""Test repeated execution of trivial quantum graph with
--replace-run.
"""
# need non-memory registry in this case
nQuanta = 5
butler, qgraph = makeSimpleQGraph(nQuanta,
root=self.root,
inMemory=False)
# should have one task and number of quanta
self.assertEqual(len(qgraph), nQuanta)
fwk = CmdLineFwk()
taskFactory = AddTaskFactoryMock()
# run whole thing
args = _makeArgs(butler_config=self.root,
input="test",
output="output",
output_run="output/run1")
# deep copy is needed because quanta are updated in place
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
self.assertEqual(taskFactory.countExec, nQuanta)
# need to refresh collections explicitly (or make new butler/registry)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
self.assertEqual(collections, {"test", "output", "output/run1"})
# number of datasets written by pipeline:
# - nQuanta of init_outputs
# - nQuanta of configs
# - packages (single dataset)
# - nQuanta * two output datasets
# - nQuanta of metadata
n_outputs = nQuanta * 5 + 1
refs = butler.registry.queryDatasets(..., collections="output/run1")
self.assertEqual(len(list(refs)), n_outputs)
# re-run with --replace-run (--inputs is not compatible)
args.input = None
args.replace_run = True
args.output_run = "output/run2"
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
self.assertEqual(collections,
{"test", "output", "output/run1", "output/run2"})
# new output collection
refs = butler.registry.queryDatasets(..., collections="output/run2")
self.assertEqual(len(list(refs)), n_outputs)
# old output collection is still there
refs = butler.registry.queryDatasets(..., collections="output/run1")
self.assertEqual(len(list(refs)), n_outputs)
# re-run with --replace-run and --prune-replaced=unstore
args.input = None
args.replace_run = True
args.prune_replaced = "unstore"
args.output_run = "output/run3"
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
self.assertEqual(
collections,
{"test", "output", "output/run1", "output/run2", "output/run3"})
# new output collection
refs = butler.registry.queryDatasets(..., collections="output/run3")
self.assertEqual(len(list(refs)), n_outputs)
# old output collection is still there, and it has all datasets but
# they are not in datastore
refs = butler.registry.queryDatasets(..., collections="output/run2")
refs = list(refs)
self.assertEqual(len(refs), n_outputs)
with self.assertRaises(FileNotFoundError):
butler.get(refs[0], collections="output/run2")
# re-run with --replace-run and --prune-replaced=purge
args.input = None
args.replace_run = True
args.prune_replaced = "purge"
args.output_run = "output/run4"
fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args)
butler.registry.refresh()
collections = set(butler.registry.queryCollections(...))
# output/run3 should disappear now
self.assertEqual(
collections,
{"test", "output", "output/run1", "output/run2", "output/run4"})
# new output collection
refs = butler.registry.queryDatasets(..., collections="output/run4")
self.assertEqual(len(list(refs)), n_outputs)