def testAddInstrumentMismatch(self): """Verify that a RuntimeError is raised if the instrument in the user query does not match the instrument in the pipeline.""" with temporaryDirectory() as root: pipeline = simpleQGraph.makeSimplePipeline( nQuanta=5, instrument="lsst.pipe.base.tests.simpleQGraph.SimpleInstrument") with self.assertRaises(RuntimeError): simpleQGraph.makeSimpleQGraph(root=root, pipeline=pipeline, userQuery="instrument = 'foo'")
def testSubgraph(self): """Test successfull execution of trivial quantum graph. """ nQuanta = 5 nNodes = 2 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) # Select first two nodes for execution. This depends on node ordering # which I assume is the same as execution order. nodeIds = [node.nodeId.number for node in qgraph] nodeIds = nodeIds[:nNodes] self.assertEqual(len(qgraph.taskGraph), 5) self.assertEqual(len(qgraph), nQuanta) with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry( ) as registryConfig: with open(tmpname, "wb") as saveFile: qgraph.save(saveFile) args = _makeArgs(qgraph=tmpname, qgraph_node_id=nodeIds, registryConfig=registryConfig) fwk = CmdLineFwk() # load graph, should only read a subset qgraph = fwk.makeGraph(pipeline=None, args=args) self.assertEqual(len(qgraph), nNodes)
def testSimpleQGraphSkipExisting(self): """Test continuing execution of trivial quantum graph with --skip-existing. """ nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) self.assertEqual(len(qgraph.taskGraph), 5) self.assertEqual(len(qgraph), nQuanta) args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock(stopAt=3) # run first three quanta with self.assertRaises(RuntimeError): fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, 3) # run remaining ones taskFactory.stopAt = -1 args.skip_existing = True args.no_versions = True fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, nQuanta)
def testDefault(self): """Simple test to verify makeSimpleQGraph can be used to make a Quantum Graph.""" with temporaryDirectory() as root: # makeSimpleQGraph calls GraphBuilder. butler, qgraph = simpleQGraph.makeSimpleQGraph(root=root) # by default makeSimpleQGraph makes a graph with 5 nodes self.assertEqual(len(qgraph), 5)
def testShowGraphWorkflow(self): fwk = CmdLineFwk() nQuanta = 2 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) args = _makeArgs(show=["workflow"]) fwk.showInfo(args, pipeline=None, graph=qgraph)
def test_saveConfigs(self): for skipExisting in (False, True): with self.subTest(skipExisting=skipExisting): with temporaryDirectory() as tmpdir: butler, qgraph = makeSimpleQGraph(root=tmpdir) preExecInit = PreExecInit(butler=butler, taskFactory=None, skipExisting=skipExisting) preExecInit.saveConfigs(qgraph)
def test_saveInitOutputs(self): taskFactory = AddTaskFactoryMock() for skipExisting in (False, True): with self.subTest(skipExisting=skipExisting): with temporaryDirectory() as tmpdir: butler, qgraph = makeSimpleQGraph(root=tmpdir) preExecInit = PreExecInit(butler=butler, taskFactory=taskFactory, skipExisting=skipExisting) preExecInit.saveInitOutputs(qgraph)
def testShowGraph(self): """Test for --show options for quantum graph. """ fwk = CmdLineFwk() nQuanta = 2 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) args = _makeArgs(show=["graph"]) fwk.showInfo(args, pipeline=None, graph=qgraph)
def testDefault(self): """Simple test to verify makeSimpleQGraph can be used to make a Quantum Graph.""" with temporaryDirectory() as root: # makeSimpleQGraph calls GraphBuilder. butler, qgraph = simpleQGraph.makeSimpleQGraph(root=root) # by default makeSimpleQGraph makes a graph with 5 nodes self.assertEqual(len(qgraph), 5) constraint = DatasetQueryConstraintVariant.OFF _, qgraph2 = simpleQGraph.makeSimpleQGraph( butler=butler, datasetQueryConstraint=constraint, callPopulateButler=False) self.assertEqual(len(qgraph2), 5) self.assertEqual(qgraph, qgraph2) constraint = DatasetQueryConstraintVariant.fromExpression( "add_dataset0") _, qgraph3 = simpleQGraph.makeSimpleQGraph( butler=butler, datasetQueryConstraint=constraint, callPopulateButler=False) self.assertEqual(qgraph2, qgraph3)
def test_saveConfigs_twice(self): for skipExisting in (False, True): with self.subTest(skipExisting=skipExisting): with temporaryDirectory() as tmpdir: butler, qgraph = makeSimpleQGraph(root=tmpdir) preExecInit = PreExecInit(butler=butler, taskFactory=None, skipExisting=skipExisting) preExecInit.saveConfigs(qgraph) if skipExisting: # will ignore this preExecInit.saveConfigs(qgraph) else: # Second time it will fail with self.assertRaises(Exception): preExecInit.saveConfigs(qgraph)
def test_savePackageVersions_twice(self): for skipExisting in (False, True): with self.subTest(skipExisting=skipExisting): with temporaryDirectory() as tmpdir: butler, qgraph = makeSimpleQGraph(root=tmpdir) preExecInit = PreExecInit(butler=butler, taskFactory=None, skipExisting=skipExisting) preExecInit.savePackageVersions(qgraph) if skipExisting: # if this is the same packages then it should not attempt to save preExecInit.savePackageVersions(qgraph) else: # second time it will fail with self.assertRaises(Exception): preExecInit.savePackageVersions(qgraph)
def testSimpleQGraph(self): """Test successfull execution of trivial quantum graph. """ nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) self.assertEqual(len(qgraph.taskGraph), 5) self.assertEqual(len(qgraph), nQuanta) args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock() # run whole thing fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, nQuanta)
def test_datastore_records(self): """Test for generating datastore records.""" with temporaryDirectory() as root: # need FileDatastore for this tests butler, qgraph1 = simpleQGraph.makeSimpleQGraph( root=root, inMemory=False, makeDatastoreRecords=True) # save and reload buffer = io.BytesIO() qgraph1.save(buffer) buffer.seek(0) qgraph2 = QuantumGraph.load(buffer, universe=butler.dimensions) del buffer for qgraph in (qgraph1, qgraph2): self.assertEqual(len(qgraph), 5) for i, qnode in enumerate(qgraph): quantum = qnode.quantum self.assertIsNotNone(quantum.datastore_records) # only the first quantum has a pre-existing input if i == 0: datastore_name = "FileDatastore@<butlerRoot>" self.assertEqual(set(quantum.datastore_records.keys()), {datastore_name}) records_data = quantum.datastore_records[ datastore_name] records = dict(records_data.records) self.assertEqual(len(records), 1) _, records = records.popitem() records = records["file_datastore_records"] self.assertEqual( [record.path for record in records], [ "test/add_dataset0/add_dataset0_INSTR_det0_test.pickle" ], ) else: self.assertEqual(quantum.datastore_records, {})
def testSimpleQGraphClobberPartialOutputs(self): """Test continuing execution of trivial quantum graph with --clobber-partial-outputs. """ nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root) # should have one task and number of quanta self.assertEqual(len(qgraph), nQuanta) args = _makeArgs() fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock(stopAt=3) # run first three quanta with self.assertRaises(RuntimeError): fwk.runPipeline(qgraph, taskFactory, args, butler=butler) self.assertEqual(taskFactory.countExec, 3) # drop one of the two outputs from one task ref = butler._findDatasetRef("add2_dataset2", instrument="INSTR", detector=0) self.assertIsNotNone(ref) butler.pruneDatasets([ref], disassociate=True, unstore=True, purge=True) taskFactory.stopAt = -1 args.skip_existing = True args.clobber_partial_outputs = True args.no_versions = True fwk.runPipeline(qgraph, taskFactory, args, butler=butler) # number of executed quanta is incremented self.assertEqual(taskFactory.countExec, nQuanta + 1)
def testSimpleQGraphReplaceRun(self): """Test repeated execution of trivial quantum graph with --replace-run. """ # need non-memory registry in this case nQuanta = 5 butler, qgraph = makeSimpleQGraph(nQuanta, root=self.root, inMemory=False) # should have one task and number of quanta self.assertEqual(len(qgraph), nQuanta) fwk = CmdLineFwk() taskFactory = AddTaskFactoryMock() # run whole thing args = _makeArgs(butler_config=self.root, input="test", output="output", output_run="output/run1") # deep copy is needed because quanta are updated in place fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) self.assertEqual(taskFactory.countExec, nQuanta) # need to refresh collections explicitly (or make new butler/registry) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) self.assertEqual(collections, {"test", "output", "output/run1"}) # number of datasets written by pipeline: # - nQuanta of init_outputs # - nQuanta of configs # - packages (single dataset) # - nQuanta * two output datasets # - nQuanta of metadata n_outputs = nQuanta * 5 + 1 refs = butler.registry.queryDatasets(..., collections="output/run1") self.assertEqual(len(list(refs)), n_outputs) # re-run with --replace-run (--inputs is not compatible) args.input = None args.replace_run = True args.output_run = "output/run2" fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) self.assertEqual(collections, {"test", "output", "output/run1", "output/run2"}) # new output collection refs = butler.registry.queryDatasets(..., collections="output/run2") self.assertEqual(len(list(refs)), n_outputs) # old output collection is still there refs = butler.registry.queryDatasets(..., collections="output/run1") self.assertEqual(len(list(refs)), n_outputs) # re-run with --replace-run and --prune-replaced=unstore args.input = None args.replace_run = True args.prune_replaced = "unstore" args.output_run = "output/run3" fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) self.assertEqual( collections, {"test", "output", "output/run1", "output/run2", "output/run3"}) # new output collection refs = butler.registry.queryDatasets(..., collections="output/run3") self.assertEqual(len(list(refs)), n_outputs) # old output collection is still there, and it has all datasets but # they are not in datastore refs = butler.registry.queryDatasets(..., collections="output/run2") refs = list(refs) self.assertEqual(len(refs), n_outputs) with self.assertRaises(FileNotFoundError): butler.get(refs[0], collections="output/run2") # re-run with --replace-run and --prune-replaced=purge args.input = None args.replace_run = True args.prune_replaced = "purge" args.output_run = "output/run4" fwk.runPipeline(copy.deepcopy(qgraph), taskFactory, args) butler.registry.refresh() collections = set(butler.registry.queryCollections(...)) # output/run3 should disappear now self.assertEqual( collections, {"test", "output", "output/run1", "output/run2", "output/run4"}) # new output collection refs = butler.registry.queryDatasets(..., collections="output/run4") self.assertEqual(len(list(refs)), n_outputs)