def testSaveLoadUriS3(self):
# Test loading a quantum graph from an mock s3 store
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket="testBucket")
uri = "s3://testBucket/qgraph.qgraph"
self.qGraph.saveUri(uri)
restore = QuantumGraph.loadUri(uri, self.universe)
self.assertEqual(self.qGraph, restore)
nodeId = list(self.qGraph)[0].nodeId
restoreSub = QuantumGraph.loadUri(uri, self.universe, nodes=(nodeId,))
self.assertEqual(len(restoreSub), 1)
self.assertEqual(list(restoreSub)[0], restore.getQuantumNodeByNodeId(nodeId))
def testSaveLoadUriS3(self):
# Test loading a quantum graph from an mock s3 store
conn = boto3.resource('s3', region_name="us-east-1")
conn.create_bucket(Bucket='testBucket')
uri = f"s3://testBucket/qgraph.qgraph"
self.qGraph.saveUri(uri)
restore = QuantumGraph.loadUri(uri, self.universe)
self._cleanGraphs(self.qGraph, restore)
self.assertEqual(self.qGraph, restore)
restoreSub = QuantumGraph.loadUri(uri, self.universe, nodes=(0, ))
self.assertEqual(len(restoreSub), 1)
self.assertEqual(
list(restoreSub)[0],
restore.getQuantumNodeByNodeId(NodeId(0, restore._buildId)))
def makeGraph(self, pipeline, args):
"""Build a graph from command line arguments.
Parameters
----------
pipeline : `~lsst.pipe.base.Pipeline`
Pipeline, can be empty or ``None`` if graph is read from a file.
args : `argparse.Namespace`
Parsed command line
Returns
-------
graph : `~lsst.pipe.base.QuantumGraph` or `None`
If resulting graph is empty then `None` is returned.
"""
registry, collections, run = _ButlerFactory.makeRegistryAndCollections(args)
if args.qgraph:
# click passes empty tuple as default value for qgraph_node_id
nodes = args.qgraph_node_id or None
qgraph = QuantumGraph.loadUri(args.qgraph, registry.dimensions,
nodes=nodes, graphID=args.qgraph_id)
# pipeline can not be provided in this case
if pipeline:
raise ValueError("Pipeline must not be given when quantum graph is read from file.")
else:
# make execution plan (a.k.a. DAG) for pipeline
graphBuilder = GraphBuilder(registry,
skipExisting=args.skip_existing)
qgraph = graphBuilder.makeGraph(pipeline, collections, run, args.data_query)
# count quanta in graph and give a warning if it's empty and return None
nQuanta = len(qgraph)
if nQuanta == 0:
warnings.warn("QuantumGraph is empty", stacklevel=2)
return None
else:
_LOG.info("QuantumGraph contains %d quanta for %d tasks, graph ID: %r",
nQuanta, len(qgraph.taskGraph), qgraph.graphID)
if args.save_qgraph:
qgraph.saveUri(args.save_qgraph)
if args.save_single_quanta:
for quantumNode in qgraph:
sqgraph = qgraph.subset(quantumNode)
uri = args.save_single_quanta.format(quantumNode.nodeId.number)
sqgraph.saveUri(uri)
if args.qgraph_dot:
graph2dot(qgraph, args.qgraph_dot)
return qgraph
def testSaveLoadUri(self):
uri = None
try:
with tempfile.NamedTemporaryFile(delete=False,
suffix=".qgraph") as tmpFile:
uri = tmpFile.name
self.qGraph.saveUri(uri)
restore = QuantumGraph.loadUri(uri, self.universe)
self._cleanGraphs(self.qGraph, restore)
self.assertEqual(self.qGraph, restore)
nodeNumber = random.randint(0, len(self.qGraph) - 1)
restoreSub = QuantumGraph.loadUri(uri,
self.universe,
nodes=(nodeNumber, ),
graphID=self.qGraph._buildId)
self.assertEqual(len(restoreSub), 1)
self.assertEqual(
list(restoreSub)[0],
restore.getQuantumNodeByNodeId(
NodeId(nodeNumber, restore.graphID)))
# verify that more than one node works
nodeNumber2 = random.randint(0, len(self.qGraph) - 1)
# ensure it is a different node number
while nodeNumber2 == nodeNumber:
nodeNumber2 = random.randint(0, len(self.qGraph) - 1)
restoreSub = QuantumGraph.loadUri(uri,
self.universe,
nodes=(nodeNumber,
nodeNumber2))
self.assertEqual(len(restoreSub), 2)
self.assertEqual(
set(restoreSub),
set((restore.getQuantumNodeByNodeId(
NodeId(nodeNumber, restore._buildId)),
restore.getQuantumNodeByNodeId(
NodeId(nodeNumber2, restore._buildId)))))
# verify an error when requesting a non existant node number
with self.assertRaises(ValueError):
QuantumGraph.loadUri(uri, self.universe, nodes=(99, ))
# verify a graphID that does not match will be an error
with self.assertRaises(ValueError):
QuantumGraph.loadUri(uri,
self.universe,
graphID="NOTRIGHT")
except Exception as e:
raise e
finally:
if uri is not None:
os.remove(uri)
with self.assertRaises(TypeError):
self.qGraph.saveUri("test.notgraph")
#!/usr/bin/env python
from lsst.pipe.base import QuantumGraph
from lsst.daf.butler import DimensionUniverse, Butler
butler = Butler('/repo/main')
du = DimensionUniverse()
qgraph = QuantumGraph.loadUri('/home/krughoff/public_html/data/two_ccd_processccd.qgraph', du)
exports = set()
def runner(nodes, exports, visited=None):
if not visited:
visited = set()
for node in nodes:
if node in visited:
continue
exports.update([ref for thing in node.quantum.inputs.values() for ref in thing])
exports.update([ref for thing in node.quantum.outputs.values() for ref in thing])
exports.update([ref for ref in node.quantum.initInputs.values()])
before = qgraph.determineAncestorsOfQuantumNode(node)
visited.add(node)
if before:
runner(before, exports, visited)
runner([node for node in qgraph.getNodesForTask(qgraph.findTaskDefByLabel('calibrate'))], exports)
resolved_refs = [butler.registry.findDataset(datasetType=ex.datasetType, dataId=ex.dataId,
collections=butler.registry.queryCollections()) for ex in exports]
with butler.export(filename='export.yaml', directory='rsp_data_export', transfer='copy') as export:
export.saveDatasets(resolved_refs)
export.saveCollection("HSC/calib")
export.saveCollection("HSC/calib/DM-28636")
export.saveCollection("HSC/calib/gen2/20180117")
export.saveCollection("HSC/calib/gen2/20180117/unbounded")