def main():
if "CI_HSC_GEN3_DIR" not in os.environ:
raise RuntimeError("ci_hsc_gen3 is not setup.")
repo_root = os.path.join(os.environ["CI_HSC_GEN3_DIR"], "DATA")
butler = Butler(repo_root)
os.makedirs("DATA", exist_ok=False)
with butler.export(directory="DATA",
filename="ci_hsc.yaml",
transfer="link") as exporter:
for element in butler.registry.dimensions.getStaticElements():
if element.hasTable() and element.viewOf is None:
exporter.saveDimensionData(
element,
butler.registry.queryDimensionRecords(element),
)
for collection in butler.registry.queryCollections(...):
exporter.saveCollection(collection)
exporter.saveDatasets(
butler.registry.queryDatasets(..., collections=...))
def rewrite(dataset: FileDataset) -> FileDataset:
# Join the datastore root to the exported path. This should yield
# absolute paths that start with $CI_HSC_GEN2_DIR.
dataset.path = os.path.join(butler.datastore.root.ospath, dataset.path)
# Remove symlinks in the path; this should result in absolute paths
# that start with $TESTDATA_CI_HSC_DIR, because ci_hsc_gen2 always
# symlinks these datasets from there.
dataset.path = os.path.realpath(dataset.path)
# Recompute the path relative to $TESTDATA_CI_HSC_DIR, so we can deal
# with that moving around after the export file is created.
dataset.path = os.path.relpath(dataset.path,
getPackageDir("testdata_ci_hsc"))
return dataset
with butler.export(filename=args.filename) as export:
for datasetTypeName in ("brightObjectMask", "ps1_pv3_3pi_20170110",
"jointcal_photoCalib", "jointcal_wcs"):
export.saveDatasets(butler.registry.queryDatasets(datasetTypeName,
collections=...),
elements=(),
rewrite=rewrite)
for datasetTypeName in ("bias", "dark", "flat", "sky"):
export.saveDatasets(butler.registry.queryDatasets(datasetTypeName,
collections=...),
elements=(),
rewrite=rewrite)
for collection in butler.registry.queryCollections("HSC/defaults",
includeChains=True,
flattenChains=True):
export.saveCollection(collection)
du = DimensionUniverse()
qgraph = QuantumGraph.loadUri('/home/krughoff/public_html/data/two_ccd_processccd.qgraph', du)
exports = set()
def runner(nodes, exports, visited=None):
if not visited:
visited = set()
for node in nodes:
if node in visited:
continue
exports.update([ref for thing in node.quantum.inputs.values() for ref in thing])
exports.update([ref for thing in node.quantum.outputs.values() for ref in thing])
exports.update([ref for ref in node.quantum.initInputs.values()])
before = qgraph.determineAncestorsOfQuantumNode(node)
visited.add(node)
if before:
runner(before, exports, visited)
runner([node for node in qgraph.getNodesForTask(qgraph.findTaskDefByLabel('calibrate'))], exports)
resolved_refs = [butler.registry.findDataset(datasetType=ex.datasetType, dataId=ex.dataId,
collections=butler.registry.queryCollections()) for ex in exports]
with butler.export(filename='export.yaml', directory='rsp_data_export', transfer='copy') as export:
export.saveDatasets(resolved_refs)
export.saveCollection("HSC/calib")
export.saveCollection("HSC/calib/DM-28636")
export.saveCollection("HSC/calib/gen2/20180117")
export.saveCollection("HSC/calib/gen2/20180117/unbounded")
export.saveCollection("HSC/calib/DM-28636/unbounded")
export.saveCollection("HSC/calib/gen2/20180117")
export.saveCollection("HSC/calib/gen2/20200115")
export.saveCollection("refcats/DM-28636")
