def testMatplotlibFormatter(self):
butler = Butler(self.root, run="testrun")
datasetType = DatasetType("test_plot", [],
"Plot",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
pyplot.imshow(np.random.randn(3, 4))
ref = butler.put(pyplot.gcf(), datasetType)
parsed = urllib.parse.urlparse(butler.getUri(ref))
with tempfile.NamedTemporaryFile(suffix=".png") as file:
pyplot.gcf().savefig(file.name)
self.assertTrue(filecmp.cmp(parsed.path, file.name, shallow=True))
self.assertTrue(butler.datasetExists(ref))
with self.assertRaises(ValueError):
butler.get(ref)
butler.remove(ref)
with self.assertRaises(LookupError):
butler.datasetExists(ref)
def testIngest(self):
butler = Butler(self.tmpConfigFile, run="ingest")
# Create and register a DatasetType
dimensions = butler.registry.dimensions.extract(
["instrument", "visit", "detector"])
storageClass = self.storageClassFactory.getStorageClass(
"StructuredDataDictYaml")
datasetTypeName = "metric"
datasetType = self.addDatasetType(datasetTypeName, dimensions,
storageClass, butler.registry)
# Add needed Dimensions
butler.registry.insertDimensionData("instrument",
{"name": "DummyCamComp"})
butler.registry.insertDimensionData("physical_filter", {
"instrument": "DummyCamComp",
"name": "d-r",
"abstract_filter": "R"
})
for detector in (1, 2):
butler.registry.insertDimensionData(
"detector", {
"instrument": "DummyCamComp",
"id": detector,
"full_name": f"detector{detector}"
})
butler.registry.insertDimensionData(
"visit", {
"instrument": "DummyCamComp",
"id": 423,
"name": "fourtwentythree",
"physical_filter": "d-r"
}, {
"instrument": "DummyCamComp",
"id": 424,
"name": "fourtwentyfour",
"physical_filter": "d-r"
})
formatter = doImport(
"lsst.daf.butler.formatters.yamlFormatter.YamlFormatter")
dataRoot = os.path.join(TESTDIR, "data", "basic")
datasets = []
for detector in (1, 2):
detector_name = f"detector_{detector}"
metricFile = os.path.join(dataRoot, f"{detector_name}.yaml")
dataId = {
"instrument": "DummyCamComp",
"visit": 423,
"detector": detector
}
# Create a DatasetRef for ingest
refIn = DatasetRef(datasetType, dataId, id=None)
datasets.append(
FileDataset(path=metricFile, refs=[refIn],
formatter=formatter))
butler.ingest(*datasets, transfer="copy")
dataId1 = {"instrument": "DummyCamComp", "detector": 1, "visit": 423}
dataId2 = {"instrument": "DummyCamComp", "detector": 2, "visit": 423}
metrics1 = butler.get(datasetTypeName, dataId1)
metrics2 = butler.get(datasetTypeName, dataId2)
self.assertNotEqual(metrics1, metrics2)
# Compare URIs
uri1 = butler.getUri(datasetTypeName, dataId1)
uri2 = butler.getUri(datasetTypeName, dataId2)
self.assertNotEqual(uri1, uri2)
# Now do a multi-dataset but single file ingest
metricFile = os.path.join(dataRoot, "detectors.yaml")
refs = []
for detector in (1, 2):
detector_name = f"detector_{detector}"
dataId = {
"instrument": "DummyCamComp",
"visit": 424,
"detector": detector
}
# Create a DatasetRef for ingest
refs.append(DatasetRef(datasetType, dataId, id=None))
datasets = []
datasets.append(
FileDataset(path=metricFile,
refs=refs,
formatter=MultiDetectorFormatter))
butler.ingest(*datasets, transfer="copy")
dataId1 = {"instrument": "DummyCamComp", "detector": 1, "visit": 424}
dataId2 = {"instrument": "DummyCamComp", "detector": 2, "visit": 424}
multi1 = butler.get(datasetTypeName, dataId1)
multi2 = butler.get(datasetTypeName, dataId2)
self.assertEqual(multi1, metrics1)
self.assertEqual(multi2, metrics2)
# Compare URIs
uri1 = butler.getUri(datasetTypeName, dataId1)
uri2 = butler.getUri(datasetTypeName, dataId2)
self.assertEqual(uri1, uri2)
# Test that removing one does not break the second
butler.remove(datasetTypeName, dataId1)
with self.assertRaises(LookupError):
butler.datasetExists(datasetTypeName, dataId1)
self.assertTrue(butler.datasetExists(datasetTypeName, dataId2))
multi2b = butler.get(datasetTypeName, dataId2)
self.assertEqual(multi2, multi2b)
