def testTransfer(self):
metrics = makeExampleMetrics()
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 2048, "filter": "Uprime"}
sc = self.storageClassFactory.getStorageClass("StructuredData")
ref = self.makeDatasetRef("metric", dataUnits, sc, dataId)
inputConfig = DatastoreConfig(self.configFile)
inputConfig['datastore.root'] = os.path.join(self.testDir,
"./test_input_datastore")
inputPosixDatastore = PosixDatastore(config=inputConfig,
registry=self.registry)
outputConfig = inputConfig.copy()
outputConfig['datastore.root'] = os.path.join(
self.testDir, "./test_output_datastore")
outputPosixDatastore = PosixDatastore(config=outputConfig,
registry=DummyRegistry())
inputPosixDatastore.put(metrics, ref)
outputPosixDatastore.transfer(inputPosixDatastore, ref)
metricsOut = outputPosixDatastore.get(ref)
self.assertEqual(metrics, metricsOut)
def testBasicPutGet(self):
catalog = self.makeExampleCatalog()
datastore = PosixDatastore(config=self.configFile,
registry=self.registry)
# Put
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 123456, "filter": "blue"}
storageClass = self.storageClassFactory.getStorageClass(
"SourceCatalog")
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
datastore.put(catalog, ref)
# Does it exist?
self.assertTrue(datastore.exists(ref))
uri = datastore.getUri(ref)
self.assertTrue(uri.endswith(".fits"))
self.assertTrue(uri.startswith("file:"))
# Get
catalogOut = datastore.get(ref, parameters=None)
self.assertCatalogEqual(catalog, catalogOut)
# These should raise
ref = self.makeDatasetRef("calexp2", dataUnits, storageClass, dataId)
with self.assertRaises(FileNotFoundError):
# non-existing file
datastore.get(ref, parameters=None)
def testTransfer(self):
catalog = self.makeExampleCatalog()
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 12345, "filter": "red"}
storageClass = self.storageClassFactory.getStorageClass(
"SourceCatalog")
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
inputConfig = DatastoreConfig(self.configFile)
inputConfig['datastore.root'] = os.path.join(self.testDir,
"./test_input_datastore")
inputPosixDatastore = PosixDatastore(config=inputConfig,
registry=self.registry)
outputConfig = inputConfig.copy()
outputConfig['datastore.root'] = os.path.join(
self.testDir, "./test_output_datastore")
outputPosixDatastore = PosixDatastore(config=outputConfig,
registry=DummyRegistry())
inputPosixDatastore.put(catalog, ref)
outputPosixDatastore.transfer(inputPosixDatastore, ref)
catalogOut = outputPosixDatastore.get(ref)
self.assertCatalogEqual(catalog, catalogOut)
def testExposurePutGet(self):
example = os.path.join(self.testDir, "data", "basic", "small.fits")
exposure = lsst.afw.image.ExposureF(example)
datastore = PosixDatastore(config=self.configFile, registry=self.registry)
# Put
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 231, "filter": "Fc"}
storageClass = datastore.storageClassFactory.getStorageClass("ExposureF")
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
datastore.put(exposure, ref)
# Does it exist?
self.assertTrue(datastore.exists(ref))
# Get
exposureOut = datastore.get(ref)
self.assertEqual(type(exposure), type(exposureOut))
# Get some components
for compName in ("wcs", "image", "mask", "coaddInputs", "psf"):
compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dataUnits,
storageClass.components[compName], dataId, id=ref.id)
component = datastore.get(compRef)
self.assertIsInstance(component, compRef.datasetType.storageClass.pytype)
# Get the WCS component to check it
wcsRef = self.makeDatasetRef(ref.datasetType.componentTypeName("wcs"), dataUnits,
storageClass.components["wcs"], dataId, id=ref.id)
wcs = datastore.get(wcsRef)
# Simple check of WCS
bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0, 0),
lsst.afw.geom.Extent2I(9, 9))
self.assertWcsAlmostEqualOverBBox(wcs, exposure.getWcs(), bbox)
def testExposureCompositePutGet(self):
example = os.path.join(self.testDir, "data", "basic", "small.fits")
exposure = lsst.afw.image.ExposureF(example)
datastore = PosixDatastore(config=self.configFile,
registry=self.registry)
# Put
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 23, "filter": "F"}
storageClass = datastore.storageClassFactory.getStorageClass(
"ExposureCompositeF")
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
# Get the predicted URI
self.assertFalse(datastore.exists(ref))
uri = datastore.getUri(ref, predict=True)
self.assertTrue(uri.endswith("#predicted"))
components = storageClass.assembler().disassemble(exposure)
self.assertTrue(components)
# Get a component
compsRead = {}
for compName in ("wcs", "image", "mask", "coaddInputs", "psf"):
compRef = self.makeDatasetRef(
ref.datasetType.componentTypeName(compName), dataUnits,
components[compName].storageClass, dataId)
datastore.put(components[compName].component, compRef)
# Does it exist?
self.assertTrue(datastore.exists(compRef))
component = datastore.get(compRef)
self.assertIsInstance(component,
compRef.datasetType.storageClass.pytype)
compsRead[compName] = component
# Simple check of WCS
bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0, 0),
lsst.afw.geom.Extent2I(9, 9))
self.assertWcsAlmostEqualOverBBox(compsRead["wcs"], exposure.getWcs(),
bbox)
# Try to reassemble the exposure
retrievedExposure = storageClass.assembler().assemble(compsRead)
self.assertIsInstance(retrievedExposure, type(exposure))
def testCompositePutGet(self):
metrics = makeExampleMetrics()
datastore = PosixDatastore(config=self.configFile,
registry=self.registry)
# Create multiple storage classes for testing different formulations
# of composites
storageClasses = [
self.storageClassFactory.getStorageClass(sc)
for sc in ("StructuredComposite", "StructuredCompositeTestA",
"StructuredCompositeTestB")
]
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 428, "filter": "R"}
for sc in storageClasses:
print("Using storageClass: {}".format(sc.name))
ref = self.makeDatasetRef("metric", dataUnits, sc, dataId)
components = sc.assembler().disassemble(metrics)
self.assertTrue(components)
compsRead = {}
for compName, compInfo in components.items():
compRef = self.makeDatasetRef(
ref.datasetType.componentTypeName(compName), dataUnits,
components[compName].storageClass, dataId)
print("Writing component {} with {}".format(
compName, compRef.datasetType.storageClass.name))
datastore.put(compInfo.component, compRef)
uri = datastore.getUri(compRef)
self.assertEqual(uri[:5], "file:")
compsRead[compName] = datastore.get(compRef)
# combine all the components we read back into a new composite
metricsOut = sc.assembler().assemble(compsRead)
self.assertEqual(metrics, metricsOut)
def testBasicPutGet(self):
metrics = makeExampleMetrics()
datastore = PosixDatastore(config=self.configFile,
registry=self.registry)
# Create multiple storage classes for testing different formulations
storageClasses = [
self.storageClassFactory.getStorageClass(sc)
for sc in ("StructuredData", "StructuredDataJson",
"StructuredDataPickle")
]
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 52, "filter": "V"}
for sc in storageClasses:
ref = self.makeDatasetRef("metric", dataUnits, sc, dataId)
print("Using storageClass: {}".format(sc.name))
datastore.put(metrics, ref)
# Does it exist?
self.assertTrue(datastore.exists(ref))
# Get
metricsOut = datastore.get(ref, parameters=None)
self.assertEqual(metrics, metricsOut)
uri = datastore.getUri(ref)
self.assertEqual(uri[:5], "file:")
# Get a component -- we need to construct new refs for them
# with derived storage classes but with parent ID
comp = "output"
compRef = self.makeDatasetRef(
ref.datasetType.componentTypeName(comp),
dataUnits,
sc.components[comp],
dataId,
id=ref.id)
output = datastore.get(compRef)
self.assertEqual(output, metricsOut.output)
uri = datastore.getUri(compRef)
self.assertEqual(uri[:5], "file:")
storageClass = sc
# These should raise
ref = self.makeDatasetRef("metrics",
dataUnits,
storageClass,
dataId,
id=10000)
with self.assertRaises(FileNotFoundError):
# non-existing file
datastore.get(ref)
# Get a URI from it
uri = datastore.getUri(ref, predict=True)
self.assertEqual(uri[:5], "file:")
with self.assertRaises(FileNotFoundError):
datastore.getUri(ref)
def testConstructor(self):
datastore = PosixDatastore(config=self.configFile,
registry=self.registry)
self.assertIsNotNone(datastore)
def testRemove(self):
metrics = makeExampleMetrics()
datastore = PosixDatastore(config=self.configFile,
registry=self.registry)
# Put
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 638, "filter": "U"}
sc = self.storageClassFactory.getStorageClass("StructuredData")
ref = self.makeDatasetRef("metric", dataUnits, sc, dataId)
datastore.put(metrics, ref)
# Does it exist?
self.assertTrue(datastore.exists(ref))
# Get
metricsOut = datastore.get(ref)
self.assertEqual(metrics, metricsOut)
# Remove
datastore.remove(ref)
# Does it exist?
self.assertFalse(datastore.exists(ref))
# Do we now get a predicted URI?
uri = datastore.getUri(ref, predict=True)
self.assertTrue(uri.endswith("#predicted"))
# Get should now fail
with self.assertRaises(FileNotFoundError):
datastore.get(ref)
# Can only delete once
with self.assertRaises(FileNotFoundError):
datastore.remove(ref)
def testRemove(self):
catalog = self.makeExampleCatalog()
datastore = PosixDatastore(config=self.configFile,
registry=self.registry)
# Put
storageClass = self.storageClassFactory.getStorageClass(
"SourceCatalog")
dataUnits = frozenset(("visit", "filter"))
dataId = {"visit": 1234567, "filter": "blue"}
ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId)
datastore.put(catalog, ref)
# Does it exist?
self.assertTrue(datastore.exists(ref))
# Get
catalogOut = datastore.get(ref)
self.assertCatalogEqual(catalog, catalogOut)
# Remove
datastore.remove(ref)
# Does it exist?
self.assertFalse(datastore.exists(ref))
# Get should now fail
with self.assertRaises(FileNotFoundError):
datastore.get(ref)
# Can only delete once
with self.assertRaises(FileNotFoundError):
datastore.remove(ref)
