def testTransfer(self): metrics = makeExampleMetrics() dataUnits = frozenset(("visit", "filter")) dataId = {"visit": 2048, "filter": "Uprime"} sc = self.storageClassFactory.getStorageClass("StructuredData") ref = self.makeDatasetRef("metric", dataUnits, sc, dataId) inputConfig = DatastoreConfig(self.configFile) inputConfig['datastore.root'] = os.path.join(self.testDir, "./test_input_datastore") inputPosixDatastore = PosixDatastore(config=inputConfig, registry=self.registry) outputConfig = inputConfig.copy() outputConfig['datastore.root'] = os.path.join( self.testDir, "./test_output_datastore") outputPosixDatastore = PosixDatastore(config=outputConfig, registry=DummyRegistry()) inputPosixDatastore.put(metrics, ref) outputPosixDatastore.transfer(inputPosixDatastore, ref) metricsOut = outputPosixDatastore.get(ref) self.assertEqual(metrics, metricsOut)
def testTransfer(self): catalog = self.makeExampleCatalog() dataUnits = frozenset(("visit", "filter")) dataId = {"visit": 12345, "filter": "red"} storageClass = self.storageClassFactory.getStorageClass( "SourceCatalog") ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId) inputConfig = DatastoreConfig(self.configFile) inputConfig['datastore.root'] = os.path.join(self.testDir, "./test_input_datastore") inputPosixDatastore = PosixDatastore(config=inputConfig, registry=self.registry) outputConfig = inputConfig.copy() outputConfig['datastore.root'] = os.path.join( self.testDir, "./test_output_datastore") outputPosixDatastore = PosixDatastore(config=outputConfig, registry=DummyRegistry()) inputPosixDatastore.put(catalog, ref) outputPosixDatastore.transfer(inputPosixDatastore, ref) catalogOut = outputPosixDatastore.get(ref) self.assertCatalogEqual(catalog, catalogOut)
def testRemove(self): metrics = makeExampleMetrics() datastore = PosixDatastore(config=self.configFile, registry=self.registry) # Put dataUnits = frozenset(("visit", "filter")) dataId = {"visit": 638, "filter": "U"} sc = self.storageClassFactory.getStorageClass("StructuredData") ref = self.makeDatasetRef("metric", dataUnits, sc, dataId) datastore.put(metrics, ref) # Does it exist? self.assertTrue(datastore.exists(ref)) # Get metricsOut = datastore.get(ref) self.assertEqual(metrics, metricsOut) # Remove datastore.remove(ref) # Does it exist? self.assertFalse(datastore.exists(ref)) # Do we now get a predicted URI? uri = datastore.getUri(ref, predict=True) self.assertTrue(uri.endswith("#predicted")) # Get should now fail with self.assertRaises(FileNotFoundError): datastore.get(ref) # Can only delete once with self.assertRaises(FileNotFoundError): datastore.remove(ref)
def testRemove(self): catalog = self.makeExampleCatalog() datastore = PosixDatastore(config=self.configFile, registry=self.registry) # Put storageClass = self.storageClassFactory.getStorageClass( "SourceCatalog") dataUnits = frozenset(("visit", "filter")) dataId = {"visit": 1234567, "filter": "blue"} ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId) datastore.put(catalog, ref) # Does it exist? self.assertTrue(datastore.exists(ref)) # Get catalogOut = datastore.get(ref) self.assertCatalogEqual(catalog, catalogOut) # Remove datastore.remove(ref) # Does it exist? self.assertFalse(datastore.exists(ref)) # Get should now fail with self.assertRaises(FileNotFoundError): datastore.get(ref) # Can only delete once with self.assertRaises(FileNotFoundError): datastore.remove(ref)
def testBasicPutGet(self): catalog = self.makeExampleCatalog() datastore = PosixDatastore(config=self.configFile, registry=self.registry) # Put dataUnits = frozenset(("visit", "filter")) dataId = {"visit": 123456, "filter": "blue"} storageClass = self.storageClassFactory.getStorageClass( "SourceCatalog") ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId) datastore.put(catalog, ref) # Does it exist? self.assertTrue(datastore.exists(ref)) uri = datastore.getUri(ref) self.assertTrue(uri.endswith(".fits")) self.assertTrue(uri.startswith("file:")) # Get catalogOut = datastore.get(ref, parameters=None) self.assertCatalogEqual(catalog, catalogOut) # These should raise ref = self.makeDatasetRef("calexp2", dataUnits, storageClass, dataId) with self.assertRaises(FileNotFoundError): # non-existing file datastore.get(ref, parameters=None)
def testExposurePutGet(self): example = os.path.join(self.testDir, "data", "basic", "small.fits") exposure = lsst.afw.image.ExposureF(example) datastore = PosixDatastore(config=self.configFile, registry=self.registry) # Put dataUnits = frozenset(("visit", "filter")) dataId = {"visit": 231, "filter": "Fc"} storageClass = datastore.storageClassFactory.getStorageClass("ExposureF") ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId) datastore.put(exposure, ref) # Does it exist? self.assertTrue(datastore.exists(ref)) # Get exposureOut = datastore.get(ref) self.assertEqual(type(exposure), type(exposureOut)) # Get some components for compName in ("wcs", "image", "mask", "coaddInputs", "psf"): compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dataUnits, storageClass.components[compName], dataId, id=ref.id) component = datastore.get(compRef) self.assertIsInstance(component, compRef.datasetType.storageClass.pytype) # Get the WCS component to check it wcsRef = self.makeDatasetRef(ref.datasetType.componentTypeName("wcs"), dataUnits, storageClass.components["wcs"], dataId, id=ref.id) wcs = datastore.get(wcsRef) # Simple check of WCS bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0, 0), lsst.afw.geom.Extent2I(9, 9)) self.assertWcsAlmostEqualOverBBox(wcs, exposure.getWcs(), bbox)
def testExposureCompositePutGet(self): example = os.path.join(self.testDir, "data", "basic", "small.fits") exposure = lsst.afw.image.ExposureF(example) datastore = PosixDatastore(config=self.configFile, registry=self.registry) # Put dataUnits = frozenset(("visit", "filter")) dataId = {"visit": 23, "filter": "F"} storageClass = datastore.storageClassFactory.getStorageClass( "ExposureCompositeF") ref = self.makeDatasetRef("calexp", dataUnits, storageClass, dataId) # Get the predicted URI self.assertFalse(datastore.exists(ref)) uri = datastore.getUri(ref, predict=True) self.assertTrue(uri.endswith("#predicted")) components = storageClass.assembler().disassemble(exposure) self.assertTrue(components) # Get a component compsRead = {} for compName in ("wcs", "image", "mask", "coaddInputs", "psf"): compRef = self.makeDatasetRef( ref.datasetType.componentTypeName(compName), dataUnits, components[compName].storageClass, dataId) datastore.put(components[compName].component, compRef) # Does it exist? self.assertTrue(datastore.exists(compRef)) component = datastore.get(compRef) self.assertIsInstance(component, compRef.datasetType.storageClass.pytype) compsRead[compName] = component # Simple check of WCS bbox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(0, 0), lsst.afw.geom.Extent2I(9, 9)) self.assertWcsAlmostEqualOverBBox(compsRead["wcs"], exposure.getWcs(), bbox) # Try to reassemble the exposure retrievedExposure = storageClass.assembler().assemble(compsRead) self.assertIsInstance(retrievedExposure, type(exposure))
def testCompositePutGet(self): metrics = makeExampleMetrics() datastore = PosixDatastore(config=self.configFile, registry=self.registry) # Create multiple storage classes for testing different formulations # of composites storageClasses = [ self.storageClassFactory.getStorageClass(sc) for sc in ("StructuredComposite", "StructuredCompositeTestA", "StructuredCompositeTestB") ] dataUnits = frozenset(("visit", "filter")) dataId = {"visit": 428, "filter": "R"} for sc in storageClasses: print("Using storageClass: {}".format(sc.name)) ref = self.makeDatasetRef("metric", dataUnits, sc, dataId) components = sc.assembler().disassemble(metrics) self.assertTrue(components) compsRead = {} for compName, compInfo in components.items(): compRef = self.makeDatasetRef( ref.datasetType.componentTypeName(compName), dataUnits, components[compName].storageClass, dataId) print("Writing component {} with {}".format( compName, compRef.datasetType.storageClass.name)) datastore.put(compInfo.component, compRef) uri = datastore.getUri(compRef) self.assertEqual(uri[:5], "file:") compsRead[compName] = datastore.get(compRef) # combine all the components we read back into a new composite metricsOut = sc.assembler().assemble(compsRead) self.assertEqual(metrics, metricsOut)
def testBasicPutGet(self): metrics = makeExampleMetrics() datastore = PosixDatastore(config=self.configFile, registry=self.registry) # Create multiple storage classes for testing different formulations storageClasses = [ self.storageClassFactory.getStorageClass(sc) for sc in ("StructuredData", "StructuredDataJson", "StructuredDataPickle") ] dataUnits = frozenset(("visit", "filter")) dataId = {"visit": 52, "filter": "V"} for sc in storageClasses: ref = self.makeDatasetRef("metric", dataUnits, sc, dataId) print("Using storageClass: {}".format(sc.name)) datastore.put(metrics, ref) # Does it exist? self.assertTrue(datastore.exists(ref)) # Get metricsOut = datastore.get(ref, parameters=None) self.assertEqual(metrics, metricsOut) uri = datastore.getUri(ref) self.assertEqual(uri[:5], "file:") # Get a component -- we need to construct new refs for them # with derived storage classes but with parent ID comp = "output" compRef = self.makeDatasetRef( ref.datasetType.componentTypeName(comp), dataUnits, sc.components[comp], dataId, id=ref.id) output = datastore.get(compRef) self.assertEqual(output, metricsOut.output) uri = datastore.getUri(compRef) self.assertEqual(uri[:5], "file:") storageClass = sc # These should raise ref = self.makeDatasetRef("metrics", dataUnits, storageClass, dataId, id=10000) with self.assertRaises(FileNotFoundError): # non-existing file datastore.get(ref) # Get a URI from it uri = datastore.getUri(ref, predict=True) self.assertEqual(uri[:5], "file:") with self.assertRaises(FileNotFoundError): datastore.getUri(ref)
def testConstructor(self): datastore = PosixDatastore(config=self.configFile, registry=self.registry) self.assertIsNotNone(datastore)