def testEquality(self): storageA = makeNewStorageClass("test_a")() storageB = makeNewStorageClass("test_b")() self.assertEqual(DatasetType("a", ("UnitA", ), storageA,), DatasetType("a", ("UnitA", ), storageA,)) self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,), DatasetType("b", ("UnitA", ), storageA,)) self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,), DatasetType("a", ("UnitA", ), storageB,)) self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,), DatasetType("a", ("UnitB", ), storageA,))
def testComponents(self): registry = Registry.fromConfig(self.configFile) storageClass = makeNewStorageClass("testComponents")() parentDatasetType = DatasetType(name="parent", dataUnits=("camera", ), storageClass=storageClass) childDatasetType1 = DatasetType(name="child1", dataUnits=("camera", ), storageClass=storageClass) childDatasetType2 = DatasetType(name="child2", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(parentDatasetType) registry.registerDatasetType(childDatasetType1) registry.registerDatasetType(childDatasetType2) run = registry.makeRun(collection="test") parent = registry.addDataset(parentDatasetType, dataId={"camera": "DummyCam"}, run=run) children = { "child1": registry.addDataset(childDatasetType1, dataId={"camera": "DummyCam"}, run=run), "child2": registry.addDataset(childDatasetType2, dataId={"camera": "DummyCam"}, run=run) } for name, child in children.items(): registry.attachComponent(name, parent, child) self.assertEqual(parent.components, children) outParent = registry.getDataset(parent.id) self.assertEqual(outParent.components, children)
def testStorageInfo(self): registry = Registry.fromConfig(self.configFile) storageClass = makeNewStorageClass("testStorageInfo")() datasetType = DatasetType(name="test", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(datasetType) run = registry.makeRun(collection="test") ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run) datastoreName = "dummystore" checksum = "d6fb1c0c8f338044b2faaf328f91f707" size = 512 storageInfo = StorageInfo(datastoreName, checksum, size) # Test adding information about a new dataset registry.addStorageInfo(ref, storageInfo) outStorageInfo = registry.getStorageInfo(ref, datastoreName) self.assertEqual(outStorageInfo, storageInfo) # Test updating storage information for an existing dataset updatedStorageInfo = StorageInfo(datastoreName, "20a38163c50f4aa3aa0f4047674f8ca7", size + 1) registry.updateStorageInfo(ref, datastoreName, updatedStorageInfo) outStorageInfo = registry.getStorageInfo(ref, datastoreName) self.assertNotEqual(outStorageInfo, storageInfo) self.assertEqual(outStorageInfo, updatedStorageInfo)
def testCollections(self): registry = Registry.fromConfig(self.configFile) storageClass = makeNewStorageClass("testCollections")() datasetType = DatasetType(name="dummytype", dataUnits=("camera", "visit"), storageClass=storageClass) registry.registerDatasetType(datasetType) collection = "ingest" run = registry.makeRun(collection=collection) dataId1 = {"camera": "DummyCam", "visit": 0} inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run) dataId2 = {"camera": "DummyCam", "visit": 1} inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run) # We should be able to find both datasets in their Run.collection outputRef = registry.find(run.collection, datasetType, dataId1) self.assertEqual(outputRef, inputRef1) outputRef = registry.find(run.collection, datasetType, dataId2) self.assertEqual(outputRef, inputRef2) # and with the associated collection newCollection = "something" registry.associate(newCollection, [inputRef1, inputRef2]) outputRef = registry.find(newCollection, datasetType, dataId1) self.assertEqual(outputRef, inputRef1) outputRef = registry.find(newCollection, datasetType, dataId2) self.assertEqual(outputRef, inputRef2) # but no more after disassociation registry.disassociate(newCollection, [ inputRef1, ], remove=False) # TODO test with removal when done self.assertIsNone(registry.find(newCollection, datasetType, dataId1)) outputRef = registry.find(newCollection, datasetType, dataId2) self.assertEqual(outputRef, inputRef2)
def testQuantum(self): registry = Registry.fromConfig(self.configFile) run = registry.makeRun(collection="test") storageClass = makeNewStorageClass("testQuantum")() # Make two predicted inputs datasetType1 = DatasetType(name="dst1", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(datasetType1) ref1 = registry.addDataset(datasetType1, dataId={"camera": "DummyCam"}, run=run) datasetType2 = DatasetType(name="dst2", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(datasetType2) ref2 = registry.addDataset(datasetType2, dataId={"camera": "DummyCam"}, run=run) # Create and add a Quantum quantum = Quantum(run=run, task="some.fully.qualified.SuperTask", startTime=datetime(2018, 1, 1), endTime=datetime(2018, 1, 2), host="localhost") quantum.addPredictedInput(ref1) quantum.addPredictedInput(ref2) # Quantum is not yet in Registry, so can't mark input as actual with self.assertRaises(KeyError): registry.markInputUsed(quantum, ref1) registry.addQuantum(quantum) # Now we can registry.markInputUsed(quantum, ref1) outQuantum = registry.getQuantum(quantum.id) self.assertEqual(outQuantum, quantum)
def testDataset(self): registry = Registry.fromConfig(self.configFile) run = registry.makeRun(collection="test") storageClass = makeNewStorageClass("testDataset")() datasetType = DatasetType(name="testtype", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(datasetType) ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run) outRef = registry.getDataset(ref.id) self.assertEqual(ref, outRef)
def testRegistry(self): """Check that storage classes can be created on the fly and stored in a registry.""" className = "TestImage" factory = storageClass.StorageClassFactory() newclass = storageClass.makeNewStorageClass(className, pytype=PythonType) factory.registerStorageClass(newclass) sc = factory.getStorageClass(className) self.assertIsInstance(sc, storageClass.StorageClass) self.assertEqual(sc.name, className) self.assertFalse(sc.components) self.assertEqual(sc.pytype, PythonType)
def testHashability(self): """Test `DatasetType.__hash__`. This test is performed by checking that `DatasetType` entries can be inserted into a `set` and that unique values of its (`name`, `storageClass`, `dataUnits`) parameters result in separate entries (and equal ones don't). This does not check for uniformity of hashing or the actual values of the hash function. """ types = [] unique = 0 storageC = makeNewStorageClass("test_c")() storageD = makeNewStorageClass("test_d")() for name in ["a", "b"]: for storageClass in [storageC, storageD]: for dataUnits in [("e", ), ("f", )]: datasetType = DatasetType(name, dataUnits, storageClass) datasetTypeCopy = DatasetType(name, dataUnits, storageClass) types.extend((datasetType, datasetTypeCopy)) unique += 1 # datasetType should always equal its copy self.assertEqual(len(set(types)), unique) # all other combinations are unique
def testConstructor(self): """Test construction preserves values. Note that construction doesn't check for valid storageClass or dataUnits parameters. These can only be verified for a particular schema. """ datasetTypeName = "test" storageClass = makeNewStorageClass("test_StructuredData")() dataUnits = frozenset(("camera", "visit")) datasetType = DatasetType(datasetTypeName, dataUnits, storageClass) self.assertEqual(datasetType.name, datasetTypeName) self.assertEqual(datasetType.storageClass, storageClass) self.assertEqual(datasetType.dataUnits, dataUnits)
def testDatasetType(self): registry = Registry.fromConfig(self.configFile) # Check valid insert datasetTypeName = "test" storageClass = makeNewStorageClass("testDatasetType")() dataUnits = ("camera", "visit") inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass) registry.registerDatasetType(inDatasetType) outDatasetType = registry.getDatasetType(datasetTypeName) self.assertEqual(outDatasetType, inDatasetType) # Re-inserting should fail with self.assertRaises(KeyError): registry.registerDatasetType(inDatasetType) # Template can be None datasetTypeName = "testNoneTemplate" storageClass = makeNewStorageClass("testDatasetType2")() dataUnits = ("camera", "visit") inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass) registry.registerDatasetType(inDatasetType) outDatasetType = registry.getDatasetType(datasetTypeName) self.assertEqual(outDatasetType, inDatasetType)
def testCreation(self): """Test that we can dynamically create storage class subclasses. This is critical for testing the factory functions.""" className = "TestImage" newclass = storageClass.makeNewStorageClass(className, pytype=dict) sc = newclass() self.assertIsInstance(sc, storageClass.StorageClass) self.assertEqual(sc.name, className) self.assertFalse(sc.components) # Test the caching by using private class attribute self.assertIsNone(newclass._pytype) self.assertEqual(sc.pytype, dict) self.assertEqual(newclass._pytype, dict) # Check we can create a storageClass using the name of an importable # type. newclass = storageClass.makeNewStorageClass( "TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") self.assertIsInstance(newclass().pytype(), storageClass.StorageClassFactory)
def testConstructor(self): """Test construction preserves values. """ datasetTypeName = "test" storageClass = makeNewStorageClass("testref_StructuredData")() dataUnits = frozenset(("camera", "visit")) dataId = dict(camera="DummyCam", visit=42) datasetType = DatasetType(datasetTypeName, dataUnits, storageClass) ref = DatasetRef(datasetType, dataId) self.assertEqual(ref.datasetType, datasetType) self.assertEqual(ref.dataId, dataId) self.assertIsNone(ref.producer) self.assertEqual(ref.predictedConsumers, dict()) self.assertEqual(ref.actualConsumers, dict()) self.assertEqual(ref.components, dict())
def testAssembler(self): registry = Registry.fromConfig(self.configFile) storageClass = makeNewStorageClass("testAssembler")() datasetType = DatasetType(name="test", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(datasetType) run = registry.makeRun(collection="test") ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run) self.assertIsNone(ref.assembler) assembler = "some.fully.qualified.assembler" # TODO replace by actual dummy assember once implemented registry.setAssembler(ref, assembler) self.assertEqual(ref.assembler, assembler)
def testFind(self): registry = Registry.fromConfig(self.configFile) storageClass = makeNewStorageClass("testFind")() datasetType = DatasetType(name="dummytype", dataUnits=("camera", "visit"), storageClass=storageClass) registry.registerDatasetType(datasetType) collection = "test" dataId = {"camera": "DummyCam", "visit": 0} run = registry.makeRun(collection=collection) inputRef = registry.addDataset(datasetType, dataId=dataId, run=run) outputRef = registry.find(collection, datasetType, dataId) self.assertEqual(outputRef, inputRef) # Check that retrieval with invalid dataId raises with self.assertRaises(ValueError): dataId = { "camera": "DummyCam", "abstract_filter": "g" } # should be visit registry.find(collection, datasetType, dataId) # Check that different dataIds match to different datasets dataId1 = {"camera": "DummyCam", "visit": 1} inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run) dataId2 = {"camera": "DummyCam", "visit": 2} inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run) dataId3 = {"camera": "MyCam", "visit": 2} inputRef3 = registry.addDataset(datasetType, dataId=dataId3, run=run) self.assertEqual(registry.find(collection, datasetType, dataId1), inputRef1) self.assertEqual(registry.find(collection, datasetType, dataId2), inputRef2) self.assertEqual(registry.find(collection, datasetType, dataId3), inputRef3) self.assertNotEqual(registry.find(collection, datasetType, dataId1), inputRef2) self.assertNotEqual(registry.find(collection, datasetType, dataId2), inputRef1) self.assertNotEqual(registry.find(collection, datasetType, dataId3), inputRef1) # Check that requesting a non-existing dataId returns None nonExistingDataId = {"camera": "DummyCam", "visit": 42} self.assertIsNone( registry.find(collection, datasetType, nonExistingDataId))
def testRegistryWithStorageClass(self): """Test that the registry can be given a StorageClass object. """ formatterTypeName = "lsst.daf.butler.formatters.yamlFormatter.YamlFormatter" storageClassName = "TestClass" sc = storageClass.makeNewStorageClass(storageClassName, dict, None) # Store using an instance self.factory.registerFormatter(sc(), formatterTypeName) # Retrieve using the class f = self.factory.getFormatter(sc) self.assertIsInstance(f, formatter.Formatter) # This might defer the import, pytest may have already loaded it from lsst.daf.butler.formatters.yamlFormatter import YamlFormatter self.assertEqual(type(f), YamlFormatter) with self.assertRaises(KeyError): # Attempt to overwrite using a different value self.factory.registerFormatter( storageClassName, "lsst.daf.butler.formatters.jsonFormatter.JsonFormatter")