def testHashability(self): """Test `DatasetType.__hash__`. This test is performed by checking that `DatasetType` entries can be inserted into a `set` and that unique values of its (`name`, `storageClass`, `dataUnits`) parameters result in separate entries (and equal ones don't). This does not check for uniformity of hashing or the actual values of the hash function. """ types = [] unique = 0 storageC = StorageClass("test_c") storageD = StorageClass("test_d") for name in ["a", "b"]: for storageClass in [storageC, storageD]: for dataUnits in [("e", ), ("f", )]: datasetType = DatasetType(name, dataUnits, storageClass) datasetTypeCopy = DatasetType(name, dataUnits, storageClass) types.extend((datasetType, datasetTypeCopy)) unique += 1 # datasetType should always equal its copy self.assertEqual(len(set(types)), unique) # all other combinations are unique
def testComponents(self): registry = Registry.fromConfig(self.configFile) storageClass = makeNewStorageClass("testComponents")() parentDatasetType = DatasetType(name="parent", dataUnits=("camera", ), storageClass=storageClass) childDatasetType1 = DatasetType(name="child1", dataUnits=("camera", ), storageClass=storageClass) childDatasetType2 = DatasetType(name="child2", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(parentDatasetType) registry.registerDatasetType(childDatasetType1) registry.registerDatasetType(childDatasetType2) run = registry.makeRun(collection="test") parent = registry.addDataset(parentDatasetType, dataId={"camera": "DummyCam"}, run=run) children = { "child1": registry.addDataset(childDatasetType1, dataId={"camera": "DummyCam"}, run=run), "child2": registry.addDataset(childDatasetType2, dataId={"camera": "DummyCam"}, run=run) } for name, child in children.items(): registry.attachComponent(name, parent, child) self.assertEqual(parent.components, children) outParent = registry.getDataset(parent.id) self.assertEqual(outParent.components, children)
def testQuantum(self): registry = Registry.fromConfig(self.configFile) run = registry.makeRun(collection="test") storageClass = makeNewStorageClass("testQuantum")() # Make two predicted inputs datasetType1 = DatasetType(name="dst1", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(datasetType1) ref1 = registry.addDataset(datasetType1, dataId={"camera": "DummyCam"}, run=run) datasetType2 = DatasetType(name="dst2", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(datasetType2) ref2 = registry.addDataset(datasetType2, dataId={"camera": "DummyCam"}, run=run) # Create and add a Quantum quantum = Quantum(run=run, task="some.fully.qualified.SuperTask", startTime=datetime(2018, 1, 1), endTime=datetime(2018, 1, 2), host="localhost") quantum.addPredictedInput(ref1) quantum.addPredictedInput(ref2) # Quantum is not yet in Registry, so can't mark input as actual with self.assertRaises(KeyError): registry.markInputUsed(quantum, ref1) registry.addQuantum(quantum) # Now we can registry.markInputUsed(quantum, ref1) outQuantum = registry.getQuantum(quantum.id) self.assertEqual(outQuantum, quantum)
def testCollections(self): registry = Registry.fromConfig(self.configFile) storageClass = makeNewStorageClass("testCollections")() datasetType = DatasetType(name="dummytype", dataUnits=("camera", "visit"), storageClass=storageClass) registry.registerDatasetType(datasetType) collection = "ingest" run = registry.makeRun(collection=collection) dataId1 = {"camera": "DummyCam", "visit": 0} inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run) dataId2 = {"camera": "DummyCam", "visit": 1} inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run) # We should be able to find both datasets in their Run.collection outputRef = registry.find(run.collection, datasetType, dataId1) self.assertEqual(outputRef, inputRef1) outputRef = registry.find(run.collection, datasetType, dataId2) self.assertEqual(outputRef, inputRef2) # and with the associated collection newCollection = "something" registry.associate(newCollection, [inputRef1, inputRef2]) outputRef = registry.find(newCollection, datasetType, dataId1) self.assertEqual(outputRef, inputRef1) outputRef = registry.find(newCollection, datasetType, dataId2) self.assertEqual(outputRef, inputRef2) # but no more after disassociation registry.disassociate(newCollection, [ inputRef1, ], remove=False) # TODO test with removal when done self.assertIsNone(registry.find(newCollection, datasetType, dataId1)) outputRef = registry.find(newCollection, datasetType, dataId2) self.assertEqual(outputRef, inputRef2)
def testStorageInfo(self): registry = Registry.fromConfig(self.configFile) storageClass = makeNewStorageClass("testStorageInfo")() datasetType = DatasetType(name="test", dataUnits=("camera", ), storageClass=storageClass) registry.registerDatasetType(datasetType) run = registry.makeRun(collection="test") ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run) datastoreName = "dummystore" checksum = "d6fb1c0c8f338044b2faaf328f91f707" size = 512 storageInfo = StorageInfo(datastoreName, checksum, size) # Test adding information about a new dataset registry.addStorageInfo(ref, storageInfo) outStorageInfo = registry.getStorageInfo(ref, datastoreName) self.assertEqual(outStorageInfo, storageInfo) # Test updating storage information for an existing dataset updatedStorageInfo = StorageInfo(datastoreName, "20a38163c50f4aa3aa0f4047674f8ca7", size + 1) registry.updateStorageInfo(ref, datastoreName, updatedStorageInfo) outStorageInfo = registry.getStorageInfo(ref, datastoreName) self.assertNotEqual(outStorageInfo, storageInfo) self.assertEqual(outStorageInfo, updatedStorageInfo)
def testFind(self): registry = Registry.fromConfig(self.configFile) storageClass = StorageClass("testFind") datasetType = DatasetType(name="dummytype", dataUnits=("Camera", "Visit"), storageClass=storageClass) registry.registerDatasetType(datasetType) collection = "test" dataId = {"camera": "DummyCam", "visit": 0} run = registry.makeRun(collection=collection) inputRef = registry.addDataset(datasetType, dataId=dataId, run=run) outputRef = registry.find(collection, datasetType, dataId) self.assertEqual(outputRef, inputRef) # Check that retrieval with invalid dataId raises with self.assertRaises(ValueError): dataId = {"camera": "DummyCam", "abstract_filter": "g"} # should be visit registry.find(collection, datasetType, dataId) # Check that different dataIds match to different datasets dataId1 = {"camera": "DummyCam", "visit": 1} inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run) dataId2 = {"camera": "DummyCam", "visit": 2} inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run) dataId3 = {"camera": "MyCam", "visit": 2} inputRef3 = registry.addDataset(datasetType, dataId=dataId3, run=run) self.assertEqual(registry.find(collection, datasetType, dataId1), inputRef1) self.assertEqual(registry.find(collection, datasetType, dataId2), inputRef2) self.assertEqual(registry.find(collection, datasetType, dataId3), inputRef3) self.assertNotEqual(registry.find(collection, datasetType, dataId1), inputRef2) self.assertNotEqual(registry.find(collection, datasetType, dataId2), inputRef1) self.assertNotEqual(registry.find(collection, datasetType, dataId3), inputRef1) # Check that requesting a non-existing dataId returns None nonExistingDataId = {"camera": "DummyCam", "visit": 42} self.assertIsNone(registry.find(collection, datasetType, nonExistingDataId))
def testDataset(self): registry = Registry.fromConfig(self.configFile) run = registry.makeRun(collection="test") storageClass = StorageClass("testDataset") datasetType = DatasetType(name="testtype", dataUnits=("Camera",), storageClass=storageClass) registry.registerDatasetType(datasetType) ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run) outRef = registry.getDataset(ref.id) self.assertEqual(ref, outRef)
def testEquality(self): storageA = makeNewStorageClass("test_a")() storageB = makeNewStorageClass("test_b")() self.assertEqual(DatasetType("a", ("UnitA", ), storageA,), DatasetType("a", ("UnitA", ), storageA,)) self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,), DatasetType("b", ("UnitA", ), storageA,)) self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,), DatasetType("a", ("UnitA", ), storageB,)) self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,), DatasetType("a", ("UnitB", ), storageA,))
def testAssembler(self): registry = Registry.fromConfig(self.configFile) storageClass = StorageClass("testAssembler") datasetType = DatasetType(name="test", dataUnits=("Camera",), storageClass=storageClass) registry.registerDatasetType(datasetType) run = registry.makeRun(collection="test") ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run) self.assertIsNone(ref.assembler) assembler = "some.fully.qualified.assembler" # TODO replace by actual dummy assember once implemented registry.setAssembler(ref, assembler) self.assertEqual(ref.assembler, assembler)
def testConstructor(self): """Test construction preserves values. Note that construction doesn't check for valid storageClass or dataUnits parameters. These can only be verified for a particular schema. """ datasetTypeName = "test" storageClass = makeNewStorageClass("test_StructuredData")() dataUnits = frozenset(("camera", "visit")) datasetType = DatasetType(datasetTypeName, dataUnits, storageClass) self.assertEqual(datasetType.name, datasetTypeName) self.assertEqual(datasetType.storageClass, storageClass) self.assertEqual(datasetType.dataUnits, dataUnits)
def testDatasetType(self): registry = Registry.fromConfig(self.configFile) # Check valid insert datasetTypeName = "test" storageClass = makeNewStorageClass("testDatasetType")() dataUnits = ("camera", "visit") inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass) registry.registerDatasetType(inDatasetType) outDatasetType = registry.getDatasetType(datasetTypeName) self.assertEqual(outDatasetType, inDatasetType) # Re-inserting should fail with self.assertRaises(KeyError): registry.registerDatasetType(inDatasetType) # Template can be None datasetTypeName = "testNoneTemplate" storageClass = makeNewStorageClass("testDatasetType2")() dataUnits = ("camera", "visit") inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass) registry.registerDatasetType(inDatasetType) outDatasetType = registry.getDatasetType(datasetTypeName) self.assertEqual(outDatasetType, inDatasetType)
def testConstructor(self): """Test construction preserves values. """ datasetTypeName = "test" storageClass = makeNewStorageClass("testref_StructuredData")() dataUnits = frozenset(("camera", "visit")) dataId = dict(camera="DummyCam", visit=42) datasetType = DatasetType(datasetTypeName, dataUnits, storageClass) ref = DatasetRef(datasetType, dataId) self.assertEqual(ref.datasetType, datasetType) self.assertEqual(ref.dataId, dataId) self.assertIsNone(ref.producer) self.assertEqual(ref.predictedConsumers, dict()) self.assertEqual(ref.actualConsumers, dict()) self.assertEqual(ref.components, dict())
def makeDatasetType(cls, dsConfig): """Create new instance of the `DatasetType` from task config. Parameters ---------- dsConfig : `pexConfig.Config` Instance of the `supertask.InputDatasetConfig` or `supertask.OutputDatasetConfig` Returns ------- `butler.core.datasets.DatasetType` instance. """ # map storage class name to storage class storageClass = cls.storageClassFactory.getStorageClass(dsConfig.storageClass) return DatasetType(name=dsConfig.name, dataUnits=dsConfig.units, storageClass=storageClass)
def testAddInputsOutputs(self): """Test of addPredictedInput() method. """ quantum = Quantum(task="some.task.object", run=None) # start with empty self.assertEqual(quantum.predictedInputs, dict()) universe = DimensionUniverse.fromConfig() instrument = "DummyCam" datasetTypeName = "test_ds" storageClass = StorageClass("testref_StructuredData") datasetType = DatasetType(datasetTypeName, universe.extract(("instrument", "visit")), storageClass) # add one ref ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42)) quantum.addPredictedInput(ref) self.assertIn(datasetTypeName, quantum.predictedInputs) self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 1) # add second ref ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43)) quantum.addPredictedInput(ref) self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 2) # mark last ref as actually used self.assertEqual(quantum.actualInputs, dict()) quantum._markInputUsed(ref) self.assertIn(datasetTypeName, quantum.actualInputs) self.assertEqual(len(quantum.actualInputs[datasetTypeName]), 1) # add couple of outputs too self.assertEqual(quantum.outputs, dict()) ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42)) quantum.addOutput(ref) self.assertIn(datasetTypeName, quantum.outputs) self.assertEqual(len(quantum.outputs[datasetTypeName]), 1) ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43)) quantum.addOutput(ref) self.assertEqual(len(quantum.outputs[datasetTypeName]), 2)