def testHashability(self):
"""Test `DatasetType.__hash__`.
This test is performed by checking that `DatasetType` entries can
be inserted into a `set` and that unique values of its
(`name`, `storageClass`, `dataUnits`) parameters result in separate
entries (and equal ones don't).
This does not check for uniformity of hashing or the actual values
of the hash function.
"""
types = []
unique = 0
storageC = StorageClass("test_c")
storageD = StorageClass("test_d")
for name in ["a", "b"]:
for storageClass in [storageC, storageD]:
for dataUnits in [("e", ), ("f", )]:
datasetType = DatasetType(name, dataUnits, storageClass)
datasetTypeCopy = DatasetType(name, dataUnits,
storageClass)
types.extend((datasetType, datasetTypeCopy))
unique += 1 # datasetType should always equal its copy
self.assertEqual(len(set(types)),
unique) # all other combinations are unique
def testCollections(self):
registry = Registry.fromConfig(self.configFile)
storageClass = StorageClass("testCollections")
datasetType = DatasetType(name="dummytype", dataUnits=("Camera", "Visit"), storageClass=storageClass)
registry.registerDatasetType(datasetType)
collection = "ingest"
run = registry.makeRun(collection=collection)
dataId1 = {"camera": "DummyCam", "visit": 0}
inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run)
dataId2 = {"camera": "DummyCam", "visit": 1}
inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run)
# We should be able to find both datasets in their Run.collection
outputRef = registry.find(run.collection, datasetType, dataId1)
self.assertEqual(outputRef, inputRef1)
outputRef = registry.find(run.collection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
# and with the associated collection
newCollection = "something"
registry.associate(newCollection, [inputRef1, inputRef2])
outputRef = registry.find(newCollection, datasetType, dataId1)
self.assertEqual(outputRef, inputRef1)
outputRef = registry.find(newCollection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
# but no more after disassociation
registry.disassociate(newCollection, [inputRef1, ], remove=False) # TODO test with removal when done
self.assertIsNone(registry.find(newCollection, datasetType, dataId1))
outputRef = registry.find(newCollection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
def testFind(self):
registry = Registry.fromConfig(self.configFile)
storageClass = StorageClass("testFind")
datasetType = DatasetType(name="dummytype", dataUnits=("Camera", "Visit"), storageClass=storageClass)
registry.registerDatasetType(datasetType)
collection = "test"
dataId = {"camera": "DummyCam", "visit": 0}
run = registry.makeRun(collection=collection)
inputRef = registry.addDataset(datasetType, dataId=dataId, run=run)
outputRef = registry.find(collection, datasetType, dataId)
self.assertEqual(outputRef, inputRef)
# Check that retrieval with invalid dataId raises
with self.assertRaises(ValueError):
dataId = {"camera": "DummyCam", "abstract_filter": "g"} # should be visit
registry.find(collection, datasetType, dataId)
# Check that different dataIds match to different datasets
dataId1 = {"camera": "DummyCam", "visit": 1}
inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run)
dataId2 = {"camera": "DummyCam", "visit": 2}
inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run)
dataId3 = {"camera": "MyCam", "visit": 2}
inputRef3 = registry.addDataset(datasetType, dataId=dataId3, run=run)
self.assertEqual(registry.find(collection, datasetType, dataId1), inputRef1)
self.assertEqual(registry.find(collection, datasetType, dataId2), inputRef2)
self.assertEqual(registry.find(collection, datasetType, dataId3), inputRef3)
self.assertNotEqual(registry.find(collection, datasetType, dataId1), inputRef2)
self.assertNotEqual(registry.find(collection, datasetType, dataId2), inputRef1)
self.assertNotEqual(registry.find(collection, datasetType, dataId3), inputRef1)
# Check that requesting a non-existing dataId returns None
nonExistingDataId = {"camera": "DummyCam", "visit": 42}
self.assertIsNone(registry.find(collection, datasetType, nonExistingDataId))
def testQuantum(self):
registry = Registry.fromConfig(self.configFile)
run = registry.makeRun(collection="test")
storageClass = StorageClass("testQuantum")
# Make two predicted inputs
datasetType1 = DatasetType(name="dst1", dataUnits=("Camera",), storageClass=storageClass)
registry.registerDatasetType(datasetType1)
ref1 = registry.addDataset(datasetType1, dataId={"camera": "DummyCam"}, run=run)
datasetType2 = DatasetType(name="dst2", dataUnits=("Camera",), storageClass=storageClass)
registry.registerDatasetType(datasetType2)
ref2 = registry.addDataset(datasetType2, dataId={"camera": "DummyCam"}, run=run)
# Create and add a Quantum
quantum = Quantum(run=run,
task="some.fully.qualified.SuperTask",
startTime=datetime(2018, 1, 1),
endTime=datetime(2018, 1, 2),
host="localhost")
quantum.addPredictedInput(ref1)
quantum.addPredictedInput(ref2)
# Quantum is not yet in Registry, so can't mark input as actual
with self.assertRaises(KeyError):
registry.markInputUsed(quantum, ref1)
registry.addQuantum(quantum)
# Now we can
registry.markInputUsed(quantum, ref1)
outQuantum = registry.getQuantum(quantum.id)
self.assertEqual(outQuantum, quantum)
def testDataset(self):
registry = Registry.fromConfig(self.configFile)
run = registry.makeRun(collection="test")
storageClass = StorageClass("testDataset")
datasetType = DatasetType(name="testtype", dataUnits=("Camera",), storageClass=storageClass)
registry.registerDatasetType(datasetType)
ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run)
outRef = registry.getDataset(ref.id)
self.assertEqual(ref, outRef)
def testAssembler(self):
registry = Registry.fromConfig(self.configFile)
storageClass = StorageClass("testAssembler")
datasetType = DatasetType(name="test", dataUnits=("Camera",), storageClass=storageClass)
registry.registerDatasetType(datasetType)
run = registry.makeRun(collection="test")
ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run)
self.assertIsNone(ref.assembler)
assembler = "some.fully.qualified.assembler" # TODO replace by actual dummy assember once implemented
registry.setAssembler(ref, assembler)
self.assertEqual(ref.assembler, assembler)
def testEquality(self):
storageA = StorageClass("test_a")
storageB = StorageClass("test_b")
self.assertEqual(DatasetType(
"a",
("UnitA", ),
storageA,
), DatasetType(
"a",
("UnitA", ),
storageA,
))
self.assertNotEqual(DatasetType(
"a",
("UnitA", ),
storageA,
), DatasetType(
"b",
("UnitA", ),
storageA,
))
self.assertNotEqual(DatasetType(
"a",
("UnitA", ),
storageA,
), DatasetType(
"a",
("UnitA", ),
storageB,
))
self.assertNotEqual(DatasetType(
"a",
("UnitA", ),
storageA,
), DatasetType(
"a",
("UnitB", ),
storageA,
))
def testConstructor(self):
"""Test construction preserves values.
Note that construction doesn't check for valid storageClass or
dataUnits parameters.
These can only be verified for a particular schema.
"""
datasetTypeName = "test"
storageClass = StorageClass("test_StructuredData")
dataUnits = frozenset(("camera", "visit"))
datasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
self.assertEqual(datasetType.name, datasetTypeName)
self.assertEqual(datasetType.storageClass, storageClass)
self.assertEqual(datasetType.dataUnits, dataUnits)
def testDatasetType(self):
registry = Registry.fromConfig(self.configFile)
# Check valid insert
datasetTypeName = "test"
storageClass = StorageClass("testDatasetType")
dataUnits = ("Camera", "Visit")
inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
registry.registerDatasetType(inDatasetType)
outDatasetType = registry.getDatasetType(datasetTypeName)
self.assertEqual(outDatasetType, inDatasetType)
# Re-inserting should fail
with self.assertRaises(KeyError):
registry.registerDatasetType(inDatasetType)
# Template can be None
datasetTypeName = "testNoneTemplate"
storageClass = StorageClass("testDatasetType2")
dataUnits = ("Camera", "Visit")
inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
registry.registerDatasetType(inDatasetType)
outDatasetType = registry.getDatasetType(datasetTypeName)
self.assertEqual(outDatasetType, inDatasetType)
def testConstructor(self):
"""Test construction preserves values.
"""
datasetTypeName = "test"
storageClass = StorageClass("testref_StructuredData")
dataUnits = frozenset(("camera", "visit"))
dataId = dict(camera="DummyCam", visit=42)
datasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
ref = DatasetRef(datasetType, dataId)
self.assertEqual(ref.datasetType, datasetType)
self.assertEqual(ref.dataId, dataId)
self.assertIsNone(ref.producer)
self.assertEqual(ref.predictedConsumers, dict())
self.assertEqual(ref.actualConsumers, dict())
self.assertEqual(ref.components, dict())
def testComponents(self):
registry = Registry.fromConfig(self.configFile)
storageClass = StorageClass("testComponents")
parentDatasetType = DatasetType(name="parent", dataUnits=("Camera",), storageClass=storageClass)
childDatasetType1 = DatasetType(name="child1", dataUnits=("Camera",), storageClass=storageClass)
childDatasetType2 = DatasetType(name="child2", dataUnits=("Camera",), storageClass=storageClass)
registry.registerDatasetType(parentDatasetType)
registry.registerDatasetType(childDatasetType1)
registry.registerDatasetType(childDatasetType2)
run = registry.makeRun(collection="test")
parent = registry.addDataset(parentDatasetType, dataId={"camera": "DummyCam"}, run=run)
children = {"child1": registry.addDataset(childDatasetType1, dataId={"camera": "DummyCam"}, run=run),
"child2": registry.addDataset(childDatasetType2, dataId={"camera": "DummyCam"}, run=run)}
for name, child in children.items():
registry.attachComponent(name, parent, child)
self.assertEqual(parent.components, children)
outParent = registry.getDataset(parent.id)
self.assertEqual(outParent.components, children)
def testAddInputsOutputs(self):
"""Test of addPredictedInput() method.
"""
quantum = Quantum(task="some.task.object", run=None)
# start with empty
self.assertEqual(quantum.predictedInputs, dict())
universe = DimensionUniverse.fromConfig()
instrument = "DummyCam"
datasetTypeName = "test_ds"
storageClass = StorageClass("testref_StructuredData")
datasetType = DatasetType(datasetTypeName,
universe.extract(("instrument", "visit")),
storageClass)
# add one ref
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42))
quantum.addPredictedInput(ref)
self.assertIn(datasetTypeName, quantum.predictedInputs)
self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 1)
# add second ref
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43))
quantum.addPredictedInput(ref)
self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 2)
# mark last ref as actually used
self.assertEqual(quantum.actualInputs, dict())
quantum._markInputUsed(ref)
self.assertIn(datasetTypeName, quantum.actualInputs)
self.assertEqual(len(quantum.actualInputs[datasetTypeName]), 1)
# add couple of outputs too
self.assertEqual(quantum.outputs, dict())
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42))
quantum.addOutput(ref)
self.assertIn(datasetTypeName, quantum.outputs)
self.assertEqual(len(quantum.outputs[datasetTypeName]), 1)
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43))
quantum.addOutput(ref)
self.assertEqual(len(quantum.outputs[datasetTypeName]), 2)
def testStorageInfo(self):
registry = Registry.fromConfig(self.configFile)
storageClass = StorageClass("testStorageInfo")
datasetType = DatasetType(name="test", dataUnits=("Camera",), storageClass=storageClass)
registry.registerDatasetType(datasetType)
run = registry.makeRun(collection="test")
ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run)
datastoreName = "dummystore"
checksum = "d6fb1c0c8f338044b2faaf328f91f707"
size = 512
storageInfo = StorageInfo(datastoreName, checksum, size)
# Test adding information about a new dataset
registry.addStorageInfo(ref, storageInfo)
outStorageInfo = registry.getStorageInfo(ref, datastoreName)
self.assertEqual(outStorageInfo, storageInfo)
# Test updating storage information for an existing dataset
updatedStorageInfo = StorageInfo(datastoreName, "20a38163c50f4aa3aa0f4047674f8ca7", size+1)
registry.updateStorageInfo(ref, datastoreName, updatedStorageInfo)
outStorageInfo = registry.getStorageInfo(ref, datastoreName)
self.assertNotEqual(outStorageInfo, storageInfo)
self.assertEqual(outStorageInfo, updatedStorageInfo)