def testHashability(self):
"""Test `DatasetType.__hash__`.
This test is performed by checking that `DatasetType` entries can
be inserted into a `set` and that unique values of its
(`name`, `storageClass`, `dataUnits`) parameters result in separate
entries (and equal ones don't).
This does not check for uniformity of hashing or the actual values
of the hash function.
"""
types = []
unique = 0
storageC = StorageClass("test_c")
storageD = StorageClass("test_d")
for name in ["a", "b"]:
for storageClass in [storageC, storageD]:
for dataUnits in [("e", ), ("f", )]:
datasetType = DatasetType(name, dataUnits, storageClass)
datasetTypeCopy = DatasetType(name, dataUnits,
storageClass)
types.extend((datasetType, datasetTypeCopy))
unique += 1 # datasetType should always equal its copy
self.assertEqual(len(set(types)),
unique) # all other combinations are unique
def testComponents(self):
registry = Registry.fromConfig(self.configFile)
storageClass = makeNewStorageClass("testComponents")()
parentDatasetType = DatasetType(name="parent",
dataUnits=("camera", ),
storageClass=storageClass)
childDatasetType1 = DatasetType(name="child1",
dataUnits=("camera", ),
storageClass=storageClass)
childDatasetType2 = DatasetType(name="child2",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(parentDatasetType)
registry.registerDatasetType(childDatasetType1)
registry.registerDatasetType(childDatasetType2)
run = registry.makeRun(collection="test")
parent = registry.addDataset(parentDatasetType,
dataId={"camera": "DummyCam"},
run=run)
children = {
"child1":
registry.addDataset(childDatasetType1,
dataId={"camera": "DummyCam"},
run=run),
"child2":
registry.addDataset(childDatasetType2,
dataId={"camera": "DummyCam"},
run=run)
}
for name, child in children.items():
registry.attachComponent(name, parent, child)
self.assertEqual(parent.components, children)
outParent = registry.getDataset(parent.id)
self.assertEqual(outParent.components, children)
def testQuantum(self):
registry = Registry.fromConfig(self.configFile)
run = registry.makeRun(collection="test")
storageClass = makeNewStorageClass("testQuantum")()
# Make two predicted inputs
datasetType1 = DatasetType(name="dst1",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(datasetType1)
ref1 = registry.addDataset(datasetType1,
dataId={"camera": "DummyCam"},
run=run)
datasetType2 = DatasetType(name="dst2",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(datasetType2)
ref2 = registry.addDataset(datasetType2,
dataId={"camera": "DummyCam"},
run=run)
# Create and add a Quantum
quantum = Quantum(run=run,
task="some.fully.qualified.SuperTask",
startTime=datetime(2018, 1, 1),
endTime=datetime(2018, 1, 2),
host="localhost")
quantum.addPredictedInput(ref1)
quantum.addPredictedInput(ref2)
# Quantum is not yet in Registry, so can't mark input as actual
with self.assertRaises(KeyError):
registry.markInputUsed(quantum, ref1)
registry.addQuantum(quantum)
# Now we can
registry.markInputUsed(quantum, ref1)
outQuantum = registry.getQuantum(quantum.id)
self.assertEqual(outQuantum, quantum)
def testCollections(self):
registry = Registry.fromConfig(self.configFile)
storageClass = makeNewStorageClass("testCollections")()
datasetType = DatasetType(name="dummytype",
dataUnits=("camera", "visit"),
storageClass=storageClass)
registry.registerDatasetType(datasetType)
collection = "ingest"
run = registry.makeRun(collection=collection)
dataId1 = {"camera": "DummyCam", "visit": 0}
inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run)
dataId2 = {"camera": "DummyCam", "visit": 1}
inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run)
# We should be able to find both datasets in their Run.collection
outputRef = registry.find(run.collection, datasetType, dataId1)
self.assertEqual(outputRef, inputRef1)
outputRef = registry.find(run.collection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
# and with the associated collection
newCollection = "something"
registry.associate(newCollection, [inputRef1, inputRef2])
outputRef = registry.find(newCollection, datasetType, dataId1)
self.assertEqual(outputRef, inputRef1)
outputRef = registry.find(newCollection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
# but no more after disassociation
registry.disassociate(newCollection, [
inputRef1,
], remove=False) # TODO test with removal when done
self.assertIsNone(registry.find(newCollection, datasetType, dataId1))
outputRef = registry.find(newCollection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
def testStorageInfo(self):
registry = Registry.fromConfig(self.configFile)
storageClass = makeNewStorageClass("testStorageInfo")()
datasetType = DatasetType(name="test",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(datasetType)
run = registry.makeRun(collection="test")
ref = registry.addDataset(datasetType,
dataId={"camera": "DummyCam"},
run=run)
datastoreName = "dummystore"
checksum = "d6fb1c0c8f338044b2faaf328f91f707"
size = 512
storageInfo = StorageInfo(datastoreName, checksum, size)
# Test adding information about a new dataset
registry.addStorageInfo(ref, storageInfo)
outStorageInfo = registry.getStorageInfo(ref, datastoreName)
self.assertEqual(outStorageInfo, storageInfo)
# Test updating storage information for an existing dataset
updatedStorageInfo = StorageInfo(datastoreName,
"20a38163c50f4aa3aa0f4047674f8ca7",
size + 1)
registry.updateStorageInfo(ref, datastoreName, updatedStorageInfo)
outStorageInfo = registry.getStorageInfo(ref, datastoreName)
self.assertNotEqual(outStorageInfo, storageInfo)
self.assertEqual(outStorageInfo, updatedStorageInfo)
def testFind(self):
registry = Registry.fromConfig(self.configFile)
storageClass = StorageClass("testFind")
datasetType = DatasetType(name="dummytype", dataUnits=("Camera", "Visit"), storageClass=storageClass)
registry.registerDatasetType(datasetType)
collection = "test"
dataId = {"camera": "DummyCam", "visit": 0}
run = registry.makeRun(collection=collection)
inputRef = registry.addDataset(datasetType, dataId=dataId, run=run)
outputRef = registry.find(collection, datasetType, dataId)
self.assertEqual(outputRef, inputRef)
# Check that retrieval with invalid dataId raises
with self.assertRaises(ValueError):
dataId = {"camera": "DummyCam", "abstract_filter": "g"} # should be visit
registry.find(collection, datasetType, dataId)
# Check that different dataIds match to different datasets
dataId1 = {"camera": "DummyCam", "visit": 1}
inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run)
dataId2 = {"camera": "DummyCam", "visit": 2}
inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run)
dataId3 = {"camera": "MyCam", "visit": 2}
inputRef3 = registry.addDataset(datasetType, dataId=dataId3, run=run)
self.assertEqual(registry.find(collection, datasetType, dataId1), inputRef1)
self.assertEqual(registry.find(collection, datasetType, dataId2), inputRef2)
self.assertEqual(registry.find(collection, datasetType, dataId3), inputRef3)
self.assertNotEqual(registry.find(collection, datasetType, dataId1), inputRef2)
self.assertNotEqual(registry.find(collection, datasetType, dataId2), inputRef1)
self.assertNotEqual(registry.find(collection, datasetType, dataId3), inputRef1)
# Check that requesting a non-existing dataId returns None
nonExistingDataId = {"camera": "DummyCam", "visit": 42}
self.assertIsNone(registry.find(collection, datasetType, nonExistingDataId))
def testDataset(self):
registry = Registry.fromConfig(self.configFile)
run = registry.makeRun(collection="test")
storageClass = StorageClass("testDataset")
datasetType = DatasetType(name="testtype", dataUnits=("Camera",), storageClass=storageClass)
registry.registerDatasetType(datasetType)
ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run)
outRef = registry.getDataset(ref.id)
self.assertEqual(ref, outRef)
def testEquality(self):
storageA = makeNewStorageClass("test_a")()
storageB = makeNewStorageClass("test_b")()
self.assertEqual(DatasetType("a", ("UnitA", ), storageA,),
DatasetType("a", ("UnitA", ), storageA,))
self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,),
DatasetType("b", ("UnitA", ), storageA,))
self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,),
DatasetType("a", ("UnitA", ), storageB,))
self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,),
DatasetType("a", ("UnitB", ), storageA,))
def testAssembler(self):
registry = Registry.fromConfig(self.configFile)
storageClass = StorageClass("testAssembler")
datasetType = DatasetType(name="test", dataUnits=("Camera",), storageClass=storageClass)
registry.registerDatasetType(datasetType)
run = registry.makeRun(collection="test")
ref = registry.addDataset(datasetType, dataId={"camera": "DummyCam"}, run=run)
self.assertIsNone(ref.assembler)
assembler = "some.fully.qualified.assembler" # TODO replace by actual dummy assember once implemented
registry.setAssembler(ref, assembler)
self.assertEqual(ref.assembler, assembler)
def testConstructor(self):
"""Test construction preserves values.
Note that construction doesn't check for valid storageClass or
dataUnits parameters.
These can only be verified for a particular schema.
"""
datasetTypeName = "test"
storageClass = makeNewStorageClass("test_StructuredData")()
dataUnits = frozenset(("camera", "visit"))
datasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
self.assertEqual(datasetType.name, datasetTypeName)
self.assertEqual(datasetType.storageClass, storageClass)
self.assertEqual(datasetType.dataUnits, dataUnits)
def testDatasetType(self):
registry = Registry.fromConfig(self.configFile)
# Check valid insert
datasetTypeName = "test"
storageClass = makeNewStorageClass("testDatasetType")()
dataUnits = ("camera", "visit")
inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
registry.registerDatasetType(inDatasetType)
outDatasetType = registry.getDatasetType(datasetTypeName)
self.assertEqual(outDatasetType, inDatasetType)
# Re-inserting should fail
with self.assertRaises(KeyError):
registry.registerDatasetType(inDatasetType)
# Template can be None
datasetTypeName = "testNoneTemplate"
storageClass = makeNewStorageClass("testDatasetType2")()
dataUnits = ("camera", "visit")
inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
registry.registerDatasetType(inDatasetType)
outDatasetType = registry.getDatasetType(datasetTypeName)
self.assertEqual(outDatasetType, inDatasetType)
def testConstructor(self):
"""Test construction preserves values.
"""
datasetTypeName = "test"
storageClass = makeNewStorageClass("testref_StructuredData")()
dataUnits = frozenset(("camera", "visit"))
dataId = dict(camera="DummyCam", visit=42)
datasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
ref = DatasetRef(datasetType, dataId)
self.assertEqual(ref.datasetType, datasetType)
self.assertEqual(ref.dataId, dataId)
self.assertIsNone(ref.producer)
self.assertEqual(ref.predictedConsumers, dict())
self.assertEqual(ref.actualConsumers, dict())
self.assertEqual(ref.components, dict())
def makeDatasetType(cls, dsConfig):
"""Create new instance of the `DatasetType` from task config.
Parameters
----------
dsConfig : `pexConfig.Config`
Instance of the `supertask.InputDatasetConfig` or
`supertask.OutputDatasetConfig`
Returns
-------
`butler.core.datasets.DatasetType` instance.
"""
# map storage class name to storage class
storageClass = cls.storageClassFactory.getStorageClass(dsConfig.storageClass)
return DatasetType(name=dsConfig.name,
dataUnits=dsConfig.units,
storageClass=storageClass)
def testAddInputsOutputs(self):
"""Test of addPredictedInput() method.
"""
quantum = Quantum(task="some.task.object", run=None)
# start with empty
self.assertEqual(quantum.predictedInputs, dict())
universe = DimensionUniverse.fromConfig()
instrument = "DummyCam"
datasetTypeName = "test_ds"
storageClass = StorageClass("testref_StructuredData")
datasetType = DatasetType(datasetTypeName,
universe.extract(("instrument", "visit")),
storageClass)
# add one ref
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42))
quantum.addPredictedInput(ref)
self.assertIn(datasetTypeName, quantum.predictedInputs)
self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 1)
# add second ref
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43))
quantum.addPredictedInput(ref)
self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 2)
# mark last ref as actually used
self.assertEqual(quantum.actualInputs, dict())
quantum._markInputUsed(ref)
self.assertIn(datasetTypeName, quantum.actualInputs)
self.assertEqual(len(quantum.actualInputs[datasetTypeName]), 1)
# add couple of outputs too
self.assertEqual(quantum.outputs, dict())
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42))
quantum.addOutput(ref)
self.assertIn(datasetTypeName, quantum.outputs)
self.assertEqual(len(quantum.outputs[datasetTypeName]), 1)
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43))
quantum.addOutput(ref)
self.assertEqual(len(quantum.outputs[datasetTypeName]), 2)