def testEquality(self):
storageA = makeNewStorageClass("test_a")()
storageB = makeNewStorageClass("test_b")()
self.assertEqual(DatasetType("a", ("UnitA", ), storageA,),
DatasetType("a", ("UnitA", ), storageA,))
self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,),
DatasetType("b", ("UnitA", ), storageA,))
self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,),
DatasetType("a", ("UnitA", ), storageB,))
self.assertNotEqual(DatasetType("a", ("UnitA", ), storageA,),
DatasetType("a", ("UnitB", ), storageA,))
def testComponents(self):
registry = Registry.fromConfig(self.configFile)
storageClass = makeNewStorageClass("testComponents")()
parentDatasetType = DatasetType(name="parent",
dataUnits=("camera", ),
storageClass=storageClass)
childDatasetType1 = DatasetType(name="child1",
dataUnits=("camera", ),
storageClass=storageClass)
childDatasetType2 = DatasetType(name="child2",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(parentDatasetType)
registry.registerDatasetType(childDatasetType1)
registry.registerDatasetType(childDatasetType2)
run = registry.makeRun(collection="test")
parent = registry.addDataset(parentDatasetType,
dataId={"camera": "DummyCam"},
run=run)
children = {
"child1":
registry.addDataset(childDatasetType1,
dataId={"camera": "DummyCam"},
run=run),
"child2":
registry.addDataset(childDatasetType2,
dataId={"camera": "DummyCam"},
run=run)
}
for name, child in children.items():
registry.attachComponent(name, parent, child)
self.assertEqual(parent.components, children)
outParent = registry.getDataset(parent.id)
self.assertEqual(outParent.components, children)
def testStorageInfo(self):
registry = Registry.fromConfig(self.configFile)
storageClass = makeNewStorageClass("testStorageInfo")()
datasetType = DatasetType(name="test",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(datasetType)
run = registry.makeRun(collection="test")
ref = registry.addDataset(datasetType,
dataId={"camera": "DummyCam"},
run=run)
datastoreName = "dummystore"
checksum = "d6fb1c0c8f338044b2faaf328f91f707"
size = 512
storageInfo = StorageInfo(datastoreName, checksum, size)
# Test adding information about a new dataset
registry.addStorageInfo(ref, storageInfo)
outStorageInfo = registry.getStorageInfo(ref, datastoreName)
self.assertEqual(outStorageInfo, storageInfo)
# Test updating storage information for an existing dataset
updatedStorageInfo = StorageInfo(datastoreName,
"20a38163c50f4aa3aa0f4047674f8ca7",
size + 1)
registry.updateStorageInfo(ref, datastoreName, updatedStorageInfo)
outStorageInfo = registry.getStorageInfo(ref, datastoreName)
self.assertNotEqual(outStorageInfo, storageInfo)
self.assertEqual(outStorageInfo, updatedStorageInfo)
def testCollections(self):
registry = Registry.fromConfig(self.configFile)
storageClass = makeNewStorageClass("testCollections")()
datasetType = DatasetType(name="dummytype",
dataUnits=("camera", "visit"),
storageClass=storageClass)
registry.registerDatasetType(datasetType)
collection = "ingest"
run = registry.makeRun(collection=collection)
dataId1 = {"camera": "DummyCam", "visit": 0}
inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run)
dataId2 = {"camera": "DummyCam", "visit": 1}
inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run)
# We should be able to find both datasets in their Run.collection
outputRef = registry.find(run.collection, datasetType, dataId1)
self.assertEqual(outputRef, inputRef1)
outputRef = registry.find(run.collection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
# and with the associated collection
newCollection = "something"
registry.associate(newCollection, [inputRef1, inputRef2])
outputRef = registry.find(newCollection, datasetType, dataId1)
self.assertEqual(outputRef, inputRef1)
outputRef = registry.find(newCollection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
# but no more after disassociation
registry.disassociate(newCollection, [
inputRef1,
], remove=False) # TODO test with removal when done
self.assertIsNone(registry.find(newCollection, datasetType, dataId1))
outputRef = registry.find(newCollection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
def testQuantum(self):
registry = Registry.fromConfig(self.configFile)
run = registry.makeRun(collection="test")
storageClass = makeNewStorageClass("testQuantum")()
# Make two predicted inputs
datasetType1 = DatasetType(name="dst1",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(datasetType1)
ref1 = registry.addDataset(datasetType1,
dataId={"camera": "DummyCam"},
run=run)
datasetType2 = DatasetType(name="dst2",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(datasetType2)
ref2 = registry.addDataset(datasetType2,
dataId={"camera": "DummyCam"},
run=run)
# Create and add a Quantum
quantum = Quantum(run=run,
task="some.fully.qualified.SuperTask",
startTime=datetime(2018, 1, 1),
endTime=datetime(2018, 1, 2),
host="localhost")
quantum.addPredictedInput(ref1)
quantum.addPredictedInput(ref2)
# Quantum is not yet in Registry, so can't mark input as actual
with self.assertRaises(KeyError):
registry.markInputUsed(quantum, ref1)
registry.addQuantum(quantum)
# Now we can
registry.markInputUsed(quantum, ref1)
outQuantum = registry.getQuantum(quantum.id)
self.assertEqual(outQuantum, quantum)
def testDataset(self):
registry = Registry.fromConfig(self.configFile)
run = registry.makeRun(collection="test")
storageClass = makeNewStorageClass("testDataset")()
datasetType = DatasetType(name="testtype",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(datasetType)
ref = registry.addDataset(datasetType,
dataId={"camera": "DummyCam"},
run=run)
outRef = registry.getDataset(ref.id)
self.assertEqual(ref, outRef)
def testRegistry(self):
"""Check that storage classes can be created on the fly and stored
in a registry."""
className = "TestImage"
factory = storageClass.StorageClassFactory()
newclass = storageClass.makeNewStorageClass(className,
pytype=PythonType)
factory.registerStorageClass(newclass)
sc = factory.getStorageClass(className)
self.assertIsInstance(sc, storageClass.StorageClass)
self.assertEqual(sc.name, className)
self.assertFalse(sc.components)
self.assertEqual(sc.pytype, PythonType)
def testHashability(self):
"""Test `DatasetType.__hash__`.
This test is performed by checking that `DatasetType` entries can
be inserted into a `set` and that unique values of its
(`name`, `storageClass`, `dataUnits`) parameters result in separate
entries (and equal ones don't).
This does not check for uniformity of hashing or the actual values
of the hash function.
"""
types = []
unique = 0
storageC = makeNewStorageClass("test_c")()
storageD = makeNewStorageClass("test_d")()
for name in ["a", "b"]:
for storageClass in [storageC, storageD]:
for dataUnits in [("e", ), ("f", )]:
datasetType = DatasetType(name, dataUnits, storageClass)
datasetTypeCopy = DatasetType(name, dataUnits, storageClass)
types.extend((datasetType, datasetTypeCopy))
unique += 1 # datasetType should always equal its copy
self.assertEqual(len(set(types)), unique) # all other combinations are unique
def testConstructor(self):
"""Test construction preserves values.
Note that construction doesn't check for valid storageClass or
dataUnits parameters.
These can only be verified for a particular schema.
"""
datasetTypeName = "test"
storageClass = makeNewStorageClass("test_StructuredData")()
dataUnits = frozenset(("camera", "visit"))
datasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
self.assertEqual(datasetType.name, datasetTypeName)
self.assertEqual(datasetType.storageClass, storageClass)
self.assertEqual(datasetType.dataUnits, dataUnits)
def testDatasetType(self):
registry = Registry.fromConfig(self.configFile)
# Check valid insert
datasetTypeName = "test"
storageClass = makeNewStorageClass("testDatasetType")()
dataUnits = ("camera", "visit")
inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
registry.registerDatasetType(inDatasetType)
outDatasetType = registry.getDatasetType(datasetTypeName)
self.assertEqual(outDatasetType, inDatasetType)
# Re-inserting should fail
with self.assertRaises(KeyError):
registry.registerDatasetType(inDatasetType)
# Template can be None
datasetTypeName = "testNoneTemplate"
storageClass = makeNewStorageClass("testDatasetType2")()
dataUnits = ("camera", "visit")
inDatasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
registry.registerDatasetType(inDatasetType)
outDatasetType = registry.getDatasetType(datasetTypeName)
self.assertEqual(outDatasetType, inDatasetType)
def testCreation(self):
"""Test that we can dynamically create storage class subclasses.
This is critical for testing the factory functions."""
className = "TestImage"
newclass = storageClass.makeNewStorageClass(className, pytype=dict)
sc = newclass()
self.assertIsInstance(sc, storageClass.StorageClass)
self.assertEqual(sc.name, className)
self.assertFalse(sc.components)
# Test the caching by using private class attribute
self.assertIsNone(newclass._pytype)
self.assertEqual(sc.pytype, dict)
self.assertEqual(newclass._pytype, dict)
# Check we can create a storageClass using the name of an importable
# type.
newclass = storageClass.makeNewStorageClass(
"TestImage2",
"lsst.daf.butler.core.storageClass.StorageClassFactory")
self.assertIsInstance(newclass().pytype(),
storageClass.StorageClassFactory)
def testConstructor(self):
"""Test construction preserves values.
"""
datasetTypeName = "test"
storageClass = makeNewStorageClass("testref_StructuredData")()
dataUnits = frozenset(("camera", "visit"))
dataId = dict(camera="DummyCam", visit=42)
datasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
ref = DatasetRef(datasetType, dataId)
self.assertEqual(ref.datasetType, datasetType)
self.assertEqual(ref.dataId, dataId)
self.assertIsNone(ref.producer)
self.assertEqual(ref.predictedConsumers, dict())
self.assertEqual(ref.actualConsumers, dict())
self.assertEqual(ref.components, dict())
def testAssembler(self):
registry = Registry.fromConfig(self.configFile)
storageClass = makeNewStorageClass("testAssembler")()
datasetType = DatasetType(name="test",
dataUnits=("camera", ),
storageClass=storageClass)
registry.registerDatasetType(datasetType)
run = registry.makeRun(collection="test")
ref = registry.addDataset(datasetType,
dataId={"camera": "DummyCam"},
run=run)
self.assertIsNone(ref.assembler)
assembler = "some.fully.qualified.assembler" # TODO replace by actual dummy assember once implemented
registry.setAssembler(ref, assembler)
self.assertEqual(ref.assembler, assembler)
def testFind(self):
registry = Registry.fromConfig(self.configFile)
storageClass = makeNewStorageClass("testFind")()
datasetType = DatasetType(name="dummytype",
dataUnits=("camera", "visit"),
storageClass=storageClass)
registry.registerDatasetType(datasetType)
collection = "test"
dataId = {"camera": "DummyCam", "visit": 0}
run = registry.makeRun(collection=collection)
inputRef = registry.addDataset(datasetType, dataId=dataId, run=run)
outputRef = registry.find(collection, datasetType, dataId)
self.assertEqual(outputRef, inputRef)
# Check that retrieval with invalid dataId raises
with self.assertRaises(ValueError):
dataId = {
"camera": "DummyCam",
"abstract_filter": "g"
} # should be visit
registry.find(collection, datasetType, dataId)
# Check that different dataIds match to different datasets
dataId1 = {"camera": "DummyCam", "visit": 1}
inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run)
dataId2 = {"camera": "DummyCam", "visit": 2}
inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run)
dataId3 = {"camera": "MyCam", "visit": 2}
inputRef3 = registry.addDataset(datasetType, dataId=dataId3, run=run)
self.assertEqual(registry.find(collection, datasetType, dataId1),
inputRef1)
self.assertEqual(registry.find(collection, datasetType, dataId2),
inputRef2)
self.assertEqual(registry.find(collection, datasetType, dataId3),
inputRef3)
self.assertNotEqual(registry.find(collection, datasetType, dataId1),
inputRef2)
self.assertNotEqual(registry.find(collection, datasetType, dataId2),
inputRef1)
self.assertNotEqual(registry.find(collection, datasetType, dataId3),
inputRef1)
# Check that requesting a non-existing dataId returns None
nonExistingDataId = {"camera": "DummyCam", "visit": 42}
self.assertIsNone(
registry.find(collection, datasetType, nonExistingDataId))
def testRegistryWithStorageClass(self):
"""Test that the registry can be given a StorageClass object.
"""
formatterTypeName = "lsst.daf.butler.formatters.yamlFormatter.YamlFormatter"
storageClassName = "TestClass"
sc = storageClass.makeNewStorageClass(storageClassName, dict, None)
# Store using an instance
self.factory.registerFormatter(sc(), formatterTypeName)
# Retrieve using the class
f = self.factory.getFormatter(sc)
self.assertIsInstance(f, formatter.Formatter)
# This might defer the import, pytest may have already loaded it
from lsst.daf.butler.formatters.yamlFormatter import YamlFormatter
self.assertEqual(type(f), YamlFormatter)
with self.assertRaises(KeyError):
# Attempt to overwrite using a different value
self.factory.registerFormatter(
storageClassName,
"lsst.daf.butler.formatters.jsonFormatter.JsonFormatter")
