def _coerceType(self, inMemoryDataset: Any, storageClass: StorageClass,
pytype: Optional[Type[Any]] = None) -> Any:
"""Coerce the supplied inMemoryDataset to type `pytype`.
Parameters
----------
inMemoryDataset : `object`
Object to coerce to expected type.
storageClass : `StorageClass`
StorageClass associated with `inMemoryDataset`.
pytype : `type`, optional
Override type to use for conversion.
Returns
-------
inMemoryDataset : `object`
Object of expected type `pytype`.
"""
if inMemoryDataset is not None and pytype is not None and not hasattr(builtins, pytype.__name__):
if storageClass.isComposite():
inMemoryDataset = storageClass.delegate().assemble(inMemoryDataset, pytype=pytype)
elif not isinstance(inMemoryDataset, pytype):
# Hope that we can pass the arguments in directly
inMemoryDataset = pytype(inMemoryDataset)
return inMemoryDataset
def testParentPlaceholder(self):
"""Test that a parent placeholder can be replaced."""
storageComp = StorageClass("component")
storageParent = StorageClass("Parent")
dimensions = self.universe.extract(["instrument"])
component = DatasetType("a.b", dimensions, storageComp,
parentStorageClass=DatasetType.PlaceholderParentStorageClass)
self.assertIsNotNone(component.parentStorageClass)
with self.assertRaises(ValueError):
component.finalizeParentStorageClass("parent")
component.finalizeParentStorageClass(storageParent)
self.assertEqual(component.parentStorageClass, storageParent)
component = DatasetType("a.b", dimensions, storageComp,
parentStorageClass=storageParent)
with self.assertRaises(ValueError):
# Can not replace unless a placeholder
component.finalizeParentStorageClass(storageComp)
datasetType = DatasetType("a", dimensions, storageParent)
with self.assertRaises(ValueError):
# Can not add parent if not component
datasetType.finalizeParentStorageClass(storageComp)
def testNameValidation(self):
"""Test that dataset type names only contain certain characters
in certain positions.
"""
dimensions = self.universe.extract(("instrument", "visit"))
goodNames = ("a", "A", "z1", "Z1", "a_1B", "A_1b")
badNames = ("1", "_", "a%b", "B+Z", "T[0]")
# Construct storage class with all the good names included as
# components so that we can test internal consistency
storageClass = StorageClass("test_StructuredData",
components={n: StorageClass("component") for n in goodNames})
for name in goodNames:
composite = DatasetType(name, dimensions, storageClass)
self.assertEqual(composite.name, name)
for suffix in goodNames:
full = DatasetType.nameWithComponent(name, suffix)
component = composite.makeComponentDatasetType(suffix)
self.assertEqual(component.name, full)
self.assertEqual(component.parentStorageClass.name, "test_StructuredData")
for suffix in badNames:
full = DatasetType.nameWithComponent(name, suffix)
with self.subTest(full=full):
with self.assertRaises(ValueError):
DatasetType(full, dimensions, storageClass)
for name in badNames:
with self.subTest(name=name):
with self.assertRaises(ValueError):
DatasetType(name, dimensions, storageClass)
def testEquality(self):
storageA = StorageClass("test_a")
storageB = StorageClass("test_b")
parent = StorageClass("test")
dimensionsA = self.universe.extract(["instrument"])
dimensionsB = self.universe.extract(["skymap"])
self.assertEqual(DatasetType("a", dimensionsA, storageA,),
DatasetType("a", dimensionsA, storageA,))
self.assertEqual(DatasetType("a", dimensionsA, "test_a",),
DatasetType("a", dimensionsA, storageA,))
self.assertEqual(DatasetType("a", dimensionsA, storageA,),
DatasetType("a", dimensionsA, "test_a",))
self.assertEqual(DatasetType("a", dimensionsA, "test_a",),
DatasetType("a", dimensionsA, "test_a",))
self.assertEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent),
DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=parent))
self.assertEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"),
DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"))
self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
DatasetType("b", dimensionsA, storageA,))
self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
DatasetType("b", dimensionsA, "test_a",))
self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
DatasetType("a", dimensionsA, storageB,))
self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
DatasetType("a", dimensionsA, "test_b",))
self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
DatasetType("a", dimensionsB, storageA,))
self.assertNotEqual(DatasetType("a", dimensionsA, storageA,),
DatasetType("a", dimensionsB, "test_a",))
self.assertNotEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageA),
DatasetType("a.b", dimensionsA, "test_b", parentStorageClass=storageB))
self.assertNotEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageA"),
DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="storageB"))
def testPickle(self):
"""Test pickle support.
"""
storageClass = StorageClass("test_pickle")
datasetTypeName = "test"
dimensions = self.universe.extract(("instrument", "visit"))
# Un-pickling requires that storage class is registered with factory.
StorageClassFactory().registerStorageClass(storageClass)
datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
self.assertIsInstance(datasetTypeOut, DatasetType)
self.assertEqual(datasetType.name, datasetTypeOut.name)
self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names)
self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass)
self.assertIsNone(datasetTypeOut.parentStorageClass)
self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration())
self.assertFalse(datasetTypeOut.isCalibration())
datasetType = DatasetType(datasetTypeName, dimensions, storageClass, isCalibration=True)
datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration())
self.assertTrue(datasetTypeOut.isCalibration())
# And again with a composite
componentStorageClass = StorageClass("pickle_component")
StorageClassFactory().registerStorageClass(componentStorageClass)
componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
dimensions, componentStorageClass,
parentStorageClass=storageClass)
datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
self.assertIsInstance(datasetTypeOut, DatasetType)
self.assertEqual(componentDatasetType.name, datasetTypeOut.name)
self.assertEqual(componentDatasetType.dimensions.names, datasetTypeOut.dimensions.names)
self.assertEqual(componentDatasetType.storageClass, datasetTypeOut.storageClass)
self.assertEqual(componentDatasetType.parentStorageClass, datasetTypeOut.parentStorageClass)
self.assertEqual(datasetTypeOut.parentStorageClass.name,
storageClass.name)
self.assertEqual(datasetTypeOut, componentDatasetType)
# Now with a string and not a real storage class to test that
# pickling doesn't force the StorageClass to be resolved
componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
dimensions, "StrangeComponent",
parentStorageClass="UnknownParent")
datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
self.assertEqual(datasetTypeOut, componentDatasetType)
self.assertEqual(datasetTypeOut._parentStorageClassName,
componentDatasetType._parentStorageClassName)
# Now with a storage class that is created by the factory
factoryStorageClassClass = StorageClassFactory.makeNewStorageClass("ParentClass")
factoryComponentStorageClassClass = StorageClassFactory.makeNewStorageClass("ComponentClass")
componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
dimensions, factoryComponentStorageClassClass(),
parentStorageClass=factoryStorageClassClass())
datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
self.assertEqual(datasetTypeOut, componentDatasetType)
self.assertEqual(datasetTypeOut._parentStorageClassName,
componentDatasetType._parentStorageClassName)
def testMap(self):
universe = DimensionUniverse()
c = CompositesMap(self.configFile, universe=universe)
# Check that a str is not supported
with self.assertRaises(ValueError):
c.shouldBeDisassembled("fred")
# These will fail (not a composite)
sc = StorageClass("StructuredDataJson")
d = DatasetType("dummyTrue", universe.empty, sc)
self.assertFalse(sc.isComposite())
self.assertFalse(d.isComposite())
self.assertFalse(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
self.assertFalse(c.shouldBeDisassembled(sc),
f"Test with StorageClass: {sc}")
# Repeat but this time use a composite storage class
sccomp = StorageClass("Dummy")
sc = StorageClass("StructuredDataJson",
components={
"dummy": sccomp,
"dummy2": sccomp
})
d = DatasetType("dummyTrue", universe.empty, sc)
self.assertTrue(sc.isComposite())
self.assertTrue(d.isComposite())
self.assertTrue(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
self.assertFalse(c.shouldBeDisassembled(sc),
f"Test with StorageClass: {sc}")
# Override with False
d = DatasetType("dummyFalse", universe.empty, sc)
self.assertFalse(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
# DatasetType that has no explicit entry
d = DatasetType("dummyFred", universe.empty, sc)
self.assertFalse(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
# StorageClass that will be disassembled
sc = StorageClass("StructuredComposite",
components={
"dummy": sccomp,
"dummy2": sccomp
})
d = DatasetType("dummyFred", universe.empty, sc)
self.assertTrue(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
# Check that we are not allowed a single component in a composite
with self.assertRaises(ValueError):
StorageClass("TestSC", components={"dummy": sccomp})
def setUp(self):
self.universe = DimensionUniverse()
datasetTypeName = "test"
self.componentStorageClass1 = StorageClass("Component1")
self.componentStorageClass2 = StorageClass("Component2")
self.parentStorageClass = StorageClass("Parent", components={"a": self.componentStorageClass1,
"b": self.componentStorageClass2})
dimensions = self.universe.extract(("instrument", "visit"))
self.dataId = dict(instrument="DummyCam", visit=42)
self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass)
def test_composites(self):
"""Test components within composite DatasetTypes."""
storageClassA = StorageClass("compA")
storageClassB = StorageClass("compB")
storageClass = StorageClass("test_composite", components={"compA": storageClassA,
"compB": storageClassB})
self.assertTrue(storageClass.isComposite())
self.assertFalse(storageClassA.isComposite())
self.assertFalse(storageClassB.isComposite())
dimensions = self.universe.extract(("instrument", "visit"))
datasetTypeComposite = DatasetType("composite", dimensions, storageClass)
datasetTypeComponentA = datasetTypeComposite.makeComponentDatasetType("compA")
datasetTypeComponentB = datasetTypeComposite.makeComponentDatasetType("compB")
self.assertTrue(datasetTypeComposite.isComposite())
self.assertFalse(datasetTypeComponentA.isComposite())
self.assertTrue(datasetTypeComponentB.isComponent())
self.assertFalse(datasetTypeComposite.isComponent())
self.assertEqual(datasetTypeComposite.name, "composite")
self.assertEqual(datasetTypeComponentA.name, "composite.compA")
self.assertEqual(datasetTypeComponentB.component(), "compB")
self.assertEqual(datasetTypeComposite.nameAndComponent(), ("composite", None))
self.assertEqual(datasetTypeComponentA.nameAndComponent(), ("composite", "compA"))
self.assertEqual(datasetTypeComponentA.parentStorageClass, storageClass)
self.assertEqual(datasetTypeComponentB.parentStorageClass, storageClass)
self.assertIsNone(datasetTypeComposite.parentStorageClass)
def testComponents(self):
registry = self.makeRegistry()
childStorageClass = StorageClass("testComponentsChild")
registry.storageClasses.registerStorageClass(childStorageClass)
parentStorageClass = StorageClass("testComponentsParent",
components={
"child1": childStorageClass,
"child2": childStorageClass
})
registry.storageClasses.registerStorageClass(parentStorageClass)
parentDatasetType = DatasetType(name="parent",
dimensions=registry.dimensions.extract(
("instrument", )),
storageClass=parentStorageClass)
childDatasetType1 = DatasetType(name="parent.child1",
dimensions=registry.dimensions.extract(
("instrument", )),
storageClass=childStorageClass)
childDatasetType2 = DatasetType(name="parent.child2",
dimensions=registry.dimensions.extract(
("instrument", )),
storageClass=childStorageClass)
registry.registerDatasetType(parentDatasetType)
registry.registerDatasetType(childDatasetType1)
registry.registerDatasetType(childDatasetType2)
dataId = {"instrument": "DummyCam"}
if not registry.limited:
registry.addDimensionEntry("instrument", dataId)
run = registry.makeRun(collection="test")
parent = registry.addDataset(parentDatasetType, dataId=dataId, run=run)
children = {
"child1":
registry.addDataset(childDatasetType1, dataId=dataId, run=run),
"child2":
registry.addDataset(childDatasetType2, dataId=dataId, run=run)
}
for name, child in children.items():
registry.attachComponent(name, parent, child)
self.assertEqual(parent.components, children)
outParent = registry.getDataset(parent.id)
self.assertEqual(outParent.components, children)
# Remove the parent; this should remove both children.
registry.removeDataset(parent)
self.assertIsNone(
registry.find(run.collection, parentDatasetType, dataId))
self.assertIsNone(
registry.find(run.collection, childDatasetType1, dataId))
self.assertIsNone(
registry.find(run.collection, childDatasetType2, dataId))
def testRegistryWithStorageClass(self):
"""Test that the registry can be given a StorageClass object.
"""
formatterTypeName = "lsst.daf.butler.formatters.yamlFormatter.YamlFormatter"
storageClassName = "TestClass"
sc = StorageClass(storageClassName, dict, None)
universe = DimensionUniverse.fromConfig()
datasetType = DatasetType("calexp", universe.extract([]), sc)
# Store using an instance
self.factory.registerFormatter(sc, formatterTypeName)
# Retrieve using the class
f = self.factory.getFormatter(sc, self.fileDescriptor)
self.assertIsFormatter(f)
self.assertEqual(f.fileDescriptor, self.fileDescriptor)
# Retrieve using the DatasetType
f2 = self.factory.getFormatter(datasetType, self.fileDescriptor)
self.assertIsFormatter(f2)
self.assertEqual(f.name(), f2.name())
# Class directly
f2cls = self.factory.getFormatterClass(datasetType)
self.assertIsFormatter(f2cls)
# This might defer the import, pytest may have already loaded it
from lsst.daf.butler.formatters.yamlFormatter import YamlFormatter
self.assertEqual(type(f), YamlFormatter)
with self.assertRaises(KeyError):
# Attempt to overwrite using a different value
self.factory.registerFormatter(storageClassName,
"lsst.daf.butler.formatters.jsonFormatter.JsonFormatter")
def setUp(self):
self.id = 0
self.factory = FormatterFactory()
# Dummy FileDescriptor for testing getFormatter
self.fileDescriptor = FileDescriptor(Location("/a/b/c", "d"),
StorageClass("DummyStorageClass", dict, None))
def setUp(self):
self.id = 0
# Create DatasetRefs to test against constraints model
self.universe = DimensionUniverse()
dimensions = self.universe.extract(
("visit", "physical_filter", "instrument"))
sc = StorageClass("DummySC", dict, None)
self.calexpA = self.makeDatasetRef("calexp",
dimensions,
sc, {
"instrument": "A",
"physical_filter": "u"
},
conform=False)
dimensions = self.universe.extract(("visit", "detector", "instrument"))
self.pviA = self.makeDatasetRef("pvi",
dimensions,
sc, {
"instrument": "A",
"visit": 1
},
conform=False)
self.pviB = self.makeDatasetRef("pvi",
dimensions,
sc, {
"instrument": "B",
"visit": 2
},
conform=False)
def testCollections(self):
registry = self.makeRegistry()
storageClass = StorageClass("testCollections")
registry.storageClasses.registerStorageClass(storageClass)
datasetType = DatasetType(name="dummytype",
dimensions=registry.dimensions.extract(
("instrument", "visit")),
storageClass=storageClass)
registry.registerDatasetType(datasetType)
if not registry.limited:
registry.addDimensionEntry("instrument",
{"instrument": "DummyCam"})
registry.addDimensionEntry("physical_filter", {
"instrument": "DummyCam",
"physical_filter": "d-r"
})
registry.addDimensionEntry("visit", {
"instrument": "DummyCam",
"visit": 0,
"physical_filter": "d-r"
})
registry.addDimensionEntry("visit", {
"instrument": "DummyCam",
"visit": 1,
"physical_filter": "d-r"
})
collection = "ingest"
run = registry.makeRun(collection=collection)
# Dataset.physical_filter should be populated as well here from the
# visit Dimension values, if the Registry isn't limited.
dataId1 = {"instrument": "DummyCam", "visit": 0}
if registry.limited:
dataId1.update(physical_filter="d-r", abstract_filter=None)
inputRef1 = registry.addDataset(datasetType, dataId=dataId1, run=run)
dataId2 = {"instrument": "DummyCam", "visit": 1}
if registry.limited:
dataId2.update(physical_filter="d-r", abstract_filter=None)
inputRef2 = registry.addDataset(datasetType, dataId=dataId2, run=run)
# We should be able to find both datasets in their Run.collection
outputRef = registry.find(run.collection, datasetType, dataId1)
self.assertEqual(outputRef, inputRef1)
outputRef = registry.find(run.collection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
# and with the associated collection
newCollection = "something"
registry.associate(newCollection, [inputRef1, inputRef2])
outputRef = registry.find(newCollection, datasetType, dataId1)
self.assertEqual(outputRef, inputRef1)
outputRef = registry.find(newCollection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
# but no more after disassociation
registry.disassociate(newCollection, [
inputRef1,
])
self.assertIsNone(registry.find(newCollection, datasetType, dataId1))
outputRef = registry.find(newCollection, datasetType, dataId2)
self.assertEqual(outputRef, inputRef2)
collections = registry.getAllCollections()
self.assertEqual(collections, {"something", "ingest"})
def testBasicPutGet(self):
metrics = makeExampleMetrics()
datastore = self.makeDatastore()
# Create multiple storage classes for testing different formulations
storageClasses = [self.storageClassFactory.getStorageClass(sc)
for sc in ("StructuredData",
"StructuredDataJson",
"StructuredDataPickle")]
dimensions = self.universe.extract(("visit", "physical_filter"))
dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
for sc in storageClasses:
ref = self.makeDatasetRef("metric", dimensions, sc, dataId)
print("Using storageClass: {}".format(sc.name))
datastore.put(metrics, ref)
# Does it exist?
self.assertTrue(datastore.exists(ref))
# Get
metricsOut = datastore.get(ref, parameters=None)
self.assertEqual(metrics, metricsOut)
uri = datastore.getUri(ref)
self.assertEqual(uri[:len(self.uriScheme)], self.uriScheme)
# Get a component -- we need to construct new refs for them
# with derived storage classes but with parent ID
comp = "output"
compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(comp), dimensions,
sc.components[comp], dataId, id=ref.id)
output = datastore.get(compRef)
self.assertEqual(output, metricsOut.output)
uri = datastore.getUri(compRef)
self.assertEqual(uri[:len(self.uriScheme)], self.uriScheme)
storageClass = sc
# Check that a put fails if the dataset type is not supported
if self.hasUnsupportedPut:
sc = StorageClass("UnsupportedSC", pytype=type(metrics))
ref = self.makeDatasetRef("unsupportedType", dimensions, sc, dataId)
with self.assertRaises(DatasetTypeNotSupportedError):
datastore.put(metrics, ref)
# These should raise
ref = self.makeDatasetRef("metrics", dimensions, storageClass, dataId, id=10000)
with self.assertRaises(FileNotFoundError):
# non-existing file
datastore.get(ref)
# Get a URI from it
uri = datastore.getUri(ref, predict=True)
self.assertEqual(uri[:len(self.uriScheme)], self.uriScheme)
with self.assertRaises(FileNotFoundError):
datastore.getUri(ref)
def testConstructor(self):
"""Test construction preserves values.
Note that construction doesn't check for valid storageClass.
This can only be verified for a particular schema.
"""
datasetTypeName = "test"
storageClass = StorageClass("test_StructuredData")
dimensions = self.universe.extract(("instrument", "visit"))
datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
self.assertEqual(datasetType.name, datasetTypeName)
self.assertEqual(datasetType.storageClass, storageClass)
self.assertEqual(datasetType.dimensions, dimensions)
with self.assertRaises(
ValueError,
msg="Construct component without parent storage class"):
DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
dimensions, storageClass)
with self.assertRaises(
ValueError,
msg="Construct non-component with parent storage class"):
DatasetType(datasetTypeName,
dimensions,
storageClass,
parentStorageClass="NotAllowed")
def testDeepCopy(self):
"""Test that we can copy a dataset type."""
storageClass = StorageClass("test_copy")
datasetTypeName = "test"
dimensions = self.universe.extract(("instrument", "visit"))
datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
dcopy = copy.deepcopy(datasetType)
self.assertEqual(dcopy, datasetType)
# And again with a composite
componentStorageClass = StorageClass("copy_component")
componentDatasetType = DatasetType(DatasetType.nameWithComponent(
datasetTypeName, "comp"),
dimensions,
componentStorageClass,
parentStorageClass=storageClass)
dcopy = copy.deepcopy(componentDatasetType)
self.assertEqual(dcopy, componentDatasetType)
def makeDatasetRef(self, datasetTypeName, dataId=None):
"""Make a simple DatasetRef"""
if dataId is None:
dataId = self.dataId
if datasetTypeName not in self.datasetTypes:
self.datasetTypes[datasetTypeName] = DatasetType(
datasetTypeName, list(dataId.keys()), StorageClass())
datasetType = self.datasetTypes[datasetTypeName]
return DatasetRef(datasetType, dataId)
def testJson(self):
storageA = StorageClass("test_a")
dimensionsA = self.universe.extract(["instrument"])
self.assertEqual(DatasetType("a", dimensionsA, storageA,),
DatasetType.from_json(DatasetType("a", dimensionsA, storageA,).to_json(),
self.universe))
self.assertEqual(DatasetType("a.b", dimensionsA, "test_b", parentStorageClass="parent"),
DatasetType.from_json(DatasetType("a.b", dimensionsA, "test_b",
parentStorageClass="parent").to_json(),
self.universe))
def setUp(self):
self.id = 0
self.factory = FormatterFactory()
self.universe = DimensionUniverse()
self.dataId = DataCoordinate.makeEmpty(self.universe)
# Dummy FileDescriptor for testing getFormatter
self.fileDescriptor = FileDescriptor(
Location("/a/b/c", "d"),
StorageClass("DummyStorageClass", dict, None))
def testPickle(self):
"""Test that we can pickle storageclasses.
"""
className = "TestImage"
sc = StorageClass(className, pytype=dict)
self.assertIsInstance(sc, StorageClass)
self.assertEqual(sc.name, className)
self.assertFalse(sc.components)
sc2 = pickle.loads(pickle.dumps(sc))
self.assertEqual(sc2, sc)
def testTypeNames(self):
# Check types and also an object
tests = [(Formatter, "lsst.daf.butler.core.formatter.Formatter"),
(doImport, "lsst.daf.butler.core.utils.doImport"),
(int, "builtins.int"),
(StorageClass, "lsst.daf.butler.core.storageClass.StorageClass"),
(StorageClass(None), "lsst.daf.butler.core.storageClass.StorageClass")]
for item, typeName in tests:
self.assertEqual(getFullTypeName(item), typeName)
def testQueryDatasetTypes(self):
self.maxDiff = None
datasetName = "test"
instrumentDimension = "instrument"
visitDimension = "visit"
storageClassName = "testDatasetType"
expectedNotVerbose = AstropyTable((("test", ), ), names=("name", ))
runner = LogCliRunner()
with runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
butler = Butler(butlerCfg, writeable=True)
storageClass = StorageClass(storageClassName)
butler.registry.storageClasses.registerStorageClass(storageClass)
dimensions = butler.registry.dimensions.extract(
(instrumentDimension, visitDimension))
datasetType = DatasetType(datasetName, dimensions, storageClass)
butler.registry.registerDatasetType(datasetType)
# check not-verbose output:
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(readTable(result.output),
expectedNotVerbose)
# check glob output:
result = runner.invoke(cli, ["query-dataset-types", "here", "t*"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(readTable(result.output),
expectedNotVerbose)
# check verbose output:
result = runner.invoke(
cli, ["query-dataset-types", "here", "--verbose"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = AstropyTable(array((
"test",
"['band', 'instrument', 'physical_filter', 'visit_system', 'visit']",
"testDatasetType")),
names=("name", "dimensions",
"storage class"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
# Now remove and check that it was removed
# First a non-existent one
result = runner.invoke(cli,
["remove-dataset-type", "here", "unreal"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Now one we now has been registered
result = runner.invoke(
cli, ["remove-dataset-type", "here", datasetName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# and check that it has gone
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn("No results", result.output)
def testConstructor2(self):
"""Test construction from StorageClass name.
"""
datasetTypeName = "test"
storageClass = StorageClass("test_constructor2")
StorageClassFactory().registerStorageClass(storageClass)
dimensions = self.universe.extract(("instrument", "visit"))
datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2")
self.assertEqual(datasetType.name, datasetTypeName)
self.assertEqual(datasetType.storageClass, storageClass)
self.assertEqual(datasetType.dimensions, dimensions)
def testTypeNames(self):
# Check types and also an object
tests = [(Formatter, "lsst.daf.butler.core.formatter.Formatter"),
(int, "int"),
(StorageClass,
"lsst.daf.butler.core.storageClass.StorageClass"),
(StorageClass(None),
"lsst.daf.butler.core.storageClass.StorageClass"),
(Registry, "lsst.daf.butler.registry.Registry")]
for item, typeName in tests:
self.assertEqual(getFullTypeName(item), typeName)
def testHashability(self):
"""Test `DatasetType.__hash__`.
This test is performed by checking that `DatasetType` entries can
be inserted into a `set` and that unique values of its
(`name`, `storageClass`, `dimensions`) parameters result in separate
entries (and equal ones don't).
This does not check for uniformity of hashing or the actual values
of the hash function.
"""
types = []
unique = 0
storageC = StorageClass("test_c")
storageD = StorageClass("test_d")
for name in ["a", "b"]:
for storageClass in [storageC, storageD]:
for dimensions in [("instrument", ), ("skymap", )]:
datasetType = DatasetType(
name, self.universe.extract(dimensions), storageClass)
datasetTypeCopy = DatasetType(
name, self.universe.extract(dimensions), storageClass)
types.extend((datasetType, datasetTypeCopy))
unique += 1 # datasetType should always equal its copy
self.assertEqual(len(set(types)),
unique) # all other combinations are unique
# also check that hashes of instances constructed with StorageClass
# name matches hashes of instances constructed with instances
dimensions = self.universe.extract(["instrument"])
self.assertEqual(hash(DatasetType("a", dimensions, storageC)),
hash(DatasetType("a", dimensions, "test_c")))
self.assertEqual(hash(DatasetType("a", dimensions, "test_c")),
hash(DatasetType("a", dimensions, "test_c")))
self.assertNotEqual(hash(DatasetType("a", dimensions, storageC)),
hash(DatasetType("a", dimensions, "test_d")))
self.assertNotEqual(hash(DatasetType("a", dimensions, storageD)),
hash(DatasetType("a", dimensions, "test_c")))
self.assertNotEqual(hash(DatasetType("a", dimensions, "test_c")),
hash(DatasetType("a", dimensions, "test_d")))
def testDatasetLocations(self):
registry = self.makeRegistry()
storageClass = StorageClass("testStorageInfo")
registry.storageClasses.registerStorageClass(storageClass)
datasetType = DatasetType(name="test",
dimensions=registry.dimensions.extract(
("instrument", )),
storageClass=storageClass)
datasetType2 = DatasetType(name="test2",
dimensions=registry.dimensions.extract(
("instrument", )),
storageClass=storageClass)
registry.registerDatasetType(datasetType)
registry.registerDatasetType(datasetType2)
if not registry.limited:
registry.addDimensionEntry("instrument",
{"instrument": "DummyCam"})
run = registry.makeRun(collection="test")
ref = registry.addDataset(datasetType,
dataId={"instrument": "DummyCam"},
run=run)
ref2 = registry.addDataset(datasetType2,
dataId={"instrument": "DummyCam"},
run=run)
datastoreName = "dummystore"
datastoreName2 = "dummystore2"
# Test adding information about a new dataset
registry.addDatasetLocation(ref, datastoreName)
addresses = registry.getDatasetLocations(ref)
self.assertIn(datastoreName, addresses)
self.assertEqual(len(addresses), 1)
registry.addDatasetLocation(ref, datastoreName2)
registry.addDatasetLocation(ref2, datastoreName2)
addresses = registry.getDatasetLocations(ref)
self.assertEqual(len(addresses), 2)
self.assertIn(datastoreName, addresses)
self.assertIn(datastoreName2, addresses)
registry.removeDatasetLocation(datastoreName, ref)
addresses = registry.getDatasetLocations(ref)
self.assertEqual(len(addresses), 1)
self.assertNotIn(datastoreName, addresses)
self.assertIn(datastoreName2, addresses)
with self.assertRaises(OrphanedRecordError):
registry.removeDataset(ref)
registry.removeDatasetLocation(datastoreName2, ref)
addresses = registry.getDatasetLocations(ref)
self.assertEqual(len(addresses), 0)
self.assertNotIn(datastoreName2, addresses)
registry.removeDataset(ref) # should not raise
addresses = registry.getDatasetLocations(ref2)
self.assertEqual(len(addresses), 1)
self.assertIn(datastoreName2, addresses)
def testConstructor(self):
"""Test construction preserves values.
Note that construction doesn't check for valid storageClass.
This can only be verified for a particular schema.
"""
datasetTypeName = "test"
storageClass = StorageClass("test_StructuredData")
dimensions = self.universe.extract(("instrument", "visit"))
datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
self.assertEqual(datasetType.name, datasetTypeName)
self.assertEqual(datasetType.storageClass, storageClass)
self.assertEqual(datasetType.dimensions, dimensions)
def makeDatasetRef(self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass",
run="run2", conform=True):
"""Make a simple DatasetRef"""
if dataId is None:
dataId = self.dataId
# Pretend we have a parent if this looks like a composite
compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
parentStorageClass = DatasetType.PlaceholderParentStorageClass if componentName else None
datasetType = DatasetType(datasetTypeName, DimensionGraph(self.universe, names=dataId.keys()),
StorageClass(storageClassName),
parentStorageClass=parentStorageClass)
return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
def testPickle(self):
"""Test pickle support.
"""
storageClass = StorageClass("test_pickle")
datasetTypeName = "test"
dimensions = self.universe.extract(("instrument", "visit"))
# Un-pickling requires that storage class is registered with factory.
StorageClassFactory().registerStorageClass(storageClass)
datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
self.assertIsInstance(datasetTypeOut, DatasetType)
self.assertEqual(datasetType.name, datasetTypeOut.name)
self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names)
self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass)
def testRegistry(self):
"""Check that storage classes can be created on the fly and stored
in a registry."""
className = "TestImage"
factory = StorageClassFactory()
newclass = StorageClass(className, pytype=PythonType)
factory.registerStorageClass(newclass)
sc = factory.getStorageClass(className)
self.assertIsInstance(sc, StorageClass)
self.assertEqual(sc.name, className)
self.assertFalse(sc.components)
self.assertEqual(sc.pytype, PythonType)
self.assertIn(sc, factory)
newclass2 = StorageClass("Temporary2", pytype=str)
self.assertNotIn(newclass2, factory)
factory.registerStorageClass(newclass2)
self.assertIn(newclass2, factory)
self.assertIn("Temporary2", factory)
self.assertNotIn("Temporary3", factory)
self.assertNotIn({}, factory)
# Make sure we can't register a storage class with the same name
# but different values
newclass3 = StorageClass("Temporary2", pytype=dict)
with self.assertRaises(ValueError):
factory.registerStorageClass(newclass3)
factory._unregisterStorageClass(newclass3.name)
self.assertNotIn(newclass3, factory)
self.assertNotIn(newclass3.name, factory)
factory.registerStorageClass(newclass3)
self.assertIn(newclass3, factory)
self.assertIn(newclass3.name, factory)
# Check you can silently insert something that is already there
factory.registerStorageClass(newclass3)