def setUp(self):
self.id = 0
# Create DatasetRefs to test against constraints model
self.universe = DimensionUniverse()
dimensions = self.universe.extract(
("visit", "physical_filter", "instrument"))
sc = StorageClass("DummySC", dict, None)
self.calexpA = self.makeDatasetRef("calexp",
dimensions,
sc, {
"instrument": "A",
"physical_filter": "u"
},
conform=False)
dimensions = self.universe.extract(("visit", "detector", "instrument"))
self.pviA = self.makeDatasetRef("pvi",
dimensions,
sc, {
"instrument": "A",
"visit": 1
},
conform=False)
self.pviB = self.makeDatasetRef("pvi",
dimensions,
sc, {
"instrument": "B",
"visit": 2
},
conform=False)
def setUp(self):
self.universe = DimensionUniverse()
datasetTypeName = "test"
self.componentStorageClass1 = StorageClass("Component1")
self.componentStorageClass2 = StorageClass("Component2")
self.parentStorageClass = StorageClass("Parent", components={"a": self.componentStorageClass1,
"b": self.componentStorageClass2})
dimensions = self.universe.extract(("instrument", "visit"))
self.dataId = dict(instrument="DummyCam", visit=42)
self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass)
def setUp(self):
self.id = 0
self.factory = FormatterFactory()
self.universe = DimensionUniverse()
self.dataId = DataCoordinate.makeEmpty(self.universe)
# Dummy FileDescriptor for testing getFormatter
self.fileDescriptor = FileDescriptor(
Location("/a/b/c", "d"),
StorageClass("DummyStorageClass", dict, None))
def makeRegistry(self) -> Registry:
prefix = f"test_{secrets.token_hex(8).lower()}_"
self._prefixes.append(prefix)
config = self.makeRegistryConfig()
# Can't use Registry.fromConfig for these tests because we don't want
# to reconnect to the server every single time. But we at least use
# OracleDatabase.fromConnection rather than the constructor so
# we can try to pass a prefix through via "+" in a namespace.
database = OracleDatabase.fromConnection(connection=self._connection,
origin=0,
namespace=f"+{prefix}")
attributes = doImport(config["managers", "attributes"])
opaque = doImport(config["managers", "opaque"])
dimensions = doImport(config["managers", "dimensions"])
collections = doImport(config["managers", "collections"])
datasets = doImport(config["managers", "datasets"])
datastoreBridges = doImport(config["managers", "datastores"])
return Registry(database=database,
attributes=attributes,
opaque=opaque,
dimensions=dimensions,
collections=collections,
datasets=datasets,
datastoreBridges=datastoreBridges,
universe=DimensionUniverse(config),
create=True)
def testConstructor(self):
"""Test DatasetTypeDescriptor init
"""
name = "testDataset"
dimensionNames = frozenset(["label"])
storageClassName = "Catalog"
universe = DimensionUniverse.fromConfig()
descriptor = pipeBase.DatasetTypeDescriptor(
name=name,
dimensionNames=dimensionNames,
storageClassName=storageClassName,
scalar=False,
manualLoad=False)
datasetType = descriptor.makeDatasetType(universe)
self.assertEqual(datasetType.name, name)
self.assertEqual(datasetType.dimensions.names, dimensionNames)
self.assertEqual(datasetType.storageClass.name, storageClassName)
self.assertFalse(descriptor.scalar)
descriptor = pipeBase.DatasetTypeDescriptor(
name=name,
dimensionNames=dimensionNames,
storageClassName=storageClassName,
scalar=True,
manualLoad=False)
datasetType = descriptor.makeDatasetType(universe)
self.assertEqual(datasetType.name, name)
self.assertEqual(datasetType.dimensions.names, dimensionNames)
self.assertEqual(datasetType.storageClass.name, storageClassName)
self.assertTrue(descriptor.scalar)
def setUp(self):
self.universe = DimensionUniverse.fromConfig()
self.given = DimensionSet(universe=self.universe, elements=["skymap"])
self.parameters = dict(skymap="unimportant",
tractMax=5,
patchNxMax=3,
patchNyMax=3)
def makeDatasetType(
self, universe: DimensionUniverse, parentStorageClass: Optional[Union[StorageClass, str]] = None
) -> DatasetType:
"""Construct a true `DatasetType` instance with normalized dimensions.
Parameters
----------
universe : `lsst.daf.butler.DimensionUniverse`
Set of all known dimensions to be used to normalize the dimension
names specified in config.
parentStorageClass : `lsst.daf.butler.StorageClass` or `str`, optional
Parent storage class for component datasets; `None` otherwise.
Returns
-------
datasetType : `DatasetType`
The `DatasetType` defined by this connection.
"""
return DatasetType(
self.name,
universe.extract(self.dimensions),
self.storageClass,
isCalibration=self.isCalibration,
parentStorageClass=parentStorageClass,
)
def testRegistryWithStorageClass(self):
"""Test that the registry can be given a StorageClass object.
"""
formatterTypeName = "lsst.daf.butler.formatters.yamlFormatter.YamlFormatter"
storageClassName = "TestClass"
sc = StorageClass(storageClassName, dict, None)
universe = DimensionUniverse.fromConfig()
datasetType = DatasetType("calexp", universe.extract([]), sc)
# Store using an instance
self.factory.registerFormatter(sc, formatterTypeName)
# Retrieve using the class
f = self.factory.getFormatter(sc, self.fileDescriptor)
self.assertIsFormatter(f)
self.assertEqual(f.fileDescriptor, self.fileDescriptor)
# Retrieve using the DatasetType
f2 = self.factory.getFormatter(datasetType, self.fileDescriptor)
self.assertIsFormatter(f2)
self.assertEqual(f.name(), f2.name())
# Class directly
f2cls = self.factory.getFormatterClass(datasetType)
self.assertIsFormatter(f2cls)
# This might defer the import, pytest may have already loaded it
from lsst.daf.butler.formatters.yamlFormatter import YamlFormatter
self.assertEqual(type(f), YamlFormatter)
with self.assertRaises(KeyError):
# Attempt to overwrite using a different value
self.factory.registerFormatter(storageClassName,
"lsst.daf.butler.formatters.jsonFormatter.JsonFormatter")
def setUp(self):
self.universe = DimensionUniverse()
self.dataId = {
"instrument": "dummy",
"visit": 52,
"physical_filter": "U"
}
def testPickling(self):
# Pickling and copying should always yield the exact same object within
# a single process (cross-process is impossible to test here).
universe1 = DimensionUniverse()
universe2 = pickle.loads(pickle.dumps(universe1))
universe3 = copy.copy(universe1)
universe4 = copy.deepcopy(universe1)
self.assertIs(universe1, universe2)
self.assertIs(universe1, universe3)
self.assertIs(universe1, universe4)
for element1 in universe1.getStaticElements():
element2 = pickle.loads(pickle.dumps(element1))
self.assertIs(element1, element2)
graph1 = element1.graph
graph2 = pickle.loads(pickle.dumps(graph1))
self.assertIs(graph1, graph2)
def testMap(self):
universe = DimensionUniverse()
c = CompositesMap(self.configFile, universe=universe)
# Check that a str is not supported
with self.assertRaises(ValueError):
c.shouldBeDisassembled("fred")
# These will fail (not a composite)
sc = StorageClass("StructuredDataJson")
d = DatasetType("dummyTrue", universe.empty, sc)
self.assertFalse(sc.isComposite())
self.assertFalse(d.isComposite())
self.assertFalse(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
self.assertFalse(c.shouldBeDisassembled(sc),
f"Test with StorageClass: {sc}")
# Repeat but this time use a composite storage class
sccomp = StorageClass("Dummy")
sc = StorageClass("StructuredDataJson",
components={
"dummy": sccomp,
"dummy2": sccomp
})
d = DatasetType("dummyTrue", universe.empty, sc)
self.assertTrue(sc.isComposite())
self.assertTrue(d.isComposite())
self.assertTrue(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
self.assertFalse(c.shouldBeDisassembled(sc),
f"Test with StorageClass: {sc}")
# Override with False
d = DatasetType("dummyFalse", universe.empty, sc)
self.assertFalse(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
# DatasetType that has no explicit entry
d = DatasetType("dummyFred", universe.empty, sc)
self.assertFalse(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
# StorageClass that will be disassembled
sc = StorageClass("StructuredComposite",
components={
"dummy": sccomp,
"dummy2": sccomp
})
d = DatasetType("dummyFred", universe.empty, sc)
self.assertTrue(c.shouldBeDisassembled(d),
f"Test with DatasetType: {d}")
# Check that we are not allowed a single component in a composite
with self.assertRaises(ValueError):
StorageClass("TestSC", components={"dummy": sccomp})
def testDatasetConfig(self):
"""Test for a config with datasets
"""
config = ConfigWithDatasets()
universe = DimensionUniverse.fromConfig()
descriptors = pipeBase.PipelineTask.getInputDatasetTypes(config)
self.assertCountEqual(descriptors.keys(), ["input1", "input2"])
descriptor = descriptors["input1"]
datasetType = descriptor.makeDatasetType(universe)
self.assertEqual(datasetType.name, config.input1.name)
self.assertCountEqual(datasetType.dimensions.names,
config.input1.dimensions)
self.assertEqual(datasetType.storageClass.name,
config.input1.storageClass)
self.assertFalse(descriptor.scalar)
descriptor = descriptors["input2"]
datasetType = descriptor.makeDatasetType(universe)
self.assertEqual(datasetType.name, config.input2.name)
self.assertCountEqual(datasetType.dimensions.names,
config.input2.dimensions)
self.assertEqual(datasetType.storageClass.name,
config.input2.storageClass)
self.assertTrue(descriptor.scalar)
descriptors = pipeBase.PipelineTask.getOutputDatasetTypes(config)
self.assertCountEqual(descriptors.keys(), ["output"])
descriptor = descriptors["output"]
datasetType = descriptor.makeDatasetType(universe)
self.assertEqual(datasetType.name, config.output.name)
self.assertCountEqual(datasetType.dimensions.names,
config.output.dimensions)
self.assertEqual(datasetType.storageClass.name,
config.output.storageClass)
self.assertFalse(descriptor.scalar)
descriptors = pipeBase.PipelineTask.getInitInputDatasetTypes(config)
self.assertCountEqual(descriptors.keys(), ["initInput"])
descriptor = descriptors["initInput"]
datasetType = descriptor.makeDatasetType(universe)
self.assertEqual(datasetType.name, config.initInput.name)
self.assertEqual(len(datasetType.dimensions), 0)
self.assertEqual(datasetType.storageClass.name,
config.initInput.storageClass)
self.assertTrue(descriptor.scalar)
descriptors = pipeBase.PipelineTask.getInitOutputDatasetTypes(config)
self.assertCountEqual(descriptors.keys(), ["initOutput"])
descriptor = descriptors["initOutput"]
datasetType = descriptor.makeDatasetType(universe)
self.assertEqual(datasetType.name, config.initOutput.name)
self.assertEqual(len(datasetType.dimensions), 0)
self.assertEqual(datasetType.storageClass.name,
config.initOutput.storageClass)
self.assertTrue(descriptor.scalar)
def testAddInputsOutputs(self):
"""Test of addPredictedInput() method.
"""
quantum = Quantum(taskName="some.task.object", run=None)
# start with empty
self.assertEqual(quantum.predictedInputs, dict())
universe = DimensionUniverse()
instrument = "DummyCam"
datasetTypeName = "test_ds"
storageClass = StorageClass("testref_StructuredData")
datasetType = DatasetType(datasetTypeName,
universe.extract(("instrument", "visit")),
storageClass)
# add one ref
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42))
quantum.addPredictedInput(ref)
self.assertIn(datasetTypeName, quantum.predictedInputs)
self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 1)
# add second ref
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43))
quantum.addPredictedInput(ref)
self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 2)
# mark last ref as actually used
self.assertEqual(quantum.actualInputs, dict())
quantum._markInputUsed(ref)
self.assertIn(datasetTypeName, quantum.actualInputs)
self.assertEqual(len(quantum.actualInputs[datasetTypeName]), 1)
# add couple of outputs too
self.assertEqual(quantum.outputs, dict())
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42))
quantum.addOutput(ref)
self.assertIn(datasetTypeName, quantum.outputs)
self.assertEqual(len(quantum.outputs[datasetTypeName]), 1)
ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43))
quantum.addOutput(ref)
self.assertEqual(len(quantum.outputs[datasetTypeName]), 2)
def setUpClass(cls):
# Storage Classes are fixed for all datastores in these tests
scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml")
cls.storageClassFactory = StorageClassFactory()
cls.storageClassFactory.addFromConfig(scConfigFile)
# Read the Datastore config so we can get the class
# information (since we should not assume the constructor
# name here, but rely on the configuration file itself)
datastoreConfig = DatastoreConfig(cls.configFile)
cls.datastoreType = doImport(datastoreConfig["cls"])
cls.universe = DimensionUniverse.fromConfig()
def testConstructor(self):
"""Test of constructor.
"""
# Quantum specific arguments
taskName = "some.task.object" # can't use a real PipelineTask due to inverted package dependency
quantum = Quantum(taskName=taskName)
self.assertEqual(quantum.taskName, taskName)
self.assertEqual(quantum.initInputs, {})
self.assertEqual(quantum.inputs, NamedKeyDict())
self.assertEqual(quantum.outputs, {})
self.assertIsNone(quantum.dataId)
universe = DimensionUniverse()
instrument = "DummyCam"
datasetTypeName = "test_ds"
storageClass = StorageClass("testref_StructuredData")
datasetType = DatasetType(datasetTypeName,
universe.extract(("instrument", "visit")),
storageClass)
predictedInputs = {
datasetType: [
DatasetRef(datasetType, dict(instrument=instrument, visit=42)),
DatasetRef(datasetType, dict(instrument=instrument, visit=43))
]
}
outputs = {
datasetType: [
DatasetRef(datasetType, dict(instrument=instrument, visit=42)),
DatasetRef(datasetType, dict(instrument=instrument, visit=43))
]
}
quantum = Quantum(taskName=taskName,
inputs=predictedInputs,
outputs=outputs)
self.assertEqual(len(quantum.inputs[datasetType]), 2)
self.assertEqual(len(quantum.outputs[datasetType]), 2)
def makeDatasetType(self, universe: DimensionUniverse):
"""Construct a true `DatasetType` instance with normalized dimensions.
Parameters
----------
universe : `lsst.daf.butler.DimensionUniverse`
Set of all known dimensions to be used to normalize the dimension
names specified in config.
Returns
-------
datasetType : `DatasetType`
The `DatasetType` defined by this connection.
"""
return DatasetType(self.name, universe.extract(self.dimensions),
self.storageClass)
def setUp(self):
self.universe = DimensionUniverse()
self.fixed = ExpandedDataCoordinate(
DimensionGraph(universe=self.universe, names=["skymap"]),
values=("unimportant", ),
records={
"skymap":
self.universe["skymap"].RecordClass.fromDict({
"name": "unimportant",
"tract_max": 5,
"patch_nx_max": 3,
"patch_ny_max": 3,
})
})
def makeRegistry(self) -> Registry:
prefix = f"test_{secrets.token_hex(8).lower()}_"
self._prefixes.append(prefix)
config = RegistryConfig()
# Can't use Registry.fromConfig for these tests because we don't want
# to reconnect to the server every single time. But we at least use
# OracleDatabase.fromConnection rather than the constructor so
# we can try to pass a prefix through via "+" in a namespace.
database = OracleDatabase.fromConnection(connection=self._connection,
origin=0,
namespace=f"+{prefix}")
return Registry(database=database,
dimensions=DimensionUniverse(config),
create=True)
def setUp(self):
self.universe = DimensionUniverse()
self.fixed = DataCoordinate.fromFullValues(
DimensionGraph(universe=self.universe, names=["skymap"]),
values=("unimportant", ),
).expanded(
records={
"skymap":
self.universe["skymap"].RecordClass(
name="unimportant",
tract_max=5,
patch_nx_max=3,
patch_ny_max=3,
)
})
def testFromConfig(self):
"""Test DatasetTypeDescriptor.fromConfig()
"""
universe = DimensionUniverse.fromConfig()
config = AddConfig()
descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.input)
datasetType = descriptor.makeDatasetType(universe)
self.assertIsInstance(descriptor, pipeBase.DatasetTypeDescriptor)
self.assertEqual(datasetType.name, "add_input")
self.assertFalse(descriptor.scalar)
descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.output)
datasetType = descriptor.makeDatasetType(universe)
self.assertIsInstance(descriptor, pipeBase.DatasetTypeDescriptor)
self.assertEqual(datasetType.name, "add_output")
self.assertFalse(descriptor.scalar)
def testRegistryConfig(self):
configFile = os.path.join(TESTDIR, "config", "basic",
"posixDatastore.yaml")
config = Config(configFile)
universe = DimensionUniverse.fromConfig()
self.factory.registerFormatters(config["datastore", "formatters"],
universe=universe)
# Create a DatasetRef with and without instrument matching the
# one in the config file.
dimensions = universe.extract(
("visit", "physical_filter", "instrument"))
sc = StorageClass("DummySC", dict, None)
refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, {
"instrument": "DummyHSC",
"physical_filter": "v"
})
refPviHscFmt = self.factory.getFormatter(refPviHsc)
self.assertIsInstance(refPviHscFmt, Formatter)
self.assertIn("JsonFormatter", refPviHscFmt.name())
refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, {
"instrument": "DummyNotHSC",
"physical_filter": "v"
})
refPviNotHscFmt = self.factory.getFormatter(refPviNotHsc)
self.assertIsInstance(refPviNotHscFmt, Formatter)
self.assertIn("PickleFormatter", refPviNotHscFmt.name())
# Create a DatasetRef that should fall back to using Dimensions
refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, {
"instrument": "DummyHSC",
"physical_filter": "v"
})
refPvixNotHscFmt = self.factory.getFormatter(refPvixHsc)
self.assertIsInstance(refPvixNotHscFmt, Formatter)
self.assertIn("PickleFormatter", refPvixNotHscFmt.name())
# Create a DatasetRef that should fall back to using StorageClass
dimensionsNoV = universe.extract(("physical_filter", "instrument"))
refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, {
"instrument": "DummyHSC",
"physical_filter": "v"
})
refPvixNotHscDims_fmt = self.factory.getFormatter(refPvixNotHscDims)
self.assertIsInstance(refPvixNotHscDims_fmt, Formatter)
self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name())
def _makeQuanta(self, config):
"""Create set of Quanta"""
universe = DimensionUniverse()
connections = config.connections.ConnectionsClass(config=config)
dstype0 = connections.input.makeDatasetType(universe)
dstype1 = connections.output.makeDatasetType(universe)
quanta = []
for visit in range(100):
inputRef = self._makeDSRefVisit(dstype0, visit, universe)
outputRef = self._makeDSRefVisit(dstype1, visit, universe)
quantum = Quantum(
inputs={inputRef.datasetType: [inputRef]}, outputs={outputRef.datasetType: [outputRef]}
)
quanta.append(quantum)
return quanta
def _makeQuanta(self, config):
"""Create set of Quanta
"""
universe = DimensionUniverse()
run = Run(collection=1, environment=None, pipeline=None)
connections = config.connections.ConnectionsClass(config=config)
dstype0 = connections.input.makeDatasetType(universe)
dstype1 = connections.output.makeDatasetType(universe)
quanta = []
for visit in range(100):
quantum = Quantum(run=run)
quantum.addPredictedInput(
self._makeDSRefVisit(dstype0, visit, universe))
quantum.addOutput(self._makeDSRefVisit(dstype1, visit, universe))
quanta.append(quantum)
return quanta
def _makeQuanta(self, config):
"""Create set of Quanta
"""
universe = DimensionUniverse.fromConfig()
run = Run(collection=1, environment=None, pipeline=None)
descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.input)
dstype0 = descriptor.makeDatasetType(universe)
descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.output)
dstype1 = descriptor.makeDatasetType(universe)
quanta = []
for visit in range(100):
quantum = Quantum(run=run, task=None)
quantum.addPredictedInput(self._makeDSRefVisit(dstype0, visit))
quantum.addOutput(self._makeDSRefVisit(dstype1, visit))
quanta.append(quantum)
return quanta
def __init__(self):
self._opaque = DummyOpaqueTableStorageManager()
self.dimensions = DimensionUniverse()
self._datastoreBridges = DummyDatastoreRegistryBridgeManager(
self._opaque, self.dimensions)
def pack_data_id(self, tract, patch, band=None):
"""Pack a skymap-based data ID into an integer.
Parameters
----------
tract : `int`
Integer ID for the tract.
patch : `tuple` (`int`) or `int`
Either a 2-element (x, y) tuple (Gen2 patch ID) or a single integer
(Gen3 patch ID, corresponding to the "sequential" patch index
methods in this package).
band : `str`, optional
If provided, a filter name present in
`SkyMapDimensionPacker.SUPPORTED_FILTERS` (which is aspirationally
a list of all Gen3 'bands', but in practice may be missing some;
see RFC-785). If not provided, the packing algorithm that does
not include the filter will be used.
Returns
-------
packed : `int`
Integer that corresponds to the data ID.
max_bits : `int`
Maximum number of bits that ``packed`` could have, assuming this
skymap and presence or absence of ``band``.
Notes
-----
This method uses a Gen3 `lsst.daf.butler.DimensionPacker` object under
the hood to guarantee consistency with pure Gen3 code, but it does not
require the caller to actually have a Gen3 butler available. It does,
however, require a filter value compatible with the Gen3 "band"
dimension.
This is a temporary interface intended to aid with the migration from
Gen2 to Gen3 middleware. It will be removed with the Gen2 middleware
or when DM-31924 provides a longer-term replacement, whichever comes
first. Pure Gen3 code should use `lsst.daf.butler.DataCoordinate.pack`
or other `lsst.daf.butler.DimensionPacker` interfaces.
"""
from lsst.daf.butler import DataCoordinate, DimensionUniverse
universe = DimensionUniverse()
dummy_skymap_name = "unimportant" # only matters to Gen3 registry
tract_info = self[tract]
patch_info = tract_info[patch]
nx, ny = tract_info.getNumPatches()
skymap_record = universe["skymap"].RecordClass(
name=dummy_skymap_name,
hash=self.getSha1(),
tract_max=len(self),
patch_nx_max=
nx, # assuming these are the same for all tracts for now
patch_ny_max=ny,
)
skymap_data_id = DataCoordinate.standardize(
skymap=dummy_skymap_name,
universe=universe,
).expanded(records={"skymap": skymap_record}, )
full_data_id = DataCoordinate.standardize(
skymap=dummy_skymap_name,
tract=tract_info.getId(),
patch=tract_info.getSequentialPatchIndex(patch_info),
universe=universe,
)
if band is None:
packer = universe.makePacker("tract_patch", skymap_data_id)
else:
packer = universe.makePacker("tract_patch_band", skymap_data_id)
full_data_id = DataCoordinate.standardize(full_data_id, band=band)
return packer.pack(full_data_id, returnMaxBits=True)
def __init__(self):
self.datasets = {}
self.registry = SimpleNamespace(dimensions=DimensionUniverse())
def setUp(self):
self.universe = DimensionUniverse()
class DimensionTestCase(unittest.TestCase):
"""Tests for dimensions.
All tests here rely on the content of ``config/dimensions.yaml``, either
to test that the definitions there are read in properly or just as generic
data for testing various operations.
"""
def setUp(self):
self.universe = DimensionUniverse()
def checkGraphInvariants(self, graph):
elements = list(graph.elements)
for n, element in enumerate(elements):
# Ordered comparisons on graphs behave like sets.
self.assertLessEqual(element.graph, graph)
# Ordered comparisons on elements correspond to the ordering within
# a DimensionUniverse (topological, with deterministic
# tiebreakers).
for other in elements[:n]:
self.assertLess(other, element)
self.assertLessEqual(other, element)
for other in elements[n + 1:]:
self.assertGreater(other, element)
self.assertGreaterEqual(other, element)
if isinstance(element, Dimension):
self.assertEqual(element.graph.required, element.required)
self.assertEqual(DimensionGraph(self.universe, graph.required), graph)
self.assertCountEqual(graph.required,
[dimension for dimension in graph.dimensions
if not any(dimension in other.graph.implied for other in graph.elements)])
self.assertCountEqual(graph.implied, graph.dimensions - graph.required)
self.assertCountEqual(graph.dimensions,
[element for element in graph.elements
if isinstance(element, Dimension)])
self.assertCountEqual(graph.dimensions, itertools.chain(graph.required, graph.implied))
# Check primary key traversal order: each element should follow any it
# requires, and element that is implied by any other in the graph
# follow at least one of those.
seen = NamedValueSet()
for element in graph.primaryKeyTraversalOrder:
with self.subTest(required=graph.required, implied=graph.implied, element=element):
seen.add(element)
self.assertLessEqual(element.graph.required, seen)
if element in graph.implied:
self.assertTrue(any(element in s.implied for s in seen))
self.assertCountEqual(seen, graph.elements)
# Test encoding and decoding of DimensionGraphs to bytes.
encoded = graph.encode()
self.assertEqual(len(encoded), self.universe.getEncodeLength())
self.assertEqual(DimensionGraph.decode(encoded, universe=self.universe), graph)
def testConfigRead(self):
self.assertEqual(self.universe.dimensions.names,
{"instrument", "visit", "visit_system", "exposure", "detector",
"physical_filter", "abstract_filter", "subfilter", "calibration_label",
"skymap", "tract", "patch", "htm7", "htm9"})
def testGraphs(self):
self.checkGraphInvariants(self.universe.empty)
self.checkGraphInvariants(self.universe)
for element in self.universe.elements:
self.checkGraphInvariants(element.graph)
def testInstrumentDimensions(self):
graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit", "calibration_label"))
self.assertCountEqual(graph.dimensions.names,
("instrument", "exposure", "detector", "calibration_label",
"visit", "physical_filter", "abstract_filter", "visit_system"))
self.assertCountEqual(graph.required.names, ("instrument", "exposure", "detector",
"calibration_label", "visit"))
self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system"))
self.assertCountEqual(graph.elements.names - graph.dimensions.names,
("visit_detector_region", "visit_definition"))
def testCalibrationDimensions(self):
graph = DimensionGraph(self.universe, names=("calibration_label", "physical_filter", "detector"))
self.assertCountEqual(graph.dimensions.names,
("instrument", "detector", "calibration_label",
"physical_filter", "abstract_filter"))
self.assertCountEqual(graph.required.names, ("instrument", "detector", "calibration_label",
"physical_filter"))
self.assertCountEqual(graph.implied.names, ("abstract_filter",))
self.assertCountEqual(graph.elements.names, graph.dimensions.names)
def testObservationDimensions(self):
graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit"))
self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "visit", "exposure",
"physical_filter", "abstract_filter", "visit_system"))
self.assertCountEqual(graph.required.names, ("instrument", "detector", "exposure", "visit"))
self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system"))
self.assertCountEqual(graph.elements.names - graph.dimensions.names,
("visit_detector_region", "visit_definition"))
self.assertCountEqual(graph.spatial.names, ("visit_detector_region",))
self.assertCountEqual(graph.temporal.names, ("exposure",))
def testSkyMapDimensions(self):
graph = DimensionGraph(self.universe, names=("patch",))
self.assertCountEqual(graph.dimensions.names, ("skymap", "tract", "patch"))
self.assertCountEqual(graph.required.names, ("skymap", "tract", "patch"))
self.assertCountEqual(graph.implied.names, ())
self.assertCountEqual(graph.elements.names, graph.dimensions.names)
self.assertCountEqual(graph.spatial.names, ("patch",))
def testSubsetCalculation(self):
"""Test that independent spatial and temporal options are computed
correctly.
"""
graph = DimensionGraph(self.universe, names=("visit", "detector", "tract", "patch", "htm7",
"exposure", "calibration_label"))
self.assertCountEqual(graph.spatial.names,
("visit_detector_region", "patch", "htm7"))
self.assertCountEqual(graph.temporal.names,
("exposure", "calibration_label"))
def testSchemaGeneration(self):
tableSpecs = NamedKeyDict({})
for element in self.universe.elements:
if element.hasTable and element.viewOf is None:
tableSpecs[element] = element.RecordClass.fields.makeTableSpec(
tsRepr=DatabaseTimespanRepresentation.Compound
)
for element, tableSpec in tableSpecs.items():
for dep in element.required:
with self.subTest(element=element.name, dep=dep.name):
if dep != element:
self.assertIn(dep.name, tableSpec.fields)
self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype)
self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length)
self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes)
self.assertFalse(tableSpec.fields[dep.name].nullable)
self.assertTrue(tableSpec.fields[dep.name].primaryKey)
else:
self.assertIn(element.primaryKey.name, tableSpec.fields)
self.assertEqual(tableSpec.fields[element.primaryKey.name].dtype,
dep.primaryKey.dtype)
self.assertEqual(tableSpec.fields[element.primaryKey.name].length,
dep.primaryKey.length)
self.assertEqual(tableSpec.fields[element.primaryKey.name].nbytes,
dep.primaryKey.nbytes)
self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable)
self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey)
for dep in element.implied:
with self.subTest(element=element.name, dep=dep.name):
self.assertIn(dep.name, tableSpec.fields)
self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype)
self.assertFalse(tableSpec.fields[dep.name].primaryKey)
for foreignKey in tableSpec.foreignKeys:
self.assertIn(foreignKey.table, tableSpecs)
self.assertIn(foreignKey.table, element.graph.dimensions.names)
self.assertEqual(len(foreignKey.source), len(foreignKey.target))
for source, target in zip(foreignKey.source, foreignKey.target):
self.assertIn(source, tableSpec.fields.names)
self.assertIn(target, tableSpecs[foreignKey.table].fields.names)
self.assertEqual(tableSpec.fields[source].dtype,
tableSpecs[foreignKey.table].fields[target].dtype)
self.assertEqual(tableSpec.fields[source].length,
tableSpecs[foreignKey.table].fields[target].length)
self.assertEqual(tableSpec.fields[source].nbytes,
tableSpecs[foreignKey.table].fields[target].nbytes)
def testPickling(self):
# Pickling and copying should always yield the exact same object within
# a single process (cross-process is impossible to test here).
universe1 = DimensionUniverse()
universe2 = pickle.loads(pickle.dumps(universe1))
universe3 = copy.copy(universe1)
universe4 = copy.deepcopy(universe1)
self.assertIs(universe1, universe2)
self.assertIs(universe1, universe3)
self.assertIs(universe1, universe4)
for element1 in universe1.elements:
element2 = pickle.loads(pickle.dumps(element1))
self.assertIs(element1, element2)
graph1 = element1.graph
graph2 = pickle.loads(pickle.dumps(graph1))
self.assertIs(graph1, graph2)
class ConstraintsTestCase(unittest.TestCase, DatasetTestHelper):
def setUp(self):
self.id = 0
# Create DatasetRefs to test against constraints model
self.universe = DimensionUniverse()
dimensions = self.universe.extract(
("visit", "physical_filter", "instrument"))
sc = StorageClass("DummySC", dict, None)
self.calexpA = self.makeDatasetRef("calexp",
dimensions,
sc, {
"instrument": "A",
"physical_filter": "u"
},
conform=False)
dimensions = self.universe.extract(("visit", "detector", "instrument"))
self.pviA = self.makeDatasetRef("pvi",
dimensions,
sc, {
"instrument": "A",
"visit": 1
},
conform=False)
self.pviB = self.makeDatasetRef("pvi",
dimensions,
sc, {
"instrument": "B",
"visit": 2
},
conform=False)
def testSimpleAccept(self):
config = ConstraintsConfig({"accept": ["calexp", "ExposureF"]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
# Dimension accept
config = ConstraintsConfig(
{"accept": ["visit+physical_filter+instrument", "ExposureF"]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
config = ConstraintsConfig(
{"accept": ["visit+detector+instrument", "ExposureF"]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Only accept instrument A pvi
config = ConstraintsConfig({"accept": [{"instrument<A>": ["pvi"]}]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertFalse(constraints.isAcceptable(self.pviB))
# Accept PVI for instrument B but not instrument A
config = ConstraintsConfig(
{"accept": ["calexp", {
"instrument<B>": ["pvi"]
}]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
self.assertTrue(constraints.isAcceptable(self.pviB))
def testSimpleReject(self):
config = ConstraintsConfig({"reject": ["calexp", "ExposureF"]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
def testAcceptReject(self):
# Reject everything except calexp
config = ConstraintsConfig({"accept": ["calexp"], "reject": ["all"]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
# Accept everything except calexp
config = ConstraintsConfig({"reject": ["calexp"], "accept": ["all"]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Reject pvi but explicitly accept pvi for instrument A
# Reject all instrument A but accept everything else
# The reject here is superfluous
config = ConstraintsConfig({
"accept": [{
"instrument<A>": ["pvi"]
}],
"reject": ["pvi"]
})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertFalse(constraints.isAcceptable(self.pviB))
# Accept everything except pvi from other than instrument A
config = ConstraintsConfig({
"accept": ["all", {
"instrument<A>": ["pvi"]
}],
"reject": ["pvi"]
})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertFalse(constraints.isAcceptable(self.pviB))
def testWildcardReject(self):
# Reject everything
config = ConstraintsConfig({"reject": ["all"]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
# Reject all instrument A but accept everything else
config = ConstraintsConfig({"reject": [{"instrument<A>": ["all"]}]})
constraints = Constraints(config, universe=self.universe)
self.assertFalse(constraints.isAcceptable(self.calexpA))
self.assertFalse(constraints.isAcceptable(self.pviA))
self.assertTrue(constraints.isAcceptable(self.pviB))
def testWildcardAccept(self):
# Accept everything
config = ConstraintsConfig({})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Accept everything
constraints = Constraints(None, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Accept everything explicitly
config = ConstraintsConfig({"accept": ["all"]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
# Accept all instrument A but reject everything else
config = ConstraintsConfig({"accept": [{"instrument<A>": ["all"]}]})
constraints = Constraints(config, universe=self.universe)
self.assertTrue(constraints.isAcceptable(self.calexpA))
self.assertTrue(constraints.isAcceptable(self.pviA))
self.assertFalse(constraints.isAcceptable(self.pviB))
def testEdgeCases(self):
# Accept everything and reject everything
config = ConstraintsConfig({"accept": ["all"], "reject": ["all"]})
with self.assertRaises(ValidationError):
Constraints(config, universe=self.universe)
def setUp(self):
self.universe = DimensionUniverse.fromConfig()
self.given = DimensionSet(universe=self.universe, elements=["skymap"])
self.parameters = dict(skymap="unimportant", tractMax=5, patchNxMax=3, patchNyMax=3)
