def setUp(self): self.id = 0 # Create DatasetRefs to test against constraints model self.universe = DimensionUniverse() dimensions = self.universe.extract( ("visit", "physical_filter", "instrument")) sc = StorageClass("DummySC", dict, None) self.calexpA = self.makeDatasetRef("calexp", dimensions, sc, { "instrument": "A", "physical_filter": "u" }, conform=False) dimensions = self.universe.extract(("visit", "detector", "instrument")) self.pviA = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "A", "visit": 1 }, conform=False) self.pviB = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "B", "visit": 2 }, conform=False)
def setUp(self): self.universe = DimensionUniverse() datasetTypeName = "test" self.componentStorageClass1 = StorageClass("Component1") self.componentStorageClass2 = StorageClass("Component2") self.parentStorageClass = StorageClass("Parent", components={"a": self.componentStorageClass1, "b": self.componentStorageClass2}) dimensions = self.universe.extract(("instrument", "visit")) self.dataId = dict(instrument="DummyCam", visit=42) self.datasetType = DatasetType(datasetTypeName, dimensions, self.parentStorageClass)
def setUp(self): self.id = 0 self.factory = FormatterFactory() self.universe = DimensionUniverse() self.dataId = DataCoordinate.makeEmpty(self.universe) # Dummy FileDescriptor for testing getFormatter self.fileDescriptor = FileDescriptor( Location("/a/b/c", "d"), StorageClass("DummyStorageClass", dict, None))
def makeRegistry(self) -> Registry: prefix = f"test_{secrets.token_hex(8).lower()}_" self._prefixes.append(prefix) config = self.makeRegistryConfig() # Can't use Registry.fromConfig for these tests because we don't want # to reconnect to the server every single time. But we at least use # OracleDatabase.fromConnection rather than the constructor so # we can try to pass a prefix through via "+" in a namespace. database = OracleDatabase.fromConnection(connection=self._connection, origin=0, namespace=f"+{prefix}") attributes = doImport(config["managers", "attributes"]) opaque = doImport(config["managers", "opaque"]) dimensions = doImport(config["managers", "dimensions"]) collections = doImport(config["managers", "collections"]) datasets = doImport(config["managers", "datasets"]) datastoreBridges = doImport(config["managers", "datastores"]) return Registry(database=database, attributes=attributes, opaque=opaque, dimensions=dimensions, collections=collections, datasets=datasets, datastoreBridges=datastoreBridges, universe=DimensionUniverse(config), create=True)
def testConstructor(self): """Test DatasetTypeDescriptor init """ name = "testDataset" dimensionNames = frozenset(["label"]) storageClassName = "Catalog" universe = DimensionUniverse.fromConfig() descriptor = pipeBase.DatasetTypeDescriptor( name=name, dimensionNames=dimensionNames, storageClassName=storageClassName, scalar=False, manualLoad=False) datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, name) self.assertEqual(datasetType.dimensions.names, dimensionNames) self.assertEqual(datasetType.storageClass.name, storageClassName) self.assertFalse(descriptor.scalar) descriptor = pipeBase.DatasetTypeDescriptor( name=name, dimensionNames=dimensionNames, storageClassName=storageClassName, scalar=True, manualLoad=False) datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, name) self.assertEqual(datasetType.dimensions.names, dimensionNames) self.assertEqual(datasetType.storageClass.name, storageClassName) self.assertTrue(descriptor.scalar)
def setUp(self): self.universe = DimensionUniverse.fromConfig() self.given = DimensionSet(universe=self.universe, elements=["skymap"]) self.parameters = dict(skymap="unimportant", tractMax=5, patchNxMax=3, patchNyMax=3)
def makeDatasetType( self, universe: DimensionUniverse, parentStorageClass: Optional[Union[StorageClass, str]] = None ) -> DatasetType: """Construct a true `DatasetType` instance with normalized dimensions. Parameters ---------- universe : `lsst.daf.butler.DimensionUniverse` Set of all known dimensions to be used to normalize the dimension names specified in config. parentStorageClass : `lsst.daf.butler.StorageClass` or `str`, optional Parent storage class for component datasets; `None` otherwise. Returns ------- datasetType : `DatasetType` The `DatasetType` defined by this connection. """ return DatasetType( self.name, universe.extract(self.dimensions), self.storageClass, isCalibration=self.isCalibration, parentStorageClass=parentStorageClass, )
def testRegistryWithStorageClass(self): """Test that the registry can be given a StorageClass object. """ formatterTypeName = "lsst.daf.butler.formatters.yamlFormatter.YamlFormatter" storageClassName = "TestClass" sc = StorageClass(storageClassName, dict, None) universe = DimensionUniverse.fromConfig() datasetType = DatasetType("calexp", universe.extract([]), sc) # Store using an instance self.factory.registerFormatter(sc, formatterTypeName) # Retrieve using the class f = self.factory.getFormatter(sc, self.fileDescriptor) self.assertIsFormatter(f) self.assertEqual(f.fileDescriptor, self.fileDescriptor) # Retrieve using the DatasetType f2 = self.factory.getFormatter(datasetType, self.fileDescriptor) self.assertIsFormatter(f2) self.assertEqual(f.name(), f2.name()) # Class directly f2cls = self.factory.getFormatterClass(datasetType) self.assertIsFormatter(f2cls) # This might defer the import, pytest may have already loaded it from lsst.daf.butler.formatters.yamlFormatter import YamlFormatter self.assertEqual(type(f), YamlFormatter) with self.assertRaises(KeyError): # Attempt to overwrite using a different value self.factory.registerFormatter(storageClassName, "lsst.daf.butler.formatters.jsonFormatter.JsonFormatter")
def setUp(self): self.universe = DimensionUniverse() self.dataId = { "instrument": "dummy", "visit": 52, "physical_filter": "U" }
def testPickling(self): # Pickling and copying should always yield the exact same object within # a single process (cross-process is impossible to test here). universe1 = DimensionUniverse() universe2 = pickle.loads(pickle.dumps(universe1)) universe3 = copy.copy(universe1) universe4 = copy.deepcopy(universe1) self.assertIs(universe1, universe2) self.assertIs(universe1, universe3) self.assertIs(universe1, universe4) for element1 in universe1.getStaticElements(): element2 = pickle.loads(pickle.dumps(element1)) self.assertIs(element1, element2) graph1 = element1.graph graph2 = pickle.loads(pickle.dumps(graph1)) self.assertIs(graph1, graph2)
def testMap(self): universe = DimensionUniverse() c = CompositesMap(self.configFile, universe=universe) # Check that a str is not supported with self.assertRaises(ValueError): c.shouldBeDisassembled("fred") # These will fail (not a composite) sc = StorageClass("StructuredDataJson") d = DatasetType("dummyTrue", universe.empty, sc) self.assertFalse(sc.isComposite()) self.assertFalse(d.isComposite()) self.assertFalse(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}") self.assertFalse(c.shouldBeDisassembled(sc), f"Test with StorageClass: {sc}") # Repeat but this time use a composite storage class sccomp = StorageClass("Dummy") sc = StorageClass("StructuredDataJson", components={ "dummy": sccomp, "dummy2": sccomp }) d = DatasetType("dummyTrue", universe.empty, sc) self.assertTrue(sc.isComposite()) self.assertTrue(d.isComposite()) self.assertTrue(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}") self.assertFalse(c.shouldBeDisassembled(sc), f"Test with StorageClass: {sc}") # Override with False d = DatasetType("dummyFalse", universe.empty, sc) self.assertFalse(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}") # DatasetType that has no explicit entry d = DatasetType("dummyFred", universe.empty, sc) self.assertFalse(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}") # StorageClass that will be disassembled sc = StorageClass("StructuredComposite", components={ "dummy": sccomp, "dummy2": sccomp }) d = DatasetType("dummyFred", universe.empty, sc) self.assertTrue(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}") # Check that we are not allowed a single component in a composite with self.assertRaises(ValueError): StorageClass("TestSC", components={"dummy": sccomp})
def testDatasetConfig(self): """Test for a config with datasets """ config = ConfigWithDatasets() universe = DimensionUniverse.fromConfig() descriptors = pipeBase.PipelineTask.getInputDatasetTypes(config) self.assertCountEqual(descriptors.keys(), ["input1", "input2"]) descriptor = descriptors["input1"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.input1.name) self.assertCountEqual(datasetType.dimensions.names, config.input1.dimensions) self.assertEqual(datasetType.storageClass.name, config.input1.storageClass) self.assertFalse(descriptor.scalar) descriptor = descriptors["input2"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.input2.name) self.assertCountEqual(datasetType.dimensions.names, config.input2.dimensions) self.assertEqual(datasetType.storageClass.name, config.input2.storageClass) self.assertTrue(descriptor.scalar) descriptors = pipeBase.PipelineTask.getOutputDatasetTypes(config) self.assertCountEqual(descriptors.keys(), ["output"]) descriptor = descriptors["output"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.output.name) self.assertCountEqual(datasetType.dimensions.names, config.output.dimensions) self.assertEqual(datasetType.storageClass.name, config.output.storageClass) self.assertFalse(descriptor.scalar) descriptors = pipeBase.PipelineTask.getInitInputDatasetTypes(config) self.assertCountEqual(descriptors.keys(), ["initInput"]) descriptor = descriptors["initInput"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.initInput.name) self.assertEqual(len(datasetType.dimensions), 0) self.assertEqual(datasetType.storageClass.name, config.initInput.storageClass) self.assertTrue(descriptor.scalar) descriptors = pipeBase.PipelineTask.getInitOutputDatasetTypes(config) self.assertCountEqual(descriptors.keys(), ["initOutput"]) descriptor = descriptors["initOutput"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.initOutput.name) self.assertEqual(len(datasetType.dimensions), 0) self.assertEqual(datasetType.storageClass.name, config.initOutput.storageClass) self.assertTrue(descriptor.scalar)
def testAddInputsOutputs(self): """Test of addPredictedInput() method. """ quantum = Quantum(taskName="some.task.object", run=None) # start with empty self.assertEqual(quantum.predictedInputs, dict()) universe = DimensionUniverse() instrument = "DummyCam" datasetTypeName = "test_ds" storageClass = StorageClass("testref_StructuredData") datasetType = DatasetType(datasetTypeName, universe.extract(("instrument", "visit")), storageClass) # add one ref ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42)) quantum.addPredictedInput(ref) self.assertIn(datasetTypeName, quantum.predictedInputs) self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 1) # add second ref ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43)) quantum.addPredictedInput(ref) self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 2) # mark last ref as actually used self.assertEqual(quantum.actualInputs, dict()) quantum._markInputUsed(ref) self.assertIn(datasetTypeName, quantum.actualInputs) self.assertEqual(len(quantum.actualInputs[datasetTypeName]), 1) # add couple of outputs too self.assertEqual(quantum.outputs, dict()) ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42)) quantum.addOutput(ref) self.assertIn(datasetTypeName, quantum.outputs) self.assertEqual(len(quantum.outputs[datasetTypeName]), 1) ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43)) quantum.addOutput(ref) self.assertEqual(len(quantum.outputs[datasetTypeName]), 2)
def setUpClass(cls): # Storage Classes are fixed for all datastores in these tests scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml") cls.storageClassFactory = StorageClassFactory() cls.storageClassFactory.addFromConfig(scConfigFile) # Read the Datastore config so we can get the class # information (since we should not assume the constructor # name here, but rely on the configuration file itself) datastoreConfig = DatastoreConfig(cls.configFile) cls.datastoreType = doImport(datastoreConfig["cls"]) cls.universe = DimensionUniverse.fromConfig()
def testConstructor(self): """Test of constructor. """ # Quantum specific arguments taskName = "some.task.object" # can't use a real PipelineTask due to inverted package dependency quantum = Quantum(taskName=taskName) self.assertEqual(quantum.taskName, taskName) self.assertEqual(quantum.initInputs, {}) self.assertEqual(quantum.inputs, NamedKeyDict()) self.assertEqual(quantum.outputs, {}) self.assertIsNone(quantum.dataId) universe = DimensionUniverse() instrument = "DummyCam" datasetTypeName = "test_ds" storageClass = StorageClass("testref_StructuredData") datasetType = DatasetType(datasetTypeName, universe.extract(("instrument", "visit")), storageClass) predictedInputs = { datasetType: [ DatasetRef(datasetType, dict(instrument=instrument, visit=42)), DatasetRef(datasetType, dict(instrument=instrument, visit=43)) ] } outputs = { datasetType: [ DatasetRef(datasetType, dict(instrument=instrument, visit=42)), DatasetRef(datasetType, dict(instrument=instrument, visit=43)) ] } quantum = Quantum(taskName=taskName, inputs=predictedInputs, outputs=outputs) self.assertEqual(len(quantum.inputs[datasetType]), 2) self.assertEqual(len(quantum.outputs[datasetType]), 2)
def makeDatasetType(self, universe: DimensionUniverse): """Construct a true `DatasetType` instance with normalized dimensions. Parameters ---------- universe : `lsst.daf.butler.DimensionUniverse` Set of all known dimensions to be used to normalize the dimension names specified in config. Returns ------- datasetType : `DatasetType` The `DatasetType` defined by this connection. """ return DatasetType(self.name, universe.extract(self.dimensions), self.storageClass)
def setUp(self): self.universe = DimensionUniverse() self.fixed = ExpandedDataCoordinate( DimensionGraph(universe=self.universe, names=["skymap"]), values=("unimportant", ), records={ "skymap": self.universe["skymap"].RecordClass.fromDict({ "name": "unimportant", "tract_max": 5, "patch_nx_max": 3, "patch_ny_max": 3, }) })
def makeRegistry(self) -> Registry: prefix = f"test_{secrets.token_hex(8).lower()}_" self._prefixes.append(prefix) config = RegistryConfig() # Can't use Registry.fromConfig for these tests because we don't want # to reconnect to the server every single time. But we at least use # OracleDatabase.fromConnection rather than the constructor so # we can try to pass a prefix through via "+" in a namespace. database = OracleDatabase.fromConnection(connection=self._connection, origin=0, namespace=f"+{prefix}") return Registry(database=database, dimensions=DimensionUniverse(config), create=True)
def setUp(self): self.universe = DimensionUniverse() self.fixed = DataCoordinate.fromFullValues( DimensionGraph(universe=self.universe, names=["skymap"]), values=("unimportant", ), ).expanded( records={ "skymap": self.universe["skymap"].RecordClass( name="unimportant", tract_max=5, patch_nx_max=3, patch_ny_max=3, ) })
def testFromConfig(self): """Test DatasetTypeDescriptor.fromConfig() """ universe = DimensionUniverse.fromConfig() config = AddConfig() descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.input) datasetType = descriptor.makeDatasetType(universe) self.assertIsInstance(descriptor, pipeBase.DatasetTypeDescriptor) self.assertEqual(datasetType.name, "add_input") self.assertFalse(descriptor.scalar) descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.output) datasetType = descriptor.makeDatasetType(universe) self.assertIsInstance(descriptor, pipeBase.DatasetTypeDescriptor) self.assertEqual(datasetType.name, "add_output") self.assertFalse(descriptor.scalar)
def testRegistryConfig(self): configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml") config = Config(configFile) universe = DimensionUniverse.fromConfig() self.factory.registerFormatters(config["datastore", "formatters"], universe=universe) # Create a DatasetRef with and without instrument matching the # one in the config file. dimensions = universe.extract( ("visit", "physical_filter", "instrument")) sc = StorageClass("DummySC", dict, None) refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "DummyHSC", "physical_filter": "v" }) refPviHscFmt = self.factory.getFormatter(refPviHsc) self.assertIsInstance(refPviHscFmt, Formatter) self.assertIn("JsonFormatter", refPviHscFmt.name()) refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "DummyNotHSC", "physical_filter": "v" }) refPviNotHscFmt = self.factory.getFormatter(refPviNotHsc) self.assertIsInstance(refPviNotHscFmt, Formatter) self.assertIn("PickleFormatter", refPviNotHscFmt.name()) # Create a DatasetRef that should fall back to using Dimensions refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, { "instrument": "DummyHSC", "physical_filter": "v" }) refPvixNotHscFmt = self.factory.getFormatter(refPvixHsc) self.assertIsInstance(refPvixNotHscFmt, Formatter) self.assertIn("PickleFormatter", refPvixNotHscFmt.name()) # Create a DatasetRef that should fall back to using StorageClass dimensionsNoV = universe.extract(("physical_filter", "instrument")) refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, { "instrument": "DummyHSC", "physical_filter": "v" }) refPvixNotHscDims_fmt = self.factory.getFormatter(refPvixNotHscDims) self.assertIsInstance(refPvixNotHscDims_fmt, Formatter) self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name())
def _makeQuanta(self, config): """Create set of Quanta""" universe = DimensionUniverse() connections = config.connections.ConnectionsClass(config=config) dstype0 = connections.input.makeDatasetType(universe) dstype1 = connections.output.makeDatasetType(universe) quanta = [] for visit in range(100): inputRef = self._makeDSRefVisit(dstype0, visit, universe) outputRef = self._makeDSRefVisit(dstype1, visit, universe) quantum = Quantum( inputs={inputRef.datasetType: [inputRef]}, outputs={outputRef.datasetType: [outputRef]} ) quanta.append(quantum) return quanta
def _makeQuanta(self, config): """Create set of Quanta """ universe = DimensionUniverse() run = Run(collection=1, environment=None, pipeline=None) connections = config.connections.ConnectionsClass(config=config) dstype0 = connections.input.makeDatasetType(universe) dstype1 = connections.output.makeDatasetType(universe) quanta = [] for visit in range(100): quantum = Quantum(run=run) quantum.addPredictedInput( self._makeDSRefVisit(dstype0, visit, universe)) quantum.addOutput(self._makeDSRefVisit(dstype1, visit, universe)) quanta.append(quantum) return quanta
def _makeQuanta(self, config): """Create set of Quanta """ universe = DimensionUniverse.fromConfig() run = Run(collection=1, environment=None, pipeline=None) descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.input) dstype0 = descriptor.makeDatasetType(universe) descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.output) dstype1 = descriptor.makeDatasetType(universe) quanta = [] for visit in range(100): quantum = Quantum(run=run, task=None) quantum.addPredictedInput(self._makeDSRefVisit(dstype0, visit)) quantum.addOutput(self._makeDSRefVisit(dstype1, visit)) quanta.append(quantum) return quanta
def __init__(self): self._opaque = DummyOpaqueTableStorageManager() self.dimensions = DimensionUniverse() self._datastoreBridges = DummyDatastoreRegistryBridgeManager( self._opaque, self.dimensions)
def pack_data_id(self, tract, patch, band=None): """Pack a skymap-based data ID into an integer. Parameters ---------- tract : `int` Integer ID for the tract. patch : `tuple` (`int`) or `int` Either a 2-element (x, y) tuple (Gen2 patch ID) or a single integer (Gen3 patch ID, corresponding to the "sequential" patch index methods in this package). band : `str`, optional If provided, a filter name present in `SkyMapDimensionPacker.SUPPORTED_FILTERS` (which is aspirationally a list of all Gen3 'bands', but in practice may be missing some; see RFC-785). If not provided, the packing algorithm that does not include the filter will be used. Returns ------- packed : `int` Integer that corresponds to the data ID. max_bits : `int` Maximum number of bits that ``packed`` could have, assuming this skymap and presence or absence of ``band``. Notes ----- This method uses a Gen3 `lsst.daf.butler.DimensionPacker` object under the hood to guarantee consistency with pure Gen3 code, but it does not require the caller to actually have a Gen3 butler available. It does, however, require a filter value compatible with the Gen3 "band" dimension. This is a temporary interface intended to aid with the migration from Gen2 to Gen3 middleware. It will be removed with the Gen2 middleware or when DM-31924 provides a longer-term replacement, whichever comes first. Pure Gen3 code should use `lsst.daf.butler.DataCoordinate.pack` or other `lsst.daf.butler.DimensionPacker` interfaces. """ from lsst.daf.butler import DataCoordinate, DimensionUniverse universe = DimensionUniverse() dummy_skymap_name = "unimportant" # only matters to Gen3 registry tract_info = self[tract] patch_info = tract_info[patch] nx, ny = tract_info.getNumPatches() skymap_record = universe["skymap"].RecordClass( name=dummy_skymap_name, hash=self.getSha1(), tract_max=len(self), patch_nx_max= nx, # assuming these are the same for all tracts for now patch_ny_max=ny, ) skymap_data_id = DataCoordinate.standardize( skymap=dummy_skymap_name, universe=universe, ).expanded(records={"skymap": skymap_record}, ) full_data_id = DataCoordinate.standardize( skymap=dummy_skymap_name, tract=tract_info.getId(), patch=tract_info.getSequentialPatchIndex(patch_info), universe=universe, ) if band is None: packer = universe.makePacker("tract_patch", skymap_data_id) else: packer = universe.makePacker("tract_patch_band", skymap_data_id) full_data_id = DataCoordinate.standardize(full_data_id, band=band) return packer.pack(full_data_id, returnMaxBits=True)
def __init__(self): self.datasets = {} self.registry = SimpleNamespace(dimensions=DimensionUniverse())
def setUp(self): self.universe = DimensionUniverse()
class DimensionTestCase(unittest.TestCase): """Tests for dimensions. All tests here rely on the content of ``config/dimensions.yaml``, either to test that the definitions there are read in properly or just as generic data for testing various operations. """ def setUp(self): self.universe = DimensionUniverse() def checkGraphInvariants(self, graph): elements = list(graph.elements) for n, element in enumerate(elements): # Ordered comparisons on graphs behave like sets. self.assertLessEqual(element.graph, graph) # Ordered comparisons on elements correspond to the ordering within # a DimensionUniverse (topological, with deterministic # tiebreakers). for other in elements[:n]: self.assertLess(other, element) self.assertLessEqual(other, element) for other in elements[n + 1:]: self.assertGreater(other, element) self.assertGreaterEqual(other, element) if isinstance(element, Dimension): self.assertEqual(element.graph.required, element.required) self.assertEqual(DimensionGraph(self.universe, graph.required), graph) self.assertCountEqual(graph.required, [dimension for dimension in graph.dimensions if not any(dimension in other.graph.implied for other in graph.elements)]) self.assertCountEqual(graph.implied, graph.dimensions - graph.required) self.assertCountEqual(graph.dimensions, [element for element in graph.elements if isinstance(element, Dimension)]) self.assertCountEqual(graph.dimensions, itertools.chain(graph.required, graph.implied)) # Check primary key traversal order: each element should follow any it # requires, and element that is implied by any other in the graph # follow at least one of those. seen = NamedValueSet() for element in graph.primaryKeyTraversalOrder: with self.subTest(required=graph.required, implied=graph.implied, element=element): seen.add(element) self.assertLessEqual(element.graph.required, seen) if element in graph.implied: self.assertTrue(any(element in s.implied for s in seen)) self.assertCountEqual(seen, graph.elements) # Test encoding and decoding of DimensionGraphs to bytes. encoded = graph.encode() self.assertEqual(len(encoded), self.universe.getEncodeLength()) self.assertEqual(DimensionGraph.decode(encoded, universe=self.universe), graph) def testConfigRead(self): self.assertEqual(self.universe.dimensions.names, {"instrument", "visit", "visit_system", "exposure", "detector", "physical_filter", "abstract_filter", "subfilter", "calibration_label", "skymap", "tract", "patch", "htm7", "htm9"}) def testGraphs(self): self.checkGraphInvariants(self.universe.empty) self.checkGraphInvariants(self.universe) for element in self.universe.elements: self.checkGraphInvariants(element.graph) def testInstrumentDimensions(self): graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit", "calibration_label")) self.assertCountEqual(graph.dimensions.names, ("instrument", "exposure", "detector", "calibration_label", "visit", "physical_filter", "abstract_filter", "visit_system")) self.assertCountEqual(graph.required.names, ("instrument", "exposure", "detector", "calibration_label", "visit")) self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system")) self.assertCountEqual(graph.elements.names - graph.dimensions.names, ("visit_detector_region", "visit_definition")) def testCalibrationDimensions(self): graph = DimensionGraph(self.universe, names=("calibration_label", "physical_filter", "detector")) self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "calibration_label", "physical_filter", "abstract_filter")) self.assertCountEqual(graph.required.names, ("instrument", "detector", "calibration_label", "physical_filter")) self.assertCountEqual(graph.implied.names, ("abstract_filter",)) self.assertCountEqual(graph.elements.names, graph.dimensions.names) def testObservationDimensions(self): graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit")) self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "visit", "exposure", "physical_filter", "abstract_filter", "visit_system")) self.assertCountEqual(graph.required.names, ("instrument", "detector", "exposure", "visit")) self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system")) self.assertCountEqual(graph.elements.names - graph.dimensions.names, ("visit_detector_region", "visit_definition")) self.assertCountEqual(graph.spatial.names, ("visit_detector_region",)) self.assertCountEqual(graph.temporal.names, ("exposure",)) def testSkyMapDimensions(self): graph = DimensionGraph(self.universe, names=("patch",)) self.assertCountEqual(graph.dimensions.names, ("skymap", "tract", "patch")) self.assertCountEqual(graph.required.names, ("skymap", "tract", "patch")) self.assertCountEqual(graph.implied.names, ()) self.assertCountEqual(graph.elements.names, graph.dimensions.names) self.assertCountEqual(graph.spatial.names, ("patch",)) def testSubsetCalculation(self): """Test that independent spatial and temporal options are computed correctly. """ graph = DimensionGraph(self.universe, names=("visit", "detector", "tract", "patch", "htm7", "exposure", "calibration_label")) self.assertCountEqual(graph.spatial.names, ("visit_detector_region", "patch", "htm7")) self.assertCountEqual(graph.temporal.names, ("exposure", "calibration_label")) def testSchemaGeneration(self): tableSpecs = NamedKeyDict({}) for element in self.universe.elements: if element.hasTable and element.viewOf is None: tableSpecs[element] = element.RecordClass.fields.makeTableSpec( tsRepr=DatabaseTimespanRepresentation.Compound ) for element, tableSpec in tableSpecs.items(): for dep in element.required: with self.subTest(element=element.name, dep=dep.name): if dep != element: self.assertIn(dep.name, tableSpec.fields) self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length) self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes) self.assertFalse(tableSpec.fields[dep.name].nullable) self.assertTrue(tableSpec.fields[dep.name].primaryKey) else: self.assertIn(element.primaryKey.name, tableSpec.fields) self.assertEqual(tableSpec.fields[element.primaryKey.name].dtype, dep.primaryKey.dtype) self.assertEqual(tableSpec.fields[element.primaryKey.name].length, dep.primaryKey.length) self.assertEqual(tableSpec.fields[element.primaryKey.name].nbytes, dep.primaryKey.nbytes) self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable) self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey) for dep in element.implied: with self.subTest(element=element.name, dep=dep.name): self.assertIn(dep.name, tableSpec.fields) self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) self.assertFalse(tableSpec.fields[dep.name].primaryKey) for foreignKey in tableSpec.foreignKeys: self.assertIn(foreignKey.table, tableSpecs) self.assertIn(foreignKey.table, element.graph.dimensions.names) self.assertEqual(len(foreignKey.source), len(foreignKey.target)) for source, target in zip(foreignKey.source, foreignKey.target): self.assertIn(source, tableSpec.fields.names) self.assertIn(target, tableSpecs[foreignKey.table].fields.names) self.assertEqual(tableSpec.fields[source].dtype, tableSpecs[foreignKey.table].fields[target].dtype) self.assertEqual(tableSpec.fields[source].length, tableSpecs[foreignKey.table].fields[target].length) self.assertEqual(tableSpec.fields[source].nbytes, tableSpecs[foreignKey.table].fields[target].nbytes) def testPickling(self): # Pickling and copying should always yield the exact same object within # a single process (cross-process is impossible to test here). universe1 = DimensionUniverse() universe2 = pickle.loads(pickle.dumps(universe1)) universe3 = copy.copy(universe1) universe4 = copy.deepcopy(universe1) self.assertIs(universe1, universe2) self.assertIs(universe1, universe3) self.assertIs(universe1, universe4) for element1 in universe1.elements: element2 = pickle.loads(pickle.dumps(element1)) self.assertIs(element1, element2) graph1 = element1.graph graph2 = pickle.loads(pickle.dumps(graph1)) self.assertIs(graph1, graph2)
class ConstraintsTestCase(unittest.TestCase, DatasetTestHelper): def setUp(self): self.id = 0 # Create DatasetRefs to test against constraints model self.universe = DimensionUniverse() dimensions = self.universe.extract( ("visit", "physical_filter", "instrument")) sc = StorageClass("DummySC", dict, None) self.calexpA = self.makeDatasetRef("calexp", dimensions, sc, { "instrument": "A", "physical_filter": "u" }, conform=False) dimensions = self.universe.extract(("visit", "detector", "instrument")) self.pviA = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "A", "visit": 1 }, conform=False) self.pviB = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "B", "visit": 2 }, conform=False) def testSimpleAccept(self): config = ConstraintsConfig({"accept": ["calexp", "ExposureF"]}) constraints = Constraints(config, universe=self.universe) self.assertTrue(constraints.isAcceptable(self.calexpA)) self.assertFalse(constraints.isAcceptable(self.pviA)) # Dimension accept config = ConstraintsConfig( {"accept": ["visit+physical_filter+instrument", "ExposureF"]}) constraints = Constraints(config, universe=self.universe) self.assertTrue(constraints.isAcceptable(self.calexpA)) self.assertFalse(constraints.isAcceptable(self.pviA)) config = ConstraintsConfig( {"accept": ["visit+detector+instrument", "ExposureF"]}) constraints = Constraints(config, universe=self.universe) self.assertFalse(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) self.assertTrue(constraints.isAcceptable(self.pviA)) # Only accept instrument A pvi config = ConstraintsConfig({"accept": [{"instrument<A>": ["pvi"]}]}) constraints = Constraints(config, universe=self.universe) self.assertFalse(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) self.assertFalse(constraints.isAcceptable(self.pviB)) # Accept PVI for instrument B but not instrument A config = ConstraintsConfig( {"accept": ["calexp", { "instrument<B>": ["pvi"] }]}) constraints = Constraints(config, universe=self.universe) self.assertTrue(constraints.isAcceptable(self.calexpA)) self.assertFalse(constraints.isAcceptable(self.pviA)) self.assertTrue(constraints.isAcceptable(self.pviB)) def testSimpleReject(self): config = ConstraintsConfig({"reject": ["calexp", "ExposureF"]}) constraints = Constraints(config, universe=self.universe) self.assertFalse(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) def testAcceptReject(self): # Reject everything except calexp config = ConstraintsConfig({"accept": ["calexp"], "reject": ["all"]}) constraints = Constraints(config, universe=self.universe) self.assertTrue(constraints.isAcceptable(self.calexpA)) self.assertFalse(constraints.isAcceptable(self.pviA)) # Accept everything except calexp config = ConstraintsConfig({"reject": ["calexp"], "accept": ["all"]}) constraints = Constraints(config, universe=self.universe) self.assertFalse(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) # Reject pvi but explicitly accept pvi for instrument A # Reject all instrument A but accept everything else # The reject here is superfluous config = ConstraintsConfig({ "accept": [{ "instrument<A>": ["pvi"] }], "reject": ["pvi"] }) constraints = Constraints(config, universe=self.universe) self.assertFalse(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) self.assertFalse(constraints.isAcceptable(self.pviB)) # Accept everything except pvi from other than instrument A config = ConstraintsConfig({ "accept": ["all", { "instrument<A>": ["pvi"] }], "reject": ["pvi"] }) constraints = Constraints(config, universe=self.universe) self.assertTrue(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) self.assertFalse(constraints.isAcceptable(self.pviB)) def testWildcardReject(self): # Reject everything config = ConstraintsConfig({"reject": ["all"]}) constraints = Constraints(config, universe=self.universe) self.assertFalse(constraints.isAcceptable(self.calexpA)) self.assertFalse(constraints.isAcceptable(self.pviA)) # Reject all instrument A but accept everything else config = ConstraintsConfig({"reject": [{"instrument<A>": ["all"]}]}) constraints = Constraints(config, universe=self.universe) self.assertFalse(constraints.isAcceptable(self.calexpA)) self.assertFalse(constraints.isAcceptable(self.pviA)) self.assertTrue(constraints.isAcceptable(self.pviB)) def testWildcardAccept(self): # Accept everything config = ConstraintsConfig({}) constraints = Constraints(config, universe=self.universe) self.assertTrue(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) # Accept everything constraints = Constraints(None, universe=self.universe) self.assertTrue(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) # Accept everything explicitly config = ConstraintsConfig({"accept": ["all"]}) constraints = Constraints(config, universe=self.universe) self.assertTrue(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) # Accept all instrument A but reject everything else config = ConstraintsConfig({"accept": [{"instrument<A>": ["all"]}]}) constraints = Constraints(config, universe=self.universe) self.assertTrue(constraints.isAcceptable(self.calexpA)) self.assertTrue(constraints.isAcceptable(self.pviA)) self.assertFalse(constraints.isAcceptable(self.pviB)) def testEdgeCases(self): # Accept everything and reject everything config = ConstraintsConfig({"accept": ["all"], "reject": ["all"]}) with self.assertRaises(ValidationError): Constraints(config, universe=self.universe)