def testConstructor(self): """Test DatasetTypeDescriptor init """ name = "testDataset" dimensionNames = frozenset(["label"]) storageClassName = "Catalog" universe = DimensionUniverse.fromConfig() descriptor = pipeBase.DatasetTypeDescriptor( name=name, dimensionNames=dimensionNames, storageClassName=storageClassName, scalar=False, manualLoad=False) datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, name) self.assertEqual(datasetType.dimensions.names, dimensionNames) self.assertEqual(datasetType.storageClass.name, storageClassName) self.assertFalse(descriptor.scalar) descriptor = pipeBase.DatasetTypeDescriptor( name=name, dimensionNames=dimensionNames, storageClassName=storageClassName, scalar=True, manualLoad=False) datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, name) self.assertEqual(datasetType.dimensions.names, dimensionNames) self.assertEqual(datasetType.storageClass.name, storageClassName) self.assertTrue(descriptor.scalar)
def testRegistryWithStorageClass(self): """Test that the registry can be given a StorageClass object. """ formatterTypeName = "lsst.daf.butler.formatters.yamlFormatter.YamlFormatter" storageClassName = "TestClass" sc = StorageClass(storageClassName, dict, None) universe = DimensionUniverse.fromConfig() datasetType = DatasetType("calexp", universe.extract([]), sc) # Store using an instance self.factory.registerFormatter(sc, formatterTypeName) # Retrieve using the class f = self.factory.getFormatter(sc, self.fileDescriptor) self.assertIsFormatter(f) self.assertEqual(f.fileDescriptor, self.fileDescriptor) # Retrieve using the DatasetType f2 = self.factory.getFormatter(datasetType, self.fileDescriptor) self.assertIsFormatter(f2) self.assertEqual(f.name(), f2.name()) # Class directly f2cls = self.factory.getFormatterClass(datasetType) self.assertIsFormatter(f2cls) # This might defer the import, pytest may have already loaded it from lsst.daf.butler.formatters.yamlFormatter import YamlFormatter self.assertEqual(type(f), YamlFormatter) with self.assertRaises(KeyError): # Attempt to overwrite using a different value self.factory.registerFormatter(storageClassName, "lsst.daf.butler.formatters.jsonFormatter.JsonFormatter")
def setUp(self): self.universe = DimensionUniverse.fromConfig() self.given = DimensionSet(universe=self.universe, elements=["skymap"]) self.parameters = dict(skymap="unimportant", tractMax=5, patchNxMax=3, patchNyMax=3)
def testDatasetConfig(self): """Test for a config with datasets """ config = ConfigWithDatasets() universe = DimensionUniverse.fromConfig() descriptors = pipeBase.PipelineTask.getInputDatasetTypes(config) self.assertCountEqual(descriptors.keys(), ["input1", "input2"]) descriptor = descriptors["input1"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.input1.name) self.assertCountEqual(datasetType.dimensions.names, config.input1.dimensions) self.assertEqual(datasetType.storageClass.name, config.input1.storageClass) self.assertFalse(descriptor.scalar) descriptor = descriptors["input2"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.input2.name) self.assertCountEqual(datasetType.dimensions.names, config.input2.dimensions) self.assertEqual(datasetType.storageClass.name, config.input2.storageClass) self.assertTrue(descriptor.scalar) descriptors = pipeBase.PipelineTask.getOutputDatasetTypes(config) self.assertCountEqual(descriptors.keys(), ["output"]) descriptor = descriptors["output"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.output.name) self.assertCountEqual(datasetType.dimensions.names, config.output.dimensions) self.assertEqual(datasetType.storageClass.name, config.output.storageClass) self.assertFalse(descriptor.scalar) descriptors = pipeBase.PipelineTask.getInitInputDatasetTypes(config) self.assertCountEqual(descriptors.keys(), ["initInput"]) descriptor = descriptors["initInput"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.initInput.name) self.assertEqual(len(datasetType.dimensions), 0) self.assertEqual(datasetType.storageClass.name, config.initInput.storageClass) self.assertTrue(descriptor.scalar) descriptors = pipeBase.PipelineTask.getInitOutputDatasetTypes(config) self.assertCountEqual(descriptors.keys(), ["initOutput"]) descriptor = descriptors["initOutput"] datasetType = descriptor.makeDatasetType(universe) self.assertEqual(datasetType.name, config.initOutput.name) self.assertEqual(len(datasetType.dimensions), 0) self.assertEqual(datasetType.storageClass.name, config.initOutput.storageClass) self.assertTrue(descriptor.scalar)
def setUpClass(cls): # Storage Classes are fixed for all datastores in these tests scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml") cls.storageClassFactory = StorageClassFactory() cls.storageClassFactory.addFromConfig(scConfigFile) # Read the Datastore config so we can get the class # information (since we should not assume the constructor # name here, but rely on the configuration file itself) datastoreConfig = DatastoreConfig(cls.configFile) cls.datastoreType = doImport(datastoreConfig["cls"]) cls.universe = DimensionUniverse.fromConfig()
def testFromConfig(self): """Test DatasetTypeDescriptor.fromConfig() """ universe = DimensionUniverse.fromConfig() config = AddConfig() descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.input) datasetType = descriptor.makeDatasetType(universe) self.assertIsInstance(descriptor, pipeBase.DatasetTypeDescriptor) self.assertEqual(datasetType.name, "add_input") self.assertFalse(descriptor.scalar) descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.output) datasetType = descriptor.makeDatasetType(universe) self.assertIsInstance(descriptor, pipeBase.DatasetTypeDescriptor) self.assertEqual(datasetType.name, "add_output") self.assertFalse(descriptor.scalar)
def testRegistryConfig(self): configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml") config = Config(configFile) universe = DimensionUniverse.fromConfig() self.factory.registerFormatters(config["datastore", "formatters"], universe=universe) # Create a DatasetRef with and without instrument matching the # one in the config file. dimensions = universe.extract( ("visit", "physical_filter", "instrument")) sc = StorageClass("DummySC", dict, None) refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "DummyHSC", "physical_filter": "v" }) refPviHscFmt = self.factory.getFormatter(refPviHsc) self.assertIsInstance(refPviHscFmt, Formatter) self.assertIn("JsonFormatter", refPviHscFmt.name()) refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "DummyNotHSC", "physical_filter": "v" }) refPviNotHscFmt = self.factory.getFormatter(refPviNotHsc) self.assertIsInstance(refPviNotHscFmt, Formatter) self.assertIn("PickleFormatter", refPviNotHscFmt.name()) # Create a DatasetRef that should fall back to using Dimensions refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, { "instrument": "DummyHSC", "physical_filter": "v" }) refPvixNotHscFmt = self.factory.getFormatter(refPvixHsc) self.assertIsInstance(refPvixNotHscFmt, Formatter) self.assertIn("PickleFormatter", refPvixNotHscFmt.name()) # Create a DatasetRef that should fall back to using StorageClass dimensionsNoV = universe.extract(("physical_filter", "instrument")) refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, { "instrument": "DummyHSC", "physical_filter": "v" }) refPvixNotHscDims_fmt = self.factory.getFormatter(refPvixNotHscDims) self.assertIsInstance(refPvixNotHscDims_fmt, Formatter) self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name())
def testMap(self): universe = DimensionUniverse.fromConfig() c = CompositesMap(self.configFile, universe=universe) # Check that a str is not supported with self.assertRaises(ValueError): c.shouldBeDisassembled("fred") # These will fail (not a composite) sc = StorageClass("StructuredDataJson") d = DatasetType("dummyTrue", universe.extract([]), sc) self.assertFalse(sc.isComposite()) self.assertFalse(d.isComposite()) self.assertFalse(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}") self.assertFalse(c.shouldBeDisassembled(sc), f"Test with StorageClass: {sc}") # Repeat but this time use a composite storage class sccomp = StorageClass("Dummy") sc = StorageClass("StructuredDataJson", components={"dummy": sccomp}) d = DatasetType("dummyTrue", universe.extract([]), sc) self.assertTrue(sc.isComposite()) self.assertTrue(d.isComposite()) self.assertTrue(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}") self.assertFalse(c.shouldBeDisassembled(sc), f"Test with StorageClass: {sc}") # Override with False d = DatasetType("dummyFalse", universe.extract([]), sc) self.assertFalse(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}") # DatasetType that has no explicit entry d = DatasetType("dummyFred", universe.extract([]), sc) self.assertFalse(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}") # StorageClass that will be disassembled sc = StorageClass("StructuredComposite", components={"dummy": sccomp}) d = DatasetType("dummyFred", universe.extract([]), sc) self.assertTrue(c.shouldBeDisassembled(d), f"Test with DatasetType: {d}")
def _makeQuanta(self, config): """Create set of Quanta """ universe = DimensionUniverse.fromConfig() run = Run(collection=1, environment=None, pipeline=None) descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.input) dstype0 = descriptor.makeDatasetType(universe) descriptor = pipeBase.DatasetTypeDescriptor.fromConfig(config.output) dstype1 = descriptor.makeDatasetType(universe) quanta = [] for visit in range(100): quantum = Quantum(run=run, task=None) quantum.addPredictedInput(self._makeDSRefVisit(dstype0, visit)) quantum.addOutput(self._makeDSRefVisit(dstype1, visit)) quanta.append(quantum) return quanta
def testAddInputsOutputs(self): """Test of addPredictedInput() method. """ quantum = Quantum(task="some.task.object", run=None) # start with empty self.assertEqual(quantum.predictedInputs, dict()) universe = DimensionUniverse.fromConfig() instrument = "DummyCam" datasetTypeName = "test_ds" storageClass = StorageClass("testref_StructuredData") datasetType = DatasetType(datasetTypeName, universe.extract(("instrument", "visit")), storageClass) # add one ref ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42)) quantum.addPredictedInput(ref) self.assertIn(datasetTypeName, quantum.predictedInputs) self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 1) # add second ref ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43)) quantum.addPredictedInput(ref) self.assertEqual(len(quantum.predictedInputs[datasetTypeName]), 2) # mark last ref as actually used self.assertEqual(quantum.actualInputs, dict()) quantum._markInputUsed(ref) self.assertIn(datasetTypeName, quantum.actualInputs) self.assertEqual(len(quantum.actualInputs[datasetTypeName]), 1) # add couple of outputs too self.assertEqual(quantum.outputs, dict()) ref = DatasetRef(datasetType, dict(instrument=instrument, visit=42)) quantum.addOutput(ref) self.assertIn(datasetTypeName, quantum.outputs) self.assertEqual(len(quantum.outputs[datasetTypeName]), 1) ref = DatasetRef(datasetType, dict(instrument=instrument, visit=43)) quantum.addOutput(ref) self.assertEqual(len(quantum.outputs[datasetTypeName]), 2)
def setUp(self): self.id = 0 # Create DatasetRefs to test against constraints model self.universe = DimensionUniverse.fromConfig() dimensions = self.universe.extract( ("visit", "physical_filter", "instrument")) sc = StorageClass("DummySC", dict, None) self.calexpA = self.makeDatasetRef("calexp", dimensions, sc, { "instrument": "A", "physical_filter": "u" }) dimensions = self.universe.extract( ("visit", "calibration_label", "instrument")) self.pviA = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "A", "visit": 1 }) self.pviB = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "B", "visit": 2 })
def __init__(self): self.datasets = {} self.registry = SimpleNamespace( dimensions=DimensionUniverse.fromConfig())
def __init__(self): self._counter = 0 self._entries = {} self.dimensions = DimensionUniverse.fromConfig()
def setUp(self): self.universe = DimensionUniverse.fromConfig()