def setUp(self): config = Config(self.configFile) uri = ButlerURI(config[".datastore.datastore.root"]) self.bucketName = uri.netloc if self.useTempRoot: self.root = self.genRoot() rooturi = f"s3://{self.bucketName}/{self.root}" config.update({"datastore": {"datastore": {"root": rooturi}}}) # set up some fake credentials if they do not exist self.usingDummyCredentials = setAwsEnvCredentials() # MOTO needs to know that we expect Bucket bucketname to exist # (this used to be the class attribute bucketName) s3 = boto3.resource("s3") s3.create_bucket(Bucket=self.bucketName) self.datastoreStr = f"datastore={self.root}" self.datastoreName = [f"S3Datastore@{rooturi}"] Butler.makeRepo(rooturi, config=config, forceConfigRoot=False) self.tmpConfigFile = os.path.join(rooturi, "butler.yaml")
def testUpdate(self): c = Config({"a": {"b": 1}}) c.update({"a": {"c": 2}}) self.assertEqual(c[".a.b"], 1) self.assertEqual(c[".a.c"], 2) c.update({"a": {"d": [3, 4]}}) self.assertEqual(c[".a.d.0"], 3) c.update({"z": [5, 6, {"g": 2, "h": 3}]}) self.assertEqual(c[".z.1"], 6) # This is detached from parent c2 = c[".z.2"] self.assertEqual(c2["g"], 2) c2.update({"h": 4, "j": 5}) self.assertEqual(c2["h"], 4) self.assertNotIn(".z.2.j", c) self.assertNotEqual(c[".z.2.h"], 4) with self.assertRaises(RuntimeError): c.update([1, 2, 3])
def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None: """Set any filesystem-dependent config options for child Datastores to be appropriate for a new empty repository with the given root. Parameters ---------- root : `str` Filesystem path to the root of the data repository. config : `Config` A `Config` to update. Only the subset understood by this component will be updated. Will not expand defaults. full : `Config` A complete config with all defaults expanded that can be converted to a `DatastoreConfig`. Read-only and will not be modified by this method. Repository-specific options that should not be obtained from defaults when Butler instances are constructed should be copied from ``full`` to ``config``. overwrite : `bool`, optional If `False`, do not modify a value in ``config`` if the value already exists. Default is always to overwrite with the provided ``root``. Notes ----- If a keyword is explicitly defined in the supplied ``config`` it will not be overridden by this method if ``overwrite`` is `False`. This allows explicit values set in external configs to be retained. """ # Extract the part of the config we care about updating datastoreConfig = DatastoreConfig(config, mergeDefaults=False) # And the subset of the full config that we can use for reference. # Do not bother with defaults because we are told this already has # them. fullDatastoreConfig = DatastoreConfig(full, mergeDefaults=False) # Loop over each datastore config and pass the subsets to the # child datastores to process. containerKey = cls.containerKey for idx, (child, fullChild) in enumerate( zip(datastoreConfig[containerKey], fullDatastoreConfig[containerKey])): childConfig = DatastoreConfig(child, mergeDefaults=False) fullChildConfig = DatastoreConfig(fullChild, mergeDefaults=False) datastoreClass = doImport(fullChildConfig["cls"]) newroot = "{}/{}_{}".format(root, datastoreClass.__qualname__, idx) datastoreClass.setConfigRoot(newroot, childConfig, fullChildConfig, overwrite=overwrite) # Reattach to parent datastoreConfig[containerKey, idx] = childConfig # Reattach modified datastore config to parent # If this has a datastore key we attach there, otherwise we assume # this information goes at the top of the config hierarchy. if DatastoreConfig.component in config: config[DatastoreConfig.component] = datastoreConfig else: config.update(datastoreConfig) return
def makeTestRepo(root, dataIds, *, config=None, **kwargs): """Create an empty repository with dummy data IDs. Parameters ---------- root : `str` The location of the root directory for the repository. dataIds : `~collections.abc.Mapping` [`str`, `iterable`] A mapping keyed by the dimensions used in the test. Each value is an iterable of names for that dimension (e.g., detector IDs for `"detector"`). Related dimensions (e.g., instruments and detectors) are linked arbitrarily. config : `lsst.daf.butler.Config`, optional A configuration for the repository (for details, see `lsst.daf.butler.Butler.makeRepo`). If omitted, creates a repository with default dataset and storage types, but optimized for speed. The defaults set ``.datastore.cls``, ``.datastore.checksum`` and ``.registry.db``. If a supplied config does not specify these values the internal defaults will be used to ensure that we have a usable configuration. **kwargs Extra arguments to `lsst.daf.butler.Butler.makeRepo`. Returns ------- butler : `lsst.daf.butler.Butler` A Butler referring to the new repository. This Butler is provided only for additional setup; to keep test cases isolated, it is highly recommended that each test create its own Butler with a unique run/collection. See `makeTestCollection`. Notes ----- This function provides a "quick and dirty" repository for simple unit tests that don't depend on complex data relationships. Because it assigns dimension relationships and other metadata abitrarily, it is ill-suited for tests where the structure of the data matters. If you need such a dataset, create it directly or use a saved test dataset. Since the values in ``dataIds`` uniquely determine the repository's data IDs, the fully linked IDs can be recovered by calling `expandUniqueId`, so long as no other code has inserted dimensions into the repository registry. """ defaults = Config() defaults[ "datastore", "cls"] = "lsst.daf.butler.datastores.inMemoryDatastore.InMemoryDatastore" defaults["datastore", "checksum"] = False # In case of future changes defaults["registry", "db"] = "sqlite:///:memory:" if config: defaults.update(config) # Disable config root by default so that our registry override will # not be ignored. # newConfig guards against location-related keywords like outfile newConfig = Butler.makeRepo(root, config=defaults, forceConfigRoot=False, **kwargs) butler = Butler(newConfig, writeable=True) dimensionRecords = _makeRecords(dataIds, butler.registry.dimensions) for dimension, records in dimensionRecords.items(): butler.registry.insertDimensionData(dimension, *records) return butler