def setUp(self):
config = Config(self.configFile)
uri = ButlerURI(config[".datastore.datastore.root"])
self.bucketName = uri.netloc
if self.useTempRoot:
self.root = self.genRoot()
rooturi = f"s3://{self.bucketName}/{self.root}"
config.update({"datastore": {"datastore": {"root": rooturi}}})
# set up some fake credentials if they do not exist
self.usingDummyCredentials = setAwsEnvCredentials()
# MOTO needs to know that we expect Bucket bucketname to exist
# (this used to be the class attribute bucketName)
s3 = boto3.resource("s3")
s3.create_bucket(Bucket=self.bucketName)
self.datastoreStr = f"datastore={self.root}"
self.datastoreName = [f"S3Datastore@{rooturi}"]
Butler.makeRepo(rooturi, config=config, forceConfigRoot=False)
self.tmpConfigFile = os.path.join(rooturi, "butler.yaml")
def testUpdate(self):
c = Config({"a": {"b": 1}})
c.update({"a": {"c": 2}})
self.assertEqual(c[".a.b"], 1)
self.assertEqual(c[".a.c"], 2)
c.update({"a": {"d": [3, 4]}})
self.assertEqual(c[".a.d.0"], 3)
c.update({"z": [5, 6, {"g": 2, "h": 3}]})
self.assertEqual(c[".z.1"], 6)
# This is detached from parent
c2 = c[".z.2"]
self.assertEqual(c2["g"], 2)
c2.update({"h": 4, "j": 5})
self.assertEqual(c2["h"], 4)
self.assertNotIn(".z.2.j", c)
self.assertNotEqual(c[".z.2.h"], 4)
with self.assertRaises(RuntimeError):
c.update([1, 2, 3])
def setConfigRoot(cls,
root: str,
config: Config,
full: Config,
overwrite: bool = True) -> None:
"""Set any filesystem-dependent config options for child Datastores to
be appropriate for a new empty repository with the given root.
Parameters
----------
root : `str`
Filesystem path to the root of the data repository.
config : `Config`
A `Config` to update. Only the subset understood by
this component will be updated. Will not expand
defaults.
full : `Config`
A complete config with all defaults expanded that can be
converted to a `DatastoreConfig`. Read-only and will not be
modified by this method.
Repository-specific options that should not be obtained
from defaults when Butler instances are constructed
should be copied from ``full`` to ``config``.
overwrite : `bool`, optional
If `False`, do not modify a value in ``config`` if the value
already exists. Default is always to overwrite with the provided
``root``.
Notes
-----
If a keyword is explicitly defined in the supplied ``config`` it
will not be overridden by this method if ``overwrite`` is `False`.
This allows explicit values set in external configs to be retained.
"""
# Extract the part of the config we care about updating
datastoreConfig = DatastoreConfig(config, mergeDefaults=False)
# And the subset of the full config that we can use for reference.
# Do not bother with defaults because we are told this already has
# them.
fullDatastoreConfig = DatastoreConfig(full, mergeDefaults=False)
# Loop over each datastore config and pass the subsets to the
# child datastores to process.
containerKey = cls.containerKey
for idx, (child, fullChild) in enumerate(
zip(datastoreConfig[containerKey],
fullDatastoreConfig[containerKey])):
childConfig = DatastoreConfig(child, mergeDefaults=False)
fullChildConfig = DatastoreConfig(fullChild, mergeDefaults=False)
datastoreClass = doImport(fullChildConfig["cls"])
newroot = "{}/{}_{}".format(root, datastoreClass.__qualname__, idx)
datastoreClass.setConfigRoot(newroot,
childConfig,
fullChildConfig,
overwrite=overwrite)
# Reattach to parent
datastoreConfig[containerKey, idx] = childConfig
# Reattach modified datastore config to parent
# If this has a datastore key we attach there, otherwise we assume
# this information goes at the top of the config hierarchy.
if DatastoreConfig.component in config:
config[DatastoreConfig.component] = datastoreConfig
else:
config.update(datastoreConfig)
return
def makeTestRepo(root, dataIds, *, config=None, **kwargs):
"""Create an empty repository with dummy data IDs.
Parameters
----------
root : `str`
The location of the root directory for the repository.
dataIds : `~collections.abc.Mapping` [`str`, `iterable`]
A mapping keyed by the dimensions used in the test. Each value
is an iterable of names for that dimension (e.g., detector IDs for
`"detector"`). Related dimensions (e.g., instruments and detectors)
are linked arbitrarily.
config : `lsst.daf.butler.Config`, optional
A configuration for the repository (for details, see
`lsst.daf.butler.Butler.makeRepo`). If omitted, creates a repository
with default dataset and storage types, but optimized for speed.
The defaults set ``.datastore.cls``, ``.datastore.checksum`` and
``.registry.db``. If a supplied config does not specify these values
the internal defaults will be used to ensure that we have a usable
configuration.
**kwargs
Extra arguments to `lsst.daf.butler.Butler.makeRepo`.
Returns
-------
butler : `lsst.daf.butler.Butler`
A Butler referring to the new repository. This Butler is provided only
for additional setup; to keep test cases isolated, it is highly
recommended that each test create its own Butler with a
unique run/collection. See `makeTestCollection`.
Notes
-----
This function provides a "quick and dirty" repository for simple unit
tests that don't depend on complex data relationships. Because it assigns
dimension relationships and other metadata abitrarily, it is ill-suited
for tests where the structure of the data matters. If you need such a
dataset, create it directly or use a saved test dataset.
Since the values in ``dataIds`` uniquely determine the repository's
data IDs, the fully linked IDs can be recovered by calling
`expandUniqueId`, so long as no other code has inserted dimensions into
the repository registry.
"""
defaults = Config()
defaults[
"datastore",
"cls"] = "lsst.daf.butler.datastores.inMemoryDatastore.InMemoryDatastore"
defaults["datastore", "checksum"] = False # In case of future changes
defaults["registry", "db"] = "sqlite:///:memory:"
if config:
defaults.update(config)
# Disable config root by default so that our registry override will
# not be ignored.
# newConfig guards against location-related keywords like outfile
newConfig = Butler.makeRepo(root,
config=defaults,
forceConfigRoot=False,
**kwargs)
butler = Butler(newConfig, writeable=True)
dimensionRecords = _makeRecords(dataIds, butler.registry.dimensions)
for dimension, records in dimensionRecords.items():
butler.registry.insertDimensionData(dimension, *records)
return butler
