def fromConfig(cls, registryConfig):
"""Parses the 'db' key in the config, and if they exist username,
password, host, port and database keys, and returns an connection
string object.
If no username and password are found in the connection string, or in
the config, they are retrieved from a file at `DB_AUTH_PATH` or
`DB_AUTH_ENVVAR`. Sqlite dialect does not require a password.
Parameters
----------
config : `ButlerConfig`, `RegistryConfig`, `Config` or `str`
Registry configuration
Returns
-------
connectionString : `sqlalchemy.engine.url.URL`
URL object representing the connection string.
Raises
------
DBAuthError
If the credentials file has incorrect permissions.
"""
# this import can not live on the top because of circular import issue
from lsst.daf.butler.registry import RegistryConfig
regConf = RegistryConfig(registryConfig)
conStr = url.make_url(regConf['db'])
for key in cls.keys:
if getattr(conStr, key) is None:
setattr(conStr, key, regConf.get(key))
# when host is None we cross our fingers and return
if conStr.host is None:
return conStr
# allow other mechanisms to insert username and password by not forcing
# the credentials to exist. If other mechanisms are used it's possible
# that credentials were never set-up, or that there would be no match
# in the credentials file. Both need to be ignored.
try:
dbAuth = DbAuth(DB_AUTH_PATH, DB_AUTH_ENVVAR)
auth = dbAuth.getAuth(conStr.drivername, conStr.username, conStr.host,
conStr.port, conStr.database)
except DbAuthPermissionsError:
# re-raise permission error for safety
raise
except DbAuthError:
# credentials file doesn't exist or no match was found
pass
else:
conStr.username = auth[0]
conStr.password = auth[1]
return conStr
def loadDimensionData() -> DataCoordinateSequence:
"""Load dimension data from an export file included in the code repository.
Returns
-------
dataIds : `DataCoordinateSet`
A set containing all data IDs in the export file.
"""
# Create an in-memory SQLite database and Registry just to import the YAML
# data and retreive it as a set of DataCoordinate objects.
config = RegistryConfig()
config["db"] = "sqlite://"
registry = Registry.fromConfig(config, create=True)
with open(DIMENSION_DATA_FILE, 'r') as stream:
backend = YamlRepoImportBackend(stream, registry)
backend.register()
backend.load(datastore=None)
dimensions = DimensionGraph(registry.dimensions,
names=["visit", "detector", "tract", "patch"])
return DataCoordinateSequence(
dataIds=tuple(registry.queryDimensions(dimensions, expand=True)),
graph=dimensions,
hasFull=True,
hasRecords=True,
)
def testRelVsAbsPath(self):
"""Tests that relative and absolute paths are preserved."""
regConf = RegistryConfig(os.path.join(self.configDir, 'conf1.yaml'))
regConf['db'] = 'sqlite:///relative/path/conf1.sqlite3'
conStrFactory = ConnectionStringFactory()
conStr = conStrFactory.fromConfig(regConf)
self.assertEqual(str(conStr), 'sqlite:///relative/path/conf1.sqlite3')
def testRaises(self):
"""Test that DbAuthError propagates through the class."""
ConnectionStringModule.DB_AUTH_PATH = os.path.join(
self.configDir, "badDbAuth2.yaml")
regConf = RegistryConfig(
os.path.join(self.configDir, 'registryConf2.yaml'))
conStrFactory = ConnectionStringFactory()
with self.assertRaises(DbAuthError):
conStrFactory.fromConfig(regConf)
def testBuilder(self):
"""Tests ConnectionStringBuilder builds correct connection strings.
"""
regConfigs = [RegistryConfig(os.path.join(self.configDir, name)) for name in self.configFiles]
conStrFactory = ConnectionStringFactory()
for regConf, fileName in zip(regConfigs, self.configFiles):
conStr = conStrFactory.fromConfig(regConf)
with self.subTest(confFile=fileName):
self.assertEqual(str(conStr), regConf['expected'],
"test connection string built from config")
def makeRegistry(self) -> Registry:
prefix = f"test_{secrets.token_hex(8).lower()}_"
self._prefixes.append(prefix)
config = RegistryConfig()
# Can't use Registry.fromConfig for these tests because we don't want
# to reconnect to the server every single time. But we at least use
# OracleDatabase.fromConnection rather than the constructor so
# we can try to pass a prefix through via "+" in a namespace.
database = OracleDatabase.fromConnection(connection=self._connection,
origin=0,
namespace=f"+{prefix}")
return Registry(database=database,
dimensions=DimensionUniverse(config),
create=True)
def makeSQLiteRegistry(create=True):
"""Context manager to create new empty registry database.
Yields
------
config : `RegistryConfig`
Registry configuration for initialized registry database.
"""
with temporaryDirectory() as tmpdir:
uri = f"sqlite:///{tmpdir}/gen3.sqlite"
config = RegistryConfig()
config["db"] = uri
if create:
Registry.createFromConfig(config)
yield config
def makeButler(self, **kwargs: Any) -> Butler:
"""Return new Butler instance on each call.
"""
config = ButlerConfig()
# make separate temporary directory for registry of this instance
tmpdir = tempfile.mkdtemp(dir=self.root)
config["registry", "db"] = f"sqlite:///{tmpdir}/gen3.sqlite3"
config["root"] = self.root
# have to make a registry first
registryConfig = RegistryConfig(config.get("registry"))
Registry.createFromConfig(registryConfig)
butler = Butler(config, **kwargs)
DatastoreMock.apply(butler)
return butler
def makeRegistry(self) -> Registry:
config = RegistryConfig()
config["db"] = f"sqlite://"
return Registry.fromConfig(config, create=True)
def makeRegistry(self) -> Registry:
_, filename = tempfile.mkstemp(dir=self.root, suffix=".sqlite3")
config = RegistryConfig()
config["db"] = f"sqlite:///{filename}"
return Registry.fromConfig(config, create=True, butlerRoot=self.root)
def makeRegistry(self) -> Registry:
namespace = f"namespace_{secrets.token_hex(8).lower()}"
config = RegistryConfig()
config["db"] = self.server.url()
config["namespace"] = namespace
return Registry.fromConfig(config, create=True)
