def get_config(config_file='web/web'):
"""Read the configuration file `config_file`.
If an environment variable SWH_CONFIG_FILENAME is defined, this
takes precedence over the config_file parameter.
In any case, update the app with parameters (secret_key, conf)
and return the parsed configuration as a dict.
If no configuration file is provided, return a default
configuration.
"""
if not swhweb_config:
config_filename = os.environ.get('SWH_CONFIG_FILENAME')
if config_filename:
config_file = config_filename
cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
swhweb_config.update(cfg)
config.prepare_folders(swhweb_config, 'log_dir')
swhweb_config['storage'] = get_storage(**swhweb_config['storage'])
swhweb_config['vault'] = get_vault(**swhweb_config['vault'])
swhweb_config['indexer_storage'] = \
get_indexer_storage(**swhweb_config['indexer_storage'])
swhweb_config['scheduler'] = get_scheduler(
**swhweb_config['scheduler'])
return swhweb_config
def get_config(config_file="web/web"):
"""Read the configuration file `config_file`.
If an environment variable SWH_CONFIG_FILENAME is defined, this
takes precedence over the config_file parameter.
In any case, update the app with parameters (secret_key, conf)
and return the parsed configuration as a dict.
If no configuration file is provided, return a default
configuration.
"""
if not swhweb_config:
config_filename = os.environ.get("SWH_CONFIG_FILENAME")
if config_filename:
config_file = config_filename
cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
swhweb_config.update(cfg)
config.prepare_folders(swhweb_config, "log_dir")
if swhweb_config.get("search"):
swhweb_config["search"] = get_search(**swhweb_config["search"])
else:
swhweb_config["search"] = None
swhweb_config["storage"] = get_storage(**swhweb_config["storage"])
swhweb_config["vault"] = get_vault(**swhweb_config["vault"])
swhweb_config["indexer_storage"] = get_indexer_storage(
**swhweb_config["indexer_storage"])
swhweb_config["scheduler"] = get_scheduler(
**swhweb_config["scheduler"])
return swhweb_config
def get_cooker(bundle_type: str, swhid: CoreSWHID):
"""Instantiate a cooker class of type bundle_type.
Returns:
Cooker class in charge of cooking the bundle_type with id swhid.
Raises:
ValueError in case of a missing top-level vault key configuration or a storage
key.
EnvironmentError in case the vault configuration reference a non remote class.
"""
if "SWH_CONFIG_FILENAME" in os.environ:
cfg = read_config(os.environ["SWH_CONFIG_FILENAME"], DEFAULT_CONFIG)
else:
cfg = load_named_config(DEFAULT_CONFIG_PATH, DEFAULT_CONFIG)
cooker_cls = get_cooker_cls(bundle_type, swhid.object_type)
cfg = check_config(cfg)
vcfg = cfg["vault"]
storage = get_storage(**vcfg.pop("storage"))
backend = get_vault(**vcfg)
try:
from swh.graph.client import RemoteGraphClient # optional dependency
graph = RemoteGraphClient(
**vcfg["graph"]) if vcfg.get("graph") else None
except ModuleNotFoundError:
if vcfg.get("graph"):
raise EnvironmentError(
"Graph configuration required but module is not installed.")
else:
graph = None
kwargs = {
k: v
for (k, v) in cfg.items()
if k in ("max_bundle_size", "thread_pool_size")
}
return cooker_cls(
swhid,
backend=backend,
storage=storage,
graph=graph,
**kwargs,
)
def swh_vault(swh_vault_config):
return get_vault("local", **swh_vault_config)
def test_init_get_vault_ok(swh_vault_config):
concrete_vault = get_vault("local", **swh_vault_config)
assert isinstance(concrete_vault, VaultBackend)
def test_init_get_vault_deprecation_warning(class_name, expected_class, kwargs,
mock_psycopg2):
with pytest.warns(DeprecationWarning):
concrete_vault = get_vault(class_name, args=kwargs)
assert isinstance(concrete_vault, expected_class)
def test_init_get_vault(class_name, expected_class, kwargs, mock_psycopg2):
concrete_vault = get_vault(class_name, **kwargs)
assert isinstance(concrete_vault, expected_class)
def test_init_get_vault_failure():
with pytest.raises(ValueError, match="Unknown Vault class"):
get_vault("unknown-vault-storage")