def web(ctx: Context, config_file: str):
"""Software Heritage web client"""
import logging
from swh.core import config
from swh.web.client.client import WebAPIClient
if not config_file:
config_file = DEFAULT_CONFIG_PATH
try:
conf = config.read_raw_config(config.config_basepath(config_file))
if not conf:
raise ValueError(f"Cannot parse configuration file: {config_file}")
# TODO: Determine what the following conditional is for
if config_file == DEFAULT_CONFIG_PATH:
try:
conf = conf["swh"]["web"]["client"]
except KeyError:
pass
# recursive merge not done by config.read
conf = config.merge_configs(DEFAULT_CONFIG, conf)
except Exception:
logging.warning("Using default configuration (cannot load custom one)",
exc_info=True)
conf = DEFAULT_CONFIG
ctx.ensure_object(dict)
ctx.obj["client"] = WebAPIClient(conf["api_url"], conf["bearer_token"])
def scanner(ctx, config_file: Optional[str]):
env_config_path = os.environ.get(CONFIG_ENVVAR)
# read_raw_config do not fail if file does not exist, so check it beforehand
# while enforcing loading priority
if config_file:
if not config.config_exists(config_file):
raise click.BadParameter(
f"File '{config_file}' cannot be opened.", param_hint="--config-file"
)
elif env_config_path:
if not config.config_exists(env_config_path):
raise click.BadParameter(
f"File '{env_config_path}' cannot be opened.", param_hint=CONFIG_ENVVAR
)
config_file = env_config_path
elif config.config_exists(DEFAULT_CONFIG_PATH):
config_file = DEFAULT_CONFIG_PATH
conf = DEFAULT_CONFIG
if config_file is not None:
conf = config.read_raw_config(config.config_basepath(config_file))
conf = config.merge_configs(DEFAULT_CONFIG, conf)
ctx.ensure_object(dict)
ctx.obj["config"] = conf
def test_merge_config_type_error():
for v in (1, "str", None):
with pytest.raises(TypeError):
config.merge_configs(v, {})
with pytest.raises(TypeError):
config.merge_configs({}, v)
for v in (1, "str"):
with pytest.raises(TypeError):
config.merge_configs({"a": v}, {"a": {}})
with pytest.raises(TypeError):
config.merge_configs({"a": {}}, {"a": v})
def __init__(self, config=None, **kw) -> None:
"""Prepare and check that the indexer is ready to run.
"""
super().__init__()
if config is not None:
self.config = config
elif SWH_CONFIG:
self.config = SWH_CONFIG.copy()
else:
self.config = load_from_envvar()
self.config = merge_configs(DEFAULT_CONFIG, self.config)
self.prepare()
self.check()
self.log.debug("%s: config=%s", self, self.config)
def build_app(config=None):
config = merge_configs(
{k: v for (k, (_, v)) in DEFAULT_CONFIG.items()}, config or {}
)
config["task_queues"] = CELERY_QUEUES + [
Queue(queue, Exchange(queue), routing_key=queue)
for queue in config.get("task_queues", ())
]
logger.debug("Creating a Celery app with %s", config)
# Instantiate the Celery app
app = Celery(broker=config["task_broker"], task_cls="swh.scheduler.task:SWHTask")
app.add_defaults(CELERY_DEFAULT_CONFIG)
app.add_defaults(config)
return app
def make_app_from_configfile(config_path: Optional[str] = None,
**kwargs) -> VaultServerApp:
"""Load and check configuration if ok, then instantiate (once) a vault server
application.
"""
config_path = os.environ.get("SWH_CONFIG_FILENAME", config_path)
if not config_path:
raise ValueError("Missing configuration path.")
if not os.path.isfile(config_path):
raise ValueError(f"Configuration path {config_path} should exist.")
app_config = read_raw_config(config_basepath(config_path))
app_config["vault"] = check_config(app_config)
app.config.update(merge_configs(DEFAULT_CONFIG, app_config))
return app
def fuse(ctx, config_file):
"""Software Heritage virtual file system"""
import logging
from shutil import which
import yaml
from swh.core import config
if which("fusermount3") is None:
logging.error("Missing dependency: 'fusermount3'")
ctx.exit(1)
if not config_file:
config_file = DEFAULT_CONFIG_PATH
if os.path.isfile(config_file):
try:
conf = config.read_raw_config(config.config_basepath(config_file))
if not conf:
raise ValueError(
f"Cannot parse configuration file: {config_file}")
if config_file == DEFAULT_CONFIG_PATH:
try:
conf = conf["swh"]["fuse"]
except KeyError:
pass
# recursive merge not done by config.read
conf = config.merge_configs(DEFAULT_CONFIG, conf)
except Exception:
logging.warning(
"Using default configuration (cannot load custom one)",
exc_info=True)
conf = DEFAULT_CONFIG
else:
logging.info("Using default configuration")
conf = DEFAULT_CONFIG
logging.debug("Read configuration: \n%s", yaml.dump(conf))
ctx.ensure_object(dict)
ctx.obj["config"] = conf
def random_content():
return {
"sha1": random_sha1(),
"sha1_git": random_sha1(),
"sha256": random_sha256(),
"blake2s256": random_blake2s256(),
}
_TEST_MIMETYPE_INDEXER_CONFIG = merge_configs(
_TEST_INDEXER_BASE_CONFIG,
{
"tools": {
"name": "file",
"version": "1:5.30-1+deb9u1",
"configuration": {
"type": "library",
"debian-package": "python3-magic"
},
}
},
)
_TEST_LICENSE_INDEXER_CONFIG = merge_configs(
_TEST_INDEXER_BASE_CONFIG,
{
"workdir": "/tmp/swh/indexer.fossology.license",
"tools": {
"name": "nomos",
"version": "3.1.0rc2-31-ga2cbb8c",
"configuration": {
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.config = merge_configs(DEFAULT_CONFIG, self.config)
self.working_directory = self.config["workdir"]
self.language_map = self.config["languages"]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.config = merge_configs(DEFAULT_CONFIG, self.config)
def test_merge_config():
cfg_a = {
"a": 42,
"b": [1, 2, 3],
"c": None,
"d": {"gheez": 27},
"e": {
"ea": "Mr. Bungle",
"eb": None,
"ec": [11, 12, 13],
"ed": {"eda": "Secret Chief 3", "edb": "Faith No More"},
"ee": 451,
},
"f": "Janis",
}
cfg_b = {
"a": 43,
"b": [41, 42, 43],
"c": "Tom Waits",
"d": None,
"e": {
"ea": "Igorrr",
"ec": [51, 52],
"ed": {"edb": "Sleepytime Gorilla Museum", "edc": "Nils Peter Molvaer"},
},
"g": "Hüsker Dü",
}
# merge A, B
cfg_m = config.merge_configs(cfg_a, cfg_b)
assert cfg_m == {
"a": 43, # b takes precedence
"b": [41, 42, 43], # b takes precedence
"c": "Tom Waits", # b takes precedence
"d": None, # b['d'] takes precedence (explicit None)
"e": {
"ea": "Igorrr", # a takes precedence
"eb": None, # only in a
"ec": [51, 52], # b takes precedence
"ed": {
"eda": "Secret Chief 3", # only in a
"edb": "Sleepytime Gorilla Museum", # b takes precedence
"edc": "Nils Peter Molvaer",
}, # only defined in b
"ee": 451,
},
"f": "Janis", # only defined in a
"g": "Hüsker Dü", # only defined in b
}
# merge B, A
cfg_m = config.merge_configs(cfg_b, cfg_a)
assert cfg_m == {
"a": 42, # a takes precedence
"b": [1, 2, 3], # a takes precedence
"c": None, # a takes precedence
"d": {"gheez": 27}, # a takes precedence
"e": {
"ea": "Mr. Bungle", # a takes precedence
"eb": None, # only defined in a
"ec": [11, 12, 13], # a takes precedence
"ed": {
"eda": "Secret Chief 3", # only in a
"edb": "Faith No More", # a takes precedence
"edc": "Nils Peter Molvaer",
}, # only in b
"ee": 451,
},
"f": "Janis", # only in a
"g": "Hüsker Dü", # only in b
}
