Ejemplo n.º 1
0
def get_config(config_file='web/web'):
    """Read the configuration file `config_file`.

       If an environment variable SWH_CONFIG_FILENAME is defined, this
       takes precedence over the config_file parameter.

       In any case, update the app with parameters (secret_key, conf)
       and return the parsed configuration as a dict.

       If no configuration file is provided, return a default
       configuration.

    """

    if not swhweb_config:
        config_filename = os.environ.get('SWH_CONFIG_FILENAME')
        if config_filename:
            config_file = config_filename
        cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
        swhweb_config.update(cfg)
        config.prepare_folders(swhweb_config, 'log_dir')
        swhweb_config['storage'] = get_storage(**swhweb_config['storage'])
        swhweb_config['vault'] = get_vault(**swhweb_config['vault'])
        swhweb_config['indexer_storage'] = \
            get_indexer_storage(**swhweb_config['indexer_storage'])
        swhweb_config['scheduler'] = get_scheduler(
            **swhweb_config['scheduler'])
    return swhweb_config
Ejemplo n.º 2
0
def get_config(config_file="web/web"):
    """Read the configuration file `config_file`.

       If an environment variable SWH_CONFIG_FILENAME is defined, this
       takes precedence over the config_file parameter.

       In any case, update the app with parameters (secret_key, conf)
       and return the parsed configuration as a dict.

       If no configuration file is provided, return a default
       configuration.

    """

    if not swhweb_config:
        config_filename = os.environ.get("SWH_CONFIG_FILENAME")
        if config_filename:
            config_file = config_filename
        cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
        swhweb_config.update(cfg)
        config.prepare_folders(swhweb_config, "log_dir")
        if swhweb_config.get("search"):
            swhweb_config["search"] = get_search(**swhweb_config["search"])
        else:
            swhweb_config["search"] = None
        swhweb_config["storage"] = get_storage(**swhweb_config["storage"])
        swhweb_config["vault"] = get_vault(**swhweb_config["vault"])
        swhweb_config["indexer_storage"] = get_indexer_storage(
            **swhweb_config["indexer_storage"])
        swhweb_config["scheduler"] = get_scheduler(
            **swhweb_config["scheduler"])
    return swhweb_config
Ejemplo n.º 3
0
def get_cooker(bundle_type: str, swhid: CoreSWHID):
    """Instantiate a cooker class of type bundle_type.

    Returns:
        Cooker class in charge of cooking the bundle_type with id swhid.

    Raises:
        ValueError in case of a missing top-level vault key configuration or a storage
          key.
        EnvironmentError in case the vault configuration reference a non remote class.

    """
    if "SWH_CONFIG_FILENAME" in os.environ:
        cfg = read_config(os.environ["SWH_CONFIG_FILENAME"], DEFAULT_CONFIG)
    else:
        cfg = load_named_config(DEFAULT_CONFIG_PATH, DEFAULT_CONFIG)
    cooker_cls = get_cooker_cls(bundle_type, swhid.object_type)

    cfg = check_config(cfg)
    vcfg = cfg["vault"]

    storage = get_storage(**vcfg.pop("storage"))
    backend = get_vault(**vcfg)

    try:
        from swh.graph.client import RemoteGraphClient  # optional dependency

        graph = RemoteGraphClient(
            **vcfg["graph"]) if vcfg.get("graph") else None
    except ModuleNotFoundError:
        if vcfg.get("graph"):
            raise EnvironmentError(
                "Graph configuration required but module is not installed.")
        else:
            graph = None

    kwargs = {
        k: v
        for (k, v) in cfg.items()
        if k in ("max_bundle_size", "thread_pool_size")
    }

    return cooker_cls(
        swhid,
        backend=backend,
        storage=storage,
        graph=graph,
        **kwargs,
    )
Ejemplo n.º 4
0
def swh_vault(swh_vault_config):
    return get_vault("local", **swh_vault_config)
Ejemplo n.º 5
0
def test_init_get_vault_ok(swh_vault_config):
    concrete_vault = get_vault("local", **swh_vault_config)
    assert isinstance(concrete_vault, VaultBackend)
Ejemplo n.º 6
0
def test_init_get_vault_deprecation_warning(class_name, expected_class, kwargs,
                                            mock_psycopg2):
    with pytest.warns(DeprecationWarning):
        concrete_vault = get_vault(class_name, args=kwargs)
    assert isinstance(concrete_vault, expected_class)
Ejemplo n.º 7
0
def test_init_get_vault(class_name, expected_class, kwargs, mock_psycopg2):
    concrete_vault = get_vault(class_name, **kwargs)
    assert isinstance(concrete_vault, expected_class)
Ejemplo n.º 8
0
def test_init_get_vault_failure():
    with pytest.raises(ValueError, match="Unknown Vault class"):
        get_vault("unknown-vault-storage")