def default_inventory(monkeypatch):
samples_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "samples"))
monkeypatch.setenv(*env_var("SERVICELIB_INVENTORY_CLASS", "default"))
monkeypatch.setenv(*env_var("SERVICELIB_WORKER_SERVICES_DIR", samples_dir))
monkeypatch.syspath_prepend(samples_dir)
return inventory.instance()
def cache(request, monkeypatch):
monkeypatch.setenv(*env_var("SERVICELIB_CACHE_CLASS", "memcached"))
monkeypatch.setenv(*env_var("SERVICELIB_CACHE_MEMCACHED_ADDRESSES",
'["localhost:11211"]'))
c = cache_instance()
c.flush()
try:
yield c
finally:
c.flush()
def test_invalid_http_results_port(monkeypatch, tmp_path):
monkeypatch.setenv(
*env_var("SERVICELIB_RESULTS_HTTP_HOSTNAME", "localhost"))
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_DIRS",
json.dumps([str(tmp_path / "wherever")])))
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_HTTP_PORT", "pepe"))
with pytest.raises(Exception) as exc:
results.HttpFileResults()
assert str(
exc.value).startswith("Invalid config variable results.http_port=pepe")
def local_files_results(request, monkeypatch, tmp_path):
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_CLASS", "local-files"))
dirs = [tmp_path / d for d in ("scratch01", "scratch02")]
for d in dirs:
d.mkdir()
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_DIRS", ":".join(
str(d) for d in dirs)))
return results.LocalFileResults()
def http_files_results(request, monkeypatch, tmp_path):
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_CLASS", "http-files"))
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_HTTP_PORT", "8080"))
dirs = [tmp_path / d for d in ("scratch01", "scratch02")]
for d in dirs:
d.mkdir()
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_DIRS", ":".join(
str(d) for d in dirs)))
return results.HttpFileResults()
def cds_cache_results(request, monkeypatch, tmp_path):
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_CLASS", "cds-cache"))
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_CDS_STACK", "cds-pepe"))
# monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_HTTP_PORT", "8080"))
dirs = [tmp_path / d for d in ("scratch01", "scratch02")]
for d in dirs:
d.mkdir()
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_DIRS", ":".join(
str(d) for d in dirs)))
return results.CDSCacheResults()
def test_http_results_port_defaults_to_worker_port(monkeypatch, tmp_path):
monkeypatch.setenv(
*env_var("SERVICELIB_RESULTS_HTTP_HOSTNAME", "localhost"))
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_DIRS",
json.dumps([str(tmp_path / "wherever")])))
monkeypatch.setenv(*env_var("SERVICELIB_WORKER_PORT", "42"))
r = results.HttpFileResults().create("text/plain")
assert r.location.startswith("http://localhost:42/")
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_HTTP_PORT", "8000"))
r = results.HttpFileResults().create("text/plain")
assert r.location.startswith("http://localhost:8000/")
def context(request, monkeypatch, tmp_path):
home_dir = tmp_path / "service-home"
home_dir.mkdir()
scratch_dirs = [tmp_path / d for d in ("scratch01", "scratch02")]
for d in scratch_dirs:
d.mkdir()
scratch_dirs = ":".join(str(d) for d in scratch_dirs)
monkeypatch.setenv(*env_var("SERVICELIB_SCRATCH_DIRS", scratch_dirs))
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_CLASS", "local-files"))
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_DIRS", scratch_dirs))
return ServiceContext("some-service", str(home_dir), None, Request())
def broker(request, cache, worker, monkeypatch):
monkeypatch.setenv(*env_var("SERVICELIB_CONFIG_URL",
worker.servicelib_yaml_file.as_uri()))
monkeypatch.setenv(*env_var("SERVICELIB_REGISTRY_CLASS", "redis"))
monkeypatch.setenv(
*env_var("SERVICELIB_REGISTRY_URL", "redis://localhost/0"))
b = client.Broker()
b.worker_info = {
"num_processes": worker.num_processes,
"num_threads": worker.num_threads,
}
try:
yield b
finally:
b.http_session.close()
def cds_cache_results(request, monkeypatch, tmp_path):
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_CLASS", "cds-cache"))
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_CDS_DOWNLOAD_HOST",
"some-host.copernicus-climate.eu"))
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_CDS_DOWNLOAD_PATH_PREFIX",
"/cache-compute-0000/"))
dirs = [tmp_path / d for d in ("scratch01", "scratch02")]
for d in dirs:
d.mkdir()
monkeypatch.setenv(
*env_var("SERVICELIB_RESULTS_DIRS", json.dumps([str(d)
for d in dirs])))
return results.CDSCacheResults()
def test_invalid_strategy(monkeypatch):
monkeypatch.setenv(
*env_var("SERVICELIB_SCRATCH_STRATEGY", "invalid-strategy"))
with pytest.raises(Exception) as exc:
scratch.instance()
assert str(
exc.value) == "Invalid value for `scratch_strategy`: invalid-strategy"
def test_command_line_script(worker, monkeypatch, script_runner):
monkeypatch.setenv(*env_var("SERVICELIB_CONFIG_URL",
str(worker.servicelib_yaml_file.as_uri())))
r = script_runner.run("servicelib-client", "echo",
'["foo",{"bar":42,"baz":true}]')
assert r.success
assert json.loads(r.stdout) == [["foo", {"bar": 42, "baz": True}]]
r = script_runner.run("servicelib-client", "no-such-service")
assert not r.success
def test_delete_invalid_key(script_runner, config_server, monkeypatch):
config_server.initial_config = {"foo": 42}
config_server.start()
monkeypatch.setenv(
*env_var("SERVICELIB_CONFIG_URL", config_server.client.url))
r = script_runner.run("servicelib-config-client", "delete", "no-such-key")
assert not r.success
assert r.stdout == ""
assert r.stderr == "no-such-key: Not found\n"
def test_dump(script_runner, config_server, monkeypatch):
config_server.initial_config = {"foo": 42, "bar": {"baz": 42.0}}
config_server.start()
monkeypatch.setenv(
*env_var("SERVICELIB_CONFIG_URL", config_server.client.url))
r = script_runner.run("servicelib-config-client", "dump")
assert r.success
assert json.loads(r.stdout) == {"foo": 42, "bar": {"baz": 42.0}}
assert r.stderr == ""
def redis_registry(monkeypatch):
monkeypatch.setenv(
*env_var("SERVICELIB_REGISTRY_URL", "redis://localhost/0"))
r = registry.RedisRegistry()
try:
yield r
finally:
services_by_name = r.services_by_name()
service_urls = []
for name, urls in services_by_name.items():
service_urls.extend((name, url) for url in urls)
r.unregister(service_urls)
def test_value_from_env(monkeypatch):
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR_INT", "42"))
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR_FLOAT", "42.0"))
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR_TRUE1", "True"))
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR_TRUE2", "true"))
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR_FALSE1", "False"))
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR_FALSE2", "False"))
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR_LIST", '["one", {"two": 3}]'))
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR_DICT", '{"two": 3}'))
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR_PEPE", "just a string"))
assert config.get("foo.bar_int") == 42
assert config.get("foo.bar_float") == 42.0
assert config.get("foo.bar_true1")
assert config.get("foo.bar_true2")
assert not config.get("foo.bar_false1")
assert not config.get("foo.bar_false2")
assert config.get("foo.bar_list") == ["one", {"two": 3}]
assert config.get("foo.bar_dict") == {"two": 3}
assert config.get("foo.bar_pepe") == "just a string"
def test_get_invalid_key(script_runner, config_server, monkeypatch):
config_server.initial_config = {"foo": 42, "bar": {"baz": 42.0}}
config_server.start()
monkeypatch.setenv(
*env_var("SERVICELIB_CONFIG_URL", config_server.client.url))
r = script_runner.run("servicelib-config-client", "get", "no-such-key")
assert not r.success
assert r.stdout == ""
assert r.stderr.startswith("No config value for `no-such-key`")
r = script_runner.run("servicelib-config-client", "get", "bar.no-such-key")
assert not r.success
assert r.stderr.startswith("No config value for `bar.no-such-key`")
def test_set(script_runner, config_server, monkeypatch):
config_server.initial_config = {"foo": 42, "bar": {"baz": 42.0}}
config_server.start()
monkeypatch.setenv(
*env_var("SERVICELIB_CONFIG_URL", config_server.client.url))
r = script_runner.run("servicelib-config-client", "set", "foo", "43")
assert r.success
assert r.stdout == ""
assert r.stderr == ""
assert config_server.client.get("foo") == 43
r = script_runner.run("servicelib-config-client", "set", "moo",
'[true, false, {"bzz": 43.0}]')
assert r.success
assert r.stdout == ""
assert r.stderr == ""
assert config_server.client.get("moo") == [True, False, {"bzz": 43.0}]
def test_set_invalid_values(script_runner, config_server, monkeypatch):
config_server.initial_config = {"foo": 42}
config_server.start()
monkeypatch.setenv(
*env_var("SERVICELIB_CONFIG_URL", config_server.client.url))
r = script_runner.run("servicelib-config-client", "set", "foo",
"this-is-not-valid-json")
assert not r.success
assert r.stdout == ""
assert r.stderr == "Invalid JSON: <this-is-not-valid-json>\n"
assert config_server.client.get("foo") == 42
r = script_runner.run("servicelib-config-client", "set", "foo",
'"this-is-valid-json"')
assert r.success
assert r.stdout == ""
assert r.stderr == ""
assert config_server.client.get("foo") == "this-is-valid-json"
def test_delete(script_runner, config_server, monkeypatch):
config_server.initial_config = {"foo": 42, "bar": {"baz": 42.0}}
config_server.start()
monkeypatch.setenv(
*env_var("SERVICELIB_CONFIG_URL", config_server.client.url))
assert config_server.client.get("foo") == 42
r = script_runner.run("servicelib-config-client", "delete", "foo")
assert r.success
assert r.stdout == ""
assert r.stderr == ""
with pytest.raises(Exception) as exc:
config_server.client.get("foo")
assert str(exc.value).startswith("No config value for `foo`")
r = script_runner.run("servicelib-config-client", "delete", "bar.baz")
assert r.success
assert r.stdout == ""
assert r.stderr == ""
assert config_server.client.dump() == {"bar": {}}
def test_invalid_results_factory(monkeypatch):
monkeypatch.setenv(*env_var("SERVICELIB_RESULTS_CLASS", "no-such-impl"))
with pytest.raises(Exception) as exc:
results.instance()
assert str(exc.value) == "Invalid value for `results.class`: no-such-impl"
def servicelib_yaml(request, monkeypatch):
monkeypatch.setenv(*env_var(
"SERVICELIB_CONFIG_URL",
Path(__file__, "..", "sample-servicelib.yaml").resolve().as_uri(),
))
def test_value_from_env(monkeypatch):
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR", "42"))
assert config.get("foo_bar") == "42"
def main():
p = argparse.ArgumentParser()
p.add_argument(
"--log-file",
metavar="PATH",
help="path to the log file (default: %(default)s)",
default=None,
)
p.add_argument(
"--pid-file",
metavar="PATH",
help="path to the PID file (default: %(default)s)",
default=None,
)
p.add_argument(
"--port",
metavar="PORT",
type=int,
help="listening port (default: %(default)d)",
default=9999,
)
p.add_argument("--read-only",
action="store_true",
help="disable updates",
default=False)
p.add_argument(
"--config-file",
metavar="PATH",
help="path to the config file (default: %(default)s)",
default="servicelib.yaml",
)
args = p.parse_args()
cmd = [
"uwsgi",
"--die-on-term",
"--enable-threads",
"--http-socket",
":{}".format(args.port),
"--manage-script-name",
"--master",
"--module",
"servicelib.config.wsgi",
"--need-app",
"--processes",
"1",
"--req-logger",
"file:/dev/null",
"--threads",
"1",
]
if args.pid_file is not None:
cmd.extend([
"--safe-pidfile",
args.pid_file,
"--daemonize",
args.log_file is not None and args.log_file or "/dev/null",
])
os.environ.setdefault(
*env_var("SERVICELIB_CONFIG_FILE", os.path.abspath(args.config_file)))
os.environ.setdefault(*env_var("SERVICELIB_CONFIG_SERVER_READ_ONLY",
"true" if args.read_only else "false"))
# If we're running under `pytest-cov`, call `pytest_cov.embed.cleanup()`
# before exec of uWSGI, so that we do not lose coverage info for this
# Python module.
if os.environ.get("COV_CORE_DATAFILE"):
from pytest_cov.embed import cleanup
cleanup()
os.execlp(cmd[0], *cmd[0:])
def test_env_overrides_config_file(servicelib_yaml, monkeypatch):
monkeypatch.setenv(*env_var("SERVICELIB_WORKER_NUM_PROCESSES", "42"))
assert config.get("worker.num_processes") == 42
def test_env_overrides_default(monkeypatch):
monkeypatch.setenv(*env_var("SERVICELIB_FOO_BAR", "42"))
assert config.get("foo.bar", default="43") == 42
def test_invalid_inventory_factory(monkeypatch):
monkeypatch.setenv(*env_var("SERVICELIB_INVENTORY_CLASS", "no-such-impl"))
with pytest.raises(Exception) as exc:
inventory.instance()
assert str(
exc.value) == "Invalid value for `inventory.class`: no-such-impl"
def test_invalid_cache_factory(monkeypatch):
monkeypatch.setenv(*env_var("SERVICELIB_CACHE_CLASS", "no-such-impl"))
with pytest.raises(Exception) as exc:
instance()
assert str(exc.value) == "Invalid value for `cache.class`: no-such-impl"
