def __init__(self, uwsgi_ini_file, servicelib_ini_file, pid_file, log_file,
scratch_dir):
self.servicelib_ini_file = servicelib_ini_file
self.pid_file = pid_file
self.log_file = log_file
self.host = "127.0.0.1"
self.port = utils.available_port()
servicelib_dir = Path(__file__, "..", "..").resolve()
services_dir = servicelib_dir / "samples"
self.uwsgi_ini = UWSGI_INI_TEMPLATE.format(
host=self.host,
log_file=log_file,
pid_file=pid_file,
port=self.port,
services_dir=services_dir,
)
with open(uwsgi_ini_file, "wt") as f:
f.write(self.uwsgi_ini)
self.servicelib_ini = SERVICELIB_INI_TEMPLATE.format(
host=self.host,
port=self.port,
scratch_dir=scratch_dir,
services_dir=services_dir,
uwsgi_config_file=uwsgi_ini_file,
)
with open(servicelib_ini_file, "wt") as f:
f.write(self.servicelib_ini)
scratch_dir.mkdir(parents=True, exist_ok=True)
def __exit__(self, *exc_info):
# self.log.debug("uwsgi.ini:\n%s", self.uwsgi_ini)
# self.log.debug("servicelib.ini:\n%s", self.servicelib_ini)
try:
with open(self.log_file, "rt") as f:
self.log.debug("uwsgi.log:\n%s", f.read())
except Exception:
pass
with open(self.pid_file, "rt") as f:
pid = int(f.read().strip())
os.kill(pid, signal.SIGTERM)
def start(self, background=True):
with open(self.config_file, "wb") as f:
yaml.safe_dump(self.initial_config,
f,
encoding="utf-8",
allow_unicode=True)
cmdline = [
"servicelib-config-server",
"--port",
self.port,
"--log-file",
self.log_file,
"--config-file",
self.config_file,
]
if background:
cmdline.extend([
"--pid-file",
self.pid_file,
])
if self.read_only:
cmdline.append("--read-only")
cmdline = " ".join(str(c) for c in cmdline)
p = subprocess.Popen(cmdline, shell=True)
if background:
rc = p.wait()
if rc:
raise Exception("Error running {}".format(cmdline))
else:
self.p = p
utils.wait_for_url(self.client.url)
def __enter__(self):
env = dict(os.environ)
env["SERVICELIB_CONFIG_URL"] = self.servicelib_yaml_file.resolve(
).as_uri()
cmdline = ["servicelib-worker"]
cmdline.extend(self.cmdline_args)
p = subprocess.Popen(
" ".join(cmdline),
shell=True,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, _ = p.communicate()
if p.returncode:
raise Exception(stdout)
try:
utils.wait_for_url("http://{}:{}/health".format(
self.host, self.port))
except Exception as exc:
try:
with open(self.log_file, "rt") as f:
exc = Exception(f.read())
except Exception:
pass
raise exc
return self
def stop(self):
try:
if self.p is not None:
self.p.terminate()
self.p.wait()
else:
with open(self.pid_file, "rt") as f:
pid = int(f.read().strip())
os.kill(pid, signal.SIGTERM)
except Exception:
pass
try:
with open(self.log_file, "rt") as f:
self.log.debug("uwsgi.log:\n%s", f.read())
except Exception:
pass
def test_handle_signals_in_spawn_process(context, tmp_path):
script = tmp_path / "script.py"
with open(script, "wt") as f:
f.write(SLEEP_PY)
pid_file = tmp_path / "script.pid"
cmdline = [sys.executable, str(script), str(pid_file)]
class p(process.Process):
def __init__(self):
super(p, self).__init__("sleep", cmdline)
def results(self):
return self.output.decode("utf-8")
result = []
def spawn():
try:
res = context.spawn_process(p())
result.append(res)
except Exception as exc:
result.append(exc)
t = threading.Thread(target=spawn)
t.start()
time.sleep(1)
with open(pid_file, "rt") as f:
pid = int(f.read().strip())
os.kill(pid, signal.SIGTERM)
t.join()
assert isinstance(result[0], Exception)
assert str(result[0]).startswith("'sleep' killed by signal")
def test_interleaved_output_in_spawn_process(context, tmp_path):
script = tmp_path / "script.py"
with open(script, "wt") as f:
f.write(STDERR_PY)
pid_file = tmp_path / "script.pid"
cmdline = [sys.executable, str(script), str(pid_file)]
class p(process.Process):
def __init__(self):
super(p, self).__init__("stderr", cmdline)
def results(self):
return self.output.decode("utf-8")
res = context.spawn_process(p())
assert res == "foo\nbar\nbaz\n"
def test_spawn_process_with_unlimited_output(context, tmp_path):
zeroes = tmp_path / "zeroes"
cmdline = ["cat", str(zeroes)]
class p(process.Process):
max_output_size = 0
def __init__(self):
super(p, self).__init__("cat-zeores", cmdline)
def results(self):
return self.output.decode("utf-8")
with open(zeroes, "wb") as f:
f.write(("0" * process.DEFAULT_MAX_PROCESS_OUTPUT_SIZE).encode("utf-8"))
res = context.spawn_process(p())
assert res == "0" * zeroes.stat().st_size
def test_spawn_process_with_truncated_output(context, tmp_path):
zeroes = tmp_path / "zeroes"
cmdline = ["cat", str(zeroes)]
class p(process.Process):
max_output_size = 42
def __init__(self):
super(p, self).__init__("cat-zeroes", cmdline)
def results(self):
return self.output.decode("utf-8")
with open(zeroes, "wb") as f:
f.write(("0" * p.max_output_size).encode("utf-8"))
f.write("and some extra data which will be truncated".encode("utf-8"))
res = context.spawn_process(p())
assert res == "0" * p.max_output_size
def test_spawn_process_with_truncated_output_2(context, tmp_path):
foo_bar = tmp_path / "foo-bar"
cmdline = ["cat", str(foo_bar)]
# The newline in the output will trigger two calls to the process object's
# `stdout_data`, the first one of which fill fill the output capacity.
with open(foo_bar, "wb") as f:
f.write("foo\nbar".encode("utf-8"))
class p(process.Process):
max_output_size = 3
def __init__(self):
super(p, self).__init__("cat-foo-bar", cmdline)
def results(self):
return self.output.decode("utf-8")
res = context.spawn_process(p())
assert res == "foo"
def _open(self):
self._file_obj = open(self._path, "wb", buffering=0)
def test_diff(tmp_path, script_runner):
config_a = tmp_path / "config-a.yaml"
with open(config_a, "wb") as f:
yaml.safe_dump(
{
"foo": 24,
"bzzz": 17,
"bar": {
"bazz": [
True,
False,
],
"boo": [
"some-string",
{
"a-float": 42.0
},
["one", "nested", "list"],
],
},
"boo": True,
},
f,
encoding="utf-8",
allow_unicode=True,
)
config_b = tmp_path / "config-b.yaml"
with open(config_b, "wb") as f:
yaml.safe_dump(
{
"bar": {
"bazz": [
False,
True,
],
"boo": [
"some-string",
{
"a-float": 42.0
},
["one", "nested", "list", "but", "longer"],
],
"moo":
24,
},
"foo": 42,
"boo": True,
},
f,
encoding="utf-8",
allow_unicode=True,
)
expected = _EXPECTED_DIFF.format(config_a=config_a.as_uri(),
config_b=config_b.as_uri()).strip()
r = script_runner.run("servicelib-config-client", "diff",
config_a.as_uri(), config_b.as_uri())
assert r.success
assert r.stdout.strip() == expected
assert r.stderr == ""
def __init__(self, uwsgi_ini_file, servicelib_yaml_file, pid_file,
log_file, scratch_dir, *cmdline_args):
self.servicelib_yaml_file = servicelib_yaml_file
self.pid_file = pid_file
self.log_file = log_file
self.host = "127.0.0.1"
self.port = utils.available_port()
# Some tests in `tests/test_client.py` call service `proxy`, which
# calls other services. We need several uWSGI processes to be ready
# to accept requests.
self.num_processes = 4
# Set to 1 because we're assuming `servicelib` (and the services built
# upon it) are not thread-safe.
#
# We do want to set it explicitly to 1, so that Python's threading
# machinery gets initialised.
self.num_threads = 1
scratch_dir.mkdir(parents=True, exist_ok=True)
servicelib_dir = Path(__file__, "..", "..").resolve()
scratch_dir = str(scratch_dir)
uwsgi_ini_file = str(uwsgi_ini_file)
for a in cmdline_args:
if a.startswith("--worker-services-dir="):
services_dir = a[len("--worker-services-dir="):]
break
else:
services_dir = str(servicelib_dir / "samples")
self.uwsgi_ini = UWSGI_INI_TEMPLATE.format(
host=self.host,
log_file=log_file,
num_processes=self.num_processes,
num_threads=self.num_threads,
pid_file=pid_file,
port=self.port,
services_dir=services_dir,
)
with open(uwsgi_ini_file, "wt") as f:
f.write(self.uwsgi_ini)
self.servicelib_conf = {
"worker": {
"hostname": self.host,
"port": self.port,
"serve_results": scratch_dir,
"services_dir": services_dir,
"static_map": "/services-source-code={}".format(services_dir),
"swagger_ui_path": "{}/swagger-ui".format(services_dir),
"uwsgi_config_file": uwsgi_ini_file,
},
"inventory": {
"class": "default",
},
"registry": {
"class": "redis",
"url": "redis://localhost/0",
},
"cache": {
"class": "memcached",
"memcached_addresses": ["localhost:11211"],
},
"log": {
"level": "debug",
"type": "text",
},
"results": {
"class": "http-files",
"dirs": [scratch_dir],
"http_hostname": self.host,
},
"scratch": {
"strategy": "random",
"dirs": [scratch_dir],
},
}
with open(servicelib_yaml_file, "wb") as f:
yaml.safe_dump(self.servicelib_conf,
f,
encoding="utf-8",
allow_unicode=True)
self.cmdline_args = cmdline_args