def result_filename(self, content_type):
dname = (Path(random.choice(self.result_dirs)) /
"{:02x}".format(random.randint(0, 0xFF)) /
"{:02x}".format(random.randint(0, 0xFF)))
dname.mkdir(parents=True, exist_ok=True)
fd, ret = tempfile.mkstemp(
dir=str(dname),
prefix="{}-".format(uuid.uuid4().hex),
suffix=extension_for(content_type),
)
os.close(fd)
return Path(ret)
def __init__(self, uwsgi_ini_file, servicelib_ini_file, pid_file, log_file,
scratch_dir):
self.servicelib_ini_file = servicelib_ini_file
self.pid_file = pid_file
self.log_file = log_file
self.host = "127.0.0.1"
self.port = utils.available_port()
servicelib_dir = Path(__file__, "..", "..").resolve()
services_dir = servicelib_dir / "samples"
self.uwsgi_ini = UWSGI_INI_TEMPLATE.format(
host=self.host,
log_file=log_file,
pid_file=pid_file,
port=self.port,
services_dir=services_dir,
)
with open(uwsgi_ini_file, "wt") as f:
f.write(self.uwsgi_ini)
self.servicelib_ini = SERVICELIB_INI_TEMPLATE.format(
host=self.host,
port=self.port,
scratch_dir=scratch_dir,
services_dir=services_dir,
uwsgi_config_file=uwsgi_ini_file,
)
with open(servicelib_ini_file, "wt") as f:
f.write(self.servicelib_ini)
scratch_dir.mkdir(parents=True, exist_ok=True)
def test_downloadable_result_as_argument(worker):
this_dir = str(Path(__file__, "..").resolve())
tar_file = worker.http_post("/services/tar-create",
data=json.dumps([this_dir]))
res = worker.http_post("/services/tar-list", data=json.dumps([tar_file]))
assert this_dir[1:] in res
def test_errors_in_service_loading(worker_cmd, services_dir, expected_error):
services_dir = Path(__file__, "..", "..", "samples", "invalid-services",
services_dir).resolve()
w = worker_cmd("--worker-services-dir={}".format(services_dir), )
with pytest.raises(Exception) as exc:
with w:
pass
assert expected_error in str(exc.value)
def create_temp_file(self):
dname = (Path(self.strategy.download_dir(self.scratch_dirs)) /
"{:02x}".format(random.randint(0, 0xFF)) /
"{:02x}".format(random.randint(0, 0xFF)))
dname.mkdir(parents=True, exist_ok=True)
fd, ret = tempfile.mkstemp(dir=str(dname),
prefix="{}-".format(uuid.uuid4().hex))
os.close(fd)
return ret
def as_local_file(self, result):
h = hashlib.sha256()
h.update(result["location"].encode("utf-8"))
# TODO: Update hash object with byte range fields, too, once they are
# implemented.
fname = h.hexdigest()
for dname in self.scratch_dirs:
path = os.path.join(dname, fname[0:2], fname[2:4], fname)
if os.access(path, os.F_OK):
self.log.debug("%s already downloaded, returning", path)
return Path(path)
dname = Path(random.choice(
self.scratch_dirs)) / fname[0:2] / fname[2:4]
dname.mkdir(parents=True, exist_ok=True)
path = dname / fname
self.log.debug("Downloading %s into %s", result, path)
download(result, path)
return path
def worker():
tmp_path = Path(tempfile.mkdtemp())
try:
with Worker(
tmp_path / "uwsgi.ini",
tmp_path / "servicelib.ini",
tmp_path / "uwsgi.pid",
tmp_path / "uwsgi.log",
tmp_path / "scratch",
) as s:
yield s
finally:
shutil.rmtree(str(tmp_path), ignore_errors=True)
def main():
logutils.configure_logging()
cmd = ["uwsgi"]
autoreload = int(config.get("worker_autoreload", "0"))
if autoreload > 0:
cmd.extend(["--py-autoreload", "{}".format(autoreload)])
serve_results = config.get("worker_serve_results", default=None)
if serve_results is not None:
for dname in serve_results.split(":"):
cmd.extend(["--static-map", "{}={}".format(dname, dname)])
swagger_yaml = Path(
config.get("worker_services_dir", default="/code/services"), "swagger.yaml"
)
if swagger_yaml.exists():
cmd.extend(["--static-map", "/services/swagger.yaml={}".format(swagger_yaml)])
swagger_ui = Path(
config.get("worker_swagger_ui_path", default="/usr/share/nginx/html")
)
if swagger_yaml.exists():
cmd.extend(["--static-map", "/docs={}".format(swagger_ui)])
cmd.extend(["--static-index", "index.html"])
try:
static_assets = config.get("worker_static_map")
except Exception:
pass
else:
cmd.extend(["--static-map", static_assets])
cmd.append(
config.get(
"worker_uwsgi_config_file",
default=str(Path(config.__file__, "..", "uwsgi.ini").resolve()),
)
)
os.environ.setdefault(
"SERVICELIB_WORKER_NUM_PROCESSES",
config.get("worker_num_processes", str(psutil.cpu_count())),
)
os.environ.setdefault(
"SERVICELIB_WORKER_NUM_THREADS", config.get("worker_num_threads", "1")
)
os.environ.setdefault("SERVICELIB_WORKER_PORT", config.get("worker_port", "8000"))
log = logutils.get_logger("servicelib-worker")
log.info("Running: %s", " ".join(cmd))
os.execlp(cmd[0], *cmd[0:])
def as_local_file(self, result):
ret = Path(urlparse(result["location"]).path)
for d in self.result_dirs:
try:
d = Path(d)
ret.relative_to(d)
st_size = ret.stat().st_size
if st_size == result["contentLength"]:
return ret
self.log.debug(
"as_local_file(%s): size %s does not match contentLength",
result,
st_size,
)
except Exception as exc:
self.log.info("as_local_file(%s): Not in %s: %s", result, d,
exc)
def test_result_as_local_file(local_results):
r = local_results.create("text/plain")
assert local_results.as_local_file(r.as_dict()) == Path(r.path)
def test_downloadable_result(worker):
this_dir = str(Path(__file__, "..").resolve())
res = worker.http_post("/services/tar-create", data=json.dumps([this_dir]))
tar_file = requests.get(res["location"])
assert len(tar_file.content) == res["contentLength"]
def servicelib_yaml(request, monkeypatch):
monkeypatch.setenv(*env_var(
"SERVICELIB_CONFIG_URL",
Path(__file__, "..", "sample-servicelib.yaml").resolve().as_uri(),
))
def __init__(self, uwsgi_ini_file, servicelib_yaml_file, pid_file,
log_file, scratch_dir, *cmdline_args):
self.servicelib_yaml_file = servicelib_yaml_file
self.pid_file = pid_file
self.log_file = log_file
self.host = "127.0.0.1"
self.port = utils.available_port()
# Some tests in `tests/test_client.py` call service `proxy`, which
# calls other services. We need several uWSGI processes to be ready
# to accept requests.
self.num_processes = 4
# Set to 1 because we're assuming `servicelib` (and the services built
# upon it) are not thread-safe.
#
# We do want to set it explicitly to 1, so that Python's threading
# machinery gets initialised.
self.num_threads = 1
scratch_dir.mkdir(parents=True, exist_ok=True)
servicelib_dir = Path(__file__, "..", "..").resolve()
scratch_dir = str(scratch_dir)
uwsgi_ini_file = str(uwsgi_ini_file)
for a in cmdline_args:
if a.startswith("--worker-services-dir="):
services_dir = a[len("--worker-services-dir="):]
break
else:
services_dir = str(servicelib_dir / "samples")
self.uwsgi_ini = UWSGI_INI_TEMPLATE.format(
host=self.host,
log_file=log_file,
num_processes=self.num_processes,
num_threads=self.num_threads,
pid_file=pid_file,
port=self.port,
services_dir=services_dir,
)
with open(uwsgi_ini_file, "wt") as f:
f.write(self.uwsgi_ini)
self.servicelib_conf = {
"worker": {
"hostname": self.host,
"port": self.port,
"serve_results": scratch_dir,
"services_dir": services_dir,
"static_map": "/services-source-code={}".format(services_dir),
"swagger_ui_path": "{}/swagger-ui".format(services_dir),
"uwsgi_config_file": uwsgi_ini_file,
},
"inventory": {
"class": "default",
},
"registry": {
"class": "redis",
"url": "redis://localhost/0",
},
"cache": {
"class": "memcached",
"memcached_addresses": ["localhost:11211"],
},
"log": {
"level": "debug",
"type": "text",
},
"results": {
"class": "http-files",
"dirs": [scratch_dir],
"http_hostname": self.host,
},
"scratch": {
"strategy": "random",
"dirs": [scratch_dir],
},
}
with open(servicelib_yaml_file, "wb") as f:
yaml.safe_dump(self.servicelib_conf,
f,
encoding="utf-8",
allow_unicode=True)
self.cmdline_args = cmdline_args
def main():
logutils.configure_logging()
cmdline_config = cmdline.parse_args(
"worker.autoreload",
"worker.hostname",
"worker.load_workers",
"worker.num_processes",
"worker.num_threads",
"worker.port",
"worker.services_dir",
)
for k, v in cmdline_config.items():
if isinstance(v, list):
v = json.dumps(v)
else:
v = str(v)
os.environ[env_var(k)] = v
cmd = ["uwsgi", "--req-logger", "file:/dev/null"]
autoreload = int(config.get("worker.autoreload", "0"))
if autoreload > 0:
cmd.extend(["--py-autoreload",
"{}".format(autoreload)]) # pragma: no cover
serve_results = config.get("worker.serve_results", default=None)
if serve_results is not None:
for dname in serve_results.split(":"):
cmd.extend(["--static-map", "{}={}".format(dname, dname)])
swagger_yaml = Path(
config.get("worker.services_dir", default="/code/services"),
"swagger.yaml")
if swagger_yaml.exists():
cmd.extend(
["--static-map", "/services/swagger.yaml={}".format(swagger_yaml)])
swagger_ui = Path(
config.get("worker.swagger_ui_path", default="/usr/share/nginx/html"))
if swagger_yaml.exists():
cmd.extend(["--static-map", "/docs={}".format(swagger_ui)])
cmd.extend(["--static-index", "index.html"])
static_assets = config.get("worker.static_map", default=None)
if static_assets is not None:
cmd.extend(["--static-map", static_assets])
cmd.append(
config.get(
"worker.uwsgi_config_file",
default=str(Path(logutils.__file__, "..", "uwsgi.ini").resolve()),
))
os.environ.setdefault(
env_var("worker.num_processes"),
config.get("worker.num_processes", str(psutil.cpu_count())),
)
os.environ.setdefault(env_var("worker.num_threads"),
str(config.get("worker.num_threads", 1)))
os.environ.setdefault(env_var("worker.port"),
str(config.get("worker.port", 8000)))
log = logutils.get_logger("servicelib-worker")
log.info("Environment: %s", os.environ)
log.info("Running: %s", " ".join(cmd))
# If we're running under `pytest-cov`, call `pytest_cov.embed.cleanup()`
# before exec of uWSGI, so that we do not lose coverage info for this
# Python module.
if os.environ.get("COV_CORE_DATAFILE"):
from pytest_cov.embed import cleanup
cleanup()
os.execlp(cmd[0], *cmd[0:])
