class JSONDecoder(json.JSONDecoder):
"""Decodes JSON objects with a ``_type`` field as objects of that Python
class.
"""
log = logutils.get_logger(__name__)
def __init__(self, *args, **kwargs):
"""Constructor.
Overrides keyword argument ``object_hook``.
"""
def _hook(obj):
if isinstance(obj, dict) and "_type" in obj:
tname = str(obj["_type"])
bits = tname.split(".")
modname = ".".join(bits[:-1])
clsname = bits[-1]
mod = importlib.import_module(modname)
cls = mod.__dict__[clsname]
del obj["_type"]
return cls(**obj)
return obj
kwargs["object_hook"] = _hook
json.JSONDecoder.__init__(self, *args, **kwargs)
def __init__(self, name, home, metadata, request):
super(ServiceContext, self).__init__(name, metadata)
self._name = name
self.home = home
self.request = request
self._temp_files = set()
self._results = results.instance()
self._scratch = scratch.instance()
self.log = logutils.get_logger(name).bind(user=self.uid,
tracker=self.tracker)
for k, v in self.request.kwargs.items():
self.annotate(k, v)
def main():
logutils.configure_logging()
cmd = ["uwsgi"]
autoreload = int(config.get("worker_autoreload", "0"))
if autoreload > 0:
cmd.extend(["--py-autoreload", "{}".format(autoreload)])
serve_results = config.get("worker_serve_results", default=None)
if serve_results is not None:
for dname in serve_results.split(":"):
cmd.extend(["--static-map", "{}={}".format(dname, dname)])
swagger_yaml = Path(
config.get("worker_services_dir", default="/code/services"), "swagger.yaml"
)
if swagger_yaml.exists():
cmd.extend(["--static-map", "/services/swagger.yaml={}".format(swagger_yaml)])
swagger_ui = Path(
config.get("worker_swagger_ui_path", default="/usr/share/nginx/html")
)
if swagger_yaml.exists():
cmd.extend(["--static-map", "/docs={}".format(swagger_ui)])
cmd.extend(["--static-index", "index.html"])
try:
static_assets = config.get("worker_static_map")
except Exception:
pass
else:
cmd.extend(["--static-map", static_assets])
cmd.append(
config.get(
"worker_uwsgi_config_file",
default=str(Path(config.__file__, "..", "uwsgi.ini").resolve()),
)
)
os.environ.setdefault(
"SERVICELIB_WORKER_NUM_PROCESSES",
config.get("worker_num_processes", str(psutil.cpu_count())),
)
os.environ.setdefault(
"SERVICELIB_WORKER_NUM_THREADS", config.get("worker_num_threads", "1")
)
os.environ.setdefault("SERVICELIB_WORKER_PORT", config.get("worker_port", "8000"))
log = logutils.get_logger("servicelib-worker")
log.info("Running: %s", " ".join(cmd))
os.execlp(cmd[0], *cmd[0:])
class Response(object):
log = logutils.get_logger(__name__)
def __init__(self, value, metadata, encoded=None):
self.value = value
self.metadata = metadata
self._encoded_body = encoded
@property
def http_status(self):
if hasattr(self.value, "http_response_code"):
return self.value.http_response_code
return "200 OK"
@property
def http_headers(self):
return {
"x-servicelib-{}".format(k): v
for (k, v) in self.metadata.as_http_headers().items()
}
@property
def http_body(self):
if self._encoded_body is None:
self._encoded_body = json.dumps(self.value).encode("utf-8")
return self._encoded_body
@classmethod
def from_http(cls, status, body, headers):
cls.log.debug("from_http(status=%s, body=<%s>): Entering", status,
body)
body_decoded = json.loads(body)
if status.startswith("200 "):
value = body_decoded
else:
value = errors.Serializable.from_dict(body_decoded)
metadata = Metadata.from_http_headers({
k[len("x-servicelib-"):]: v
for (k, v) in headers.items() if k.startswith("x-servicelib-")
})
return cls(value, metadata, body)
def __repr__(self):
return "Response(value={!r}, metadata={!r})".format(
self.value, self.metadata)
def __eq__(self, other):
if isinstance(other, Response):
return self.value == other.value and self.metadata == other.metadata
return False # pragma: no cover
class NoOpCache(Cache):
log = logutils.get_logger(__name__)
def get(self, key):
return None
def set(self, key, value, ttl):
pass
def delete(self, key):
pass
def flush(self):
pass
class Cache(object):
log = logutils.get_logger(__name__)
def get(self, key):
raise NotImplementedError
def set(self, key, value, ttl):
raise NotImplementedError
def delete(self, key):
raise NotImplementedError
def flush(self):
raise NotImplementedError
def get_response(self, key):
"""Return the cached response object associated with the given key
from the results cache, or `None` if no such response was found.
"""
ret = self.get(key)
if ret is None or ret == IN_FLIGHT:
self.log.debug(
"get_response(%s): Got `%s` from cache, retuning `None`", key, ret
)
return None
try:
ret = json.loads(ret)
except Exception as exc:
self.log.error(
"Cannot decode JSON object <%s> for key '%s': %s",
ret,
key,
exc,
exc_info=True,
stack_info=True,
)
# There is no point in keeping this cached value if we cannot
# decode it.
self.delete(key)
return
self.log.debug("get_response(%s): Returning %s", key, ret)
return ret
class LocalFileResults(Results):
log = logutils.get_logger(__name__)
def __init__(self):
pass
def create(self, content_type):
return LocalFileResult(self.result_filename(content_type),
content_type)
def as_local_file(self, result):
ret = Path(urlparse(result["location"]).path)
for d in self.result_dirs:
try:
d = Path(d)
ret.relative_to(d)
st_size = ret.stat().st_size
if st_size == result["contentLength"]:
return ret
self.log.debug(
"as_local_file(%s): size %s does not match contentLength",
result,
st_size,
)
except Exception as exc:
self.log.info("as_local_file(%s): Not in %s: %s", result, d,
exc)
def result_filename(self, content_type):
dname = (Path(random.choice(self.result_dirs)) /
"{:02x}".format(random.randint(0, 0xFF)) /
"{:02x}".format(random.randint(0, 0xFF)))
dname.mkdir(parents=True, exist_ok=True)
fd, ret = tempfile.mkstemp(
dir=str(dname),
prefix="{}-".format(uuid.uuid4().hex),
suffix=extension_for(content_type),
)
os.close(fd)
return Path(ret)
@property
def result_dirs(self):
return config.get("results.dirs")
class Inventory(object):
log = logutils.get_logger(__name__)
def service_modules(self):
raise NotImplementedError
def load_services(self):
for mod_name in sorted(self.service_modules()):
self.log.debug("Loading service module: %s", mod_name)
mod = importlib.import_module(mod_name)
self.log.debug("Registering services in module: %s", mod_name)
mod.main()
ret = service_instances()
self.log.debug("Services: %s", ", ".join(sorted(ret.keys())))
return ret
def __init__(self, name, metadata=None, uid=None, **kwargs):
super(ClientContext, self).__init__(name, metadata)
if uid is None:
uid = self.default_uid
self._uid = uid
for k, v in kwargs.items():
self.annotate(k, v)
self.tracker = kwargs.get("tracker", core.tracker())
bind = {"uid": self._uid, "tracker": self.tracker}
log_name = kwargs.get("log_name")
if log_name is not None:
self.log = logutils.get_logger(log_name)
self.log = self.log.bind(**bind)
class Context(object):
log = logutils.get_logger(__name__)
def __init__(self, name, metadata=None):
assert isinstance(name, compat.string_types)
self._name = name
self._broker = None
if metadata is None:
self._metadata = Metadata(name)
else:
assert isinstance(metadata, Metadata)
self._metadata = metadata
def update_metadata(self, other):
self._metadata.update_metadata(other)
def annotate(self, *args, **kwargs):
return self._metadata.annotate(*args, **kwargs)
@property
def metadata(self):
return self._metadata
@property
def broker(self):
# XXX Put the `import` statement here in order to avoid a circular
# import
from servicelib.client import Broker
if self._broker is None:
self._broker = Broker(self)
return self._broker
@property
def name(self):
return self._name
def timer(self, name):
return self._metadata.timer(name)
@property
def default_uid(self):
return _DEFAULT_UID
class RedisPool(object):
log = logutils.get_logger(__name__)
def __init__(self):
self._pool = None
self._lock = threading.RLock()
@property
def pool(self):
with self._lock:
if self._pool is None:
url = config.get("registry.url")
self._pool = redis.ConnectionPool.from_url(url)
self.log.debug("Initialized Redis connection pool for URL %s", url)
return self._pool
def connection(self):
return redis.Redis(connection_pool=self.pool)
class DefaultScratch(Scratch):
log = logutils.get_logger(__name__)
def __init__(self, strategy):
self.strategy = strategy
def create_temp_file(self):
dname = (Path(self.strategy.download_dir(self.scratch_dirs)) /
"{:02x}".format(random.randint(0, 0xFF)) /
"{:02x}".format(random.randint(0, 0xFF)))
dname.mkdir(parents=True, exist_ok=True)
fd, ret = tempfile.mkstemp(dir=str(dname),
prefix="{}-".format(uuid.uuid4().hex))
os.close(fd)
return ret
def as_local_file(self, result):
h = hashlib.sha256()
h.update(result["location"].encode("utf-8"))
# TODO: Update hash object with byte range fields, too, once they are
# implemented.
fname = h.hexdigest()
for dname in self.scratch_dirs:
path = os.path.join(dname, fname[0:2], fname[2:4], fname)
if os.access(path, os.F_OK):
self.log.debug("%s already downloaded, returning", path)
return Path(path)
dname = Path(random.choice(
self.scratch_dirs)) / fname[0:2] / fname[2:4]
dname.mkdir(parents=True, exist_ok=True)
path = dname / fname
self.log.debug("Downloading %s into %s", result, path)
download(result, path)
return path
@property
def scratch_dirs(self):
return config.get("scratch_dirs").split(":")
class ConfigServerMiddleware(object):
log = logutils.get_logger("config-server")
def __init__(self, prefix):
self.prefix = prefix
def process_request(self, req, resp):
"""'Translate the key in the HTTP path, so that slashes get
transformed in dots.
This is a workaround for a Falcon limitation:
https://github.com/falconry/falcon/issues/648
This should be done in the `ConfigServer` object.
"""
if req.path.startswith(self.prefix):
key = req.path[len(self.prefix) + 1:].replace("/", ".")
req.path = "{}/{}".format(self.prefix, key)
def wrapped_f(context, *args, **kwargs):
if not context.request.kwargs.get("cache", True):
self.annotate(context, status="off")
return f(context, *args, **kwargs)
with context.timer("cache") as timer:
if context.name is not None:
service_name = context.name
else:
service_name = f.func_name
assert service_name
request = (service_name, args, list(kwargs.items()))
try:
request_encoded = json.dumps(request, sort_keys=True).encode(
"utf-8"
)
request_md5 = hashlib.md5(request_encoded).hexdigest()
status, response = self.state_loop(
context, request_md5, timer, f, args, kwargs
)
except Exception as exc:
try:
log = context.log
except Exception:
log = logutils.get_logger(__name__)
log.warn(
"cache_control: Error handling request: %s",
exc,
exc_info=True,
stack_info=True,
)
self.cache.delete(request_md5)
raise
self.annotate(context, status, request_md5)
return response
class WorkerResource(object):
log = logutils.get_logger(__name__)
def __init__(self, service_instances):
self.service_instances = service_instances
def on_post(self, req, resp, service):
try:
svc = self.service_instances[service]
except KeyError:
self.log.error("Unknown service '%s'", service)
raise falcon.HTTPNotFound()
if req.content_type and "application/json" not in req.content_type:
self.log.error("Unsupported request content type '%s'", req.content_type)
raise falcon.HTTPUnsupportedMediaType()
try:
body = req.bounded_stream.read()
headers = req.headers
svc_req = Request.from_http(body, headers)
except Exception as exc:
self.log.error(
"Bad request (body: %s, headers: %s): %s", body, headers, exc
)
exc = errors.BadRequest(str(exc))
resp.status = exc.http_response_code
resp.data = json.dumps(exc.as_dict()).encode("utf-8")
self.log.debug("Response body: %s", resp.data)
else:
svc_resp = svc._execute(svc_req)
resp.status = svc_resp.http_status
resp.data = svc_resp.http_body
for k, v in svc_resp.http_headers.items():
resp.append_header(k, v)
class Serializable(Exception):
"""Class for serializable errors."""
log = logutils.get_logger(__name__)
def __init__(self, *args):
super(Serializable, self).__init__(*args)
self.service = None
self.origin = None
def as_dict(self):
cls = self.__class__
return {
"exc_type": "{}.{}".format(cls.__module__, cls.__name__),
"exc_args": self.args,
"exc_service": self.service,
"exc_origin": self.origin,
}
@classmethod
def from_dict(cls, d):
bits = d["exc_type"].split(".")
exc_module, exc_name = ".".join(bits[:-1]), bits[-1]
try:
exc_cls = getattr(sys.modules[exc_module], exc_name)
except KeyError:
exc_cls = cls
cls.log.debug(
"Class '%s' not found in module '%s', deserializing as %s",
exc_name,
exc_module,
exc_cls,
)
else:
from_dict = getattr(exc_cls, "from_dict", None)
if callable(from_dict):
if six.PY2:
if getattr(from_dict, "im_self", None) != cls:
cls.log.debug("from_dict(%s): Delegating to %s", d,
from_dict)
return from_dict(d)
else:
if from_dict.__qualname__ != "Serializable.from_dict":
cls.log.debug("from_dict(%s): Delegating to %s", d,
from_dict)
return from_dict(d)
exc = exc_cls(*d.get("exc_args", []))
if isinstance(exc, Serializable):
exc.service = d["exc_service"]
exc.origin = d["exc_origin"]
return exc
@property
def message(self):
try:
return self.args[0]
except IndexError:
return None
def __repr__(self):
return "{}({})".format(self.__class__.__name__,
", ".join(repr(a) for a in self.args))
def __str__(self):
return str(self.message)
def __eq__(self, other):
if other.__class__ == self.__class__:
return self.as_dict() == other.as_dict()
return False
def __hash__(self):
return hash(json.dumps(self.as_dict()))
class TaskError(Serializable):
"""Wrapper for errors raised by service implementations.
Instances of `TaskError` are raised by *client* code calling services. They
are not meant to be raised explicitly by service implementations (i.e. the
constructor should be treated as private).
"""
http_response_code = "500 Internal Server Error"
log = logutils.get_logger(__name__)
def __init__(self, service, exc_type, exc_value, exc_tb, origin=None):
if origin is None:
origin = HOSTNAME
super(TaskError, self).__init__(service, exc_type, exc_value, exc_tb,
origin)
self.origin = origin
self.service = service
self._exc_type = exc_type
self._exc_value = exc_value
try:
self._exc_tb = traceback.format_tb(exc_tb)
except Exception:
self._exc_tb = exc_tb
def as_dict(self):
d = super(TaskError, self).as_dict()
# Arguments to our constructor are not serializable, so ensure we
# do not send them.
del d["exc_args"]
# If the exception args can be serialized, use them
# unmodified. Otherwise fall back to their string
# representations.
exc_args = []
for arg in self._exc_value.args:
try:
json.dumps(arg)
except TypeError:
exc_args.append(repr(arg))
else:
exc_args.append(arg)
d.update({
"wrapped_exc_type":
"%s.%s" % (self._exc_type.__module__, self._exc_type.__name__),
"wrapped_exc_args":
exc_args,
"wrapped_exc_tb":
self._exc_tb,
})
return d
@classmethod
def from_dict(cls, d):
bits = d["wrapped_exc_type"].split(".")
exc_module, exc_name = ".".join(bits[:-1]), bits[-1]
try:
exc_type = getattr(sys.modules[exc_module], exc_name)
except KeyError:
cls.log.debug("%s not found, deserializing as 'Exception'",
exc_name)
exc_type = Exception
try:
exc_value = exc_type(*d["wrapped_exc_args"])
except Exception:
cls.log.debug(
"Cannot create %s(%s), exc_value will be string",
exc_name,
d["wrapped_exc_args"],
)
exc_value = Exception(*d["wrapped_exc_args"])
return TaskError(
service=d["exc_service"],
origin=d["exc_origin"],
exc_type=exc_type,
exc_value=exc_value,
exc_tb=d["wrapped_exc_tb"],
)
@property
def exc_type(self):
"""Class of the original error, if known, or `exceptions.Exception`
otherwise.
"""
return self._exc_type
@property
def exc_value(self):
"""Instance of the original error, if available, or instance of
exceptions.Exception` with a message value of the original error
otherwise.
"""
return self._exc_value
@property
def exc_tb(self):
"""List of strings describing the traceback of the original error.
"""
return self._exc_tb
@property
def retry(self):
return False
def __repr__(self):
return "TaskError(%s)" % (self.as_dict(), )
def __str__(self):
return "%s: %s" % (self.__class__.__name__, self.exc_value)
class Worker(object):
log = logutils.get_logger()
def __init__(self, uwsgi_ini_file, servicelib_yaml_file, pid_file,
log_file, scratch_dir, *cmdline_args):
self.servicelib_yaml_file = servicelib_yaml_file
self.pid_file = pid_file
self.log_file = log_file
self.host = "127.0.0.1"
self.port = utils.available_port()
# Some tests in `tests/test_client.py` call service `proxy`, which
# calls other services. We need several uWSGI processes to be ready
# to accept requests.
self.num_processes = 4
# Set to 1 because we're assuming `servicelib` (and the services built
# upon it) are not thread-safe.
#
# We do want to set it explicitly to 1, so that Python's threading
# machinery gets initialised.
self.num_threads = 1
scratch_dir.mkdir(parents=True, exist_ok=True)
servicelib_dir = Path(__file__, "..", "..").resolve()
scratch_dir = str(scratch_dir)
uwsgi_ini_file = str(uwsgi_ini_file)
for a in cmdline_args:
if a.startswith("--worker-services-dir="):
services_dir = a[len("--worker-services-dir="):]
break
else:
services_dir = str(servicelib_dir / "samples")
self.uwsgi_ini = UWSGI_INI_TEMPLATE.format(
host=self.host,
log_file=log_file,
num_processes=self.num_processes,
num_threads=self.num_threads,
pid_file=pid_file,
port=self.port,
services_dir=services_dir,
)
with open(uwsgi_ini_file, "wt") as f:
f.write(self.uwsgi_ini)
self.servicelib_conf = {
"worker": {
"hostname": self.host,
"port": self.port,
"serve_results": scratch_dir,
"services_dir": services_dir,
"static_map": "/services-source-code={}".format(services_dir),
"swagger_ui_path": "{}/swagger-ui".format(services_dir),
"uwsgi_config_file": uwsgi_ini_file,
},
"inventory": {
"class": "default",
},
"registry": {
"class": "redis",
"url": "redis://localhost/0",
},
"cache": {
"class": "memcached",
"memcached_addresses": ["localhost:11211"],
},
"log": {
"level": "debug",
"type": "text",
},
"results": {
"class": "http-files",
"dirs": [scratch_dir],
"http_hostname": self.host,
},
"scratch": {
"strategy": "random",
"dirs": [scratch_dir],
},
}
with open(servicelib_yaml_file, "wb") as f:
yaml.safe_dump(self.servicelib_conf,
f,
encoding="utf-8",
allow_unicode=True)
self.cmdline_args = cmdline_args
def __enter__(self):
env = dict(os.environ)
env["SERVICELIB_CONFIG_URL"] = self.servicelib_yaml_file.resolve(
).as_uri()
cmdline = ["servicelib-worker"]
cmdline.extend(self.cmdline_args)
p = subprocess.Popen(
" ".join(cmdline),
shell=True,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
stdout, _ = p.communicate()
if p.returncode:
raise Exception(stdout)
try:
utils.wait_for_url("http://{}:{}/health".format(
self.host, self.port))
except Exception as exc:
try:
with open(self.log_file, "rt") as f:
exc = Exception(f.read())
except Exception:
pass
raise exc
return self
def __exit__(self, *exc_info):
# self.log.debug("uwsgi.ini:\n%s", self.uwsgi_ini)
# self.log.debug("servicelib.ini:\n%s", self.servicelib_ini)
try:
with open(self.log_file, "rt") as f:
self.log.debug("uwsgi.log:\n%s", f.read())
except Exception:
pass
try:
with open(self.pid_file, "rt") as f:
pid = int(f.read().strip())
os.kill(pid, signal.SIGTERM)
except Exception:
pass
def http_post(self, path, **kwargs):
res = requests.post(
"http://{}:{}{}".format(self.host, self.port, path), **kwargs)
if res.status_code == 200:
return res.json()
try:
err = res.json()
except Exception:
raise Exception("{}\n\n{}".format(res.status_code, res.content))
try:
err = errors.Serializable.from_dict(err)
except Exception:
raise Exception("{}\n\n{}".format(res.status_code, res.content))
else:
raise err
def http_get(self, path, **kwargs):
return requests.get(
"http://{}:{}{}".format(self.host, self.port, path), **kwargs)
class ConfigDict(object):
log = logutils.get_logger(__name__)
def __init__(self, values=None):
if values is None:
values = {}
else:
values = copy.deepcopy(values)
self._values = values
def get(self, key):
"""Returns the value for the given key, raising ``KeyError`` if the
key is not found.
"""
bits = key.split(".")
try:
b = int(bits[-1], base=10)
except ValueError:
pass
else:
if b < 0:
raise KeyError(key)
values = self.get(".".join(bits[:-1]))
try:
return values[b]
except Exception:
raise KeyError(key)
values = self._values
while bits:
b = bits.pop(0)
if b not in values:
self.log.debug("Key %s not found in %s", key, self._values)
raise KeyError(key)
if not bits:
return values[b]
values = values[b]
def set(self, key, value):
"""Sets the value for the given key.
"""
if not key:
raise ValueError("Invalid key `{}`".format(key))
bits = key.split(".")
values = self._values
prev = None
prev_b = None
while bits:
b = bits.pop(0)
if not bits:
try:
b = int(b, base=10)
if b < 0:
raise KeyError("Invalid key `{}`".format(key))
values = prev[prev_b]
if type(values) is not list:
values = prev[prev_b] = []
values.extend([None] * (b + 1 - len(values)))
except ValueError:
pass
except KeyError as exc:
raise ValueError(exc.args[0])
values[b] = copy.deepcopy(value)
else:
prev = values
prev_b = b
values = values.setdefault(b, {})
def reset(self, new_values):
"""Resets the values.
"""
self._values = copy.deepcopy(new_values)
def delete(self, key):
"""Deletes the value for the given key, raising ``KeyError`` if the
key is not found.
"""
bits = key.split(".")
values = self._values
while bits:
b = bits.pop(0)
print("b: {}".format(b))
if not bits:
try:
b = int(b, base=10)
if b < 0:
raise KeyError("Invalid key `{}`".format(key))
except KeyError as exc:
raise ValueError(exc.args[0])
except ValueError:
pass
try:
del values[b]
except TypeError:
raise KeyError("Invalid key `{}`".format(key))
return
if b not in values:
raise KeyError(key)
values = values[b]
def as_dict(self):
"""Returns a dict version of the keys and values.
"""
return copy.deepcopy(self._values)
def __deepcopy__(self, memo):
ret = self.__class__()
ret._values = copy.deepcopy(self._values, memo)
return ret
class Worker(object):
log = logutils.get_logger()
def __init__(self, uwsgi_ini_file, servicelib_ini_file, pid_file, log_file,
scratch_dir):
self.servicelib_ini_file = servicelib_ini_file
self.pid_file = pid_file
self.log_file = log_file
self.host = "127.0.0.1"
self.port = utils.available_port()
servicelib_dir = Path(__file__, "..", "..").resolve()
services_dir = servicelib_dir / "samples"
self.uwsgi_ini = UWSGI_INI_TEMPLATE.format(
host=self.host,
log_file=log_file,
pid_file=pid_file,
port=self.port,
services_dir=services_dir,
)
with open(uwsgi_ini_file, "wt") as f:
f.write(self.uwsgi_ini)
self.servicelib_ini = SERVICELIB_INI_TEMPLATE.format(
host=self.host,
port=self.port,
scratch_dir=scratch_dir,
services_dir=services_dir,
uwsgi_config_file=uwsgi_ini_file,
)
with open(servicelib_ini_file, "wt") as f:
f.write(self.servicelib_ini)
scratch_dir.mkdir(parents=True, exist_ok=True)
def __enter__(self):
env = dict(os.environ)
env["SERVICELIB_CONFIG_FILE"] = str(self.servicelib_ini_file)
subprocess.Popen("servicelib-worker", shell=True, env=env).wait()
utils.wait_for_port_open(self.port, self.host)
return self
def __exit__(self, *exc_info):
# self.log.debug("uwsgi.ini:\n%s", self.uwsgi_ini)
# self.log.debug("servicelib.ini:\n%s", self.servicelib_ini)
try:
with open(self.log_file, "rt") as f:
self.log.debug("uwsgi.log:\n%s", f.read())
except Exception:
pass
with open(self.pid_file, "rt") as f:
pid = int(f.read().strip())
os.kill(pid, signal.SIGTERM)
def http_post(self, path, **kwargs):
res = requests.post(
"http://{}:{}{}".format(self.host, self.port, path), **kwargs)
if res.status_code == 200:
return res.json()
try:
err = res.json()
except Exception:
raise Exception("{}\n\n{}".format(res.status_code, res.content))
try:
err = errors.Serializable.from_dict(err)
except Exception:
raise Exception("{}\n\n{}".format(res.status_code, res.content))
else:
raise err
def http_get(self, path, **kwargs):
return requests.get(
"http://{}:{}{}".format(self.host, self.port, path), **kwargs)
def log(self):
try:
return self.context.log
except AttributeError:
return logutils.get_logger(__name__)
class ConfigServer:
log = logutils.get_logger()
def __init__(self, initial_config, config_file, pid_file, log_file):
self.initial_config = initial_config
self.config_file = config_file
self.pid_file = pid_file
self.log_file = log_file
self.port = utils.available_port()
self.read_only = False
self.client = config_client.instance(url=self.url)
self.client.poll_interval = 1
self.p = None
def start(self, background=True):
with open(self.config_file, "wb") as f:
yaml.safe_dump(self.initial_config,
f,
encoding="utf-8",
allow_unicode=True)
cmdline = [
"servicelib-config-server",
"--port",
self.port,
"--log-file",
self.log_file,
"--config-file",
self.config_file,
]
if background:
cmdline.extend([
"--pid-file",
self.pid_file,
])
if self.read_only:
cmdline.append("--read-only")
cmdline = " ".join(str(c) for c in cmdline)
p = subprocess.Popen(cmdline, shell=True)
if background:
rc = p.wait()
if rc:
raise Exception("Error running {}".format(cmdline))
else:
self.p = p
utils.wait_for_url(self.client.url)
def stop(self):
try:
if self.p is not None:
self.p.terminate()
self.p.wait()
else:
with open(self.pid_file, "rt") as f:
pid = int(f.read().strip())
os.kill(pid, signal.SIGTERM)
except Exception:
pass
try:
with open(self.log_file, "rt") as f:
self.log.debug("uwsgi.log:\n%s", f.read())
except Exception:
pass
@property
def url(self):
return "http://127.0.0.1:{}/settings".format(self.port)
def http_get(self, path, **kwargs):
return requests.get("http://127.0.0.1:{}{}".format(self.port, path),
**kwargs)
def http_post(self, path, **kwargs):
return requests.post("http://127.0.0.1:{}{}".format(self.port, path),
**kwargs)
class Result(object):
log = logutils.get_logger(__name__)
_default_timeout = None
def __init__(self, http_session, service, args, kwargs, context):
self.http_session = http_session
self.timeout = check_timeout(
kwargs.pop("timeout", self.default_timeout))
self.args = args
self.kwargs = dict(kwargs)
# local_only = self.kwargs.pop("local_only", False)
# self.url = registry.instance().service_url(service, local_only=local_only)
self.url = registry.instance().service_url(service)
self.id = core.call_id()
if context is None:
context = ClientContext(self.id)
self.context = context
self.timer = Timer()
self._response = None
self._thread = t = threading.Thread(target=self._runner)
t.setDaemon(True)
t.start()
def _runner(self):
self.timer.start()
try:
req = core.Request(*self.args, **self.kwargs)
self.log.debug(
"POST %s, headers: %s, body: %s",
self.url,
req.http_headers,
req.http_body,
)
# XXX It's not entirely clear that `requests.Session` is thread-safe.
res = self.http_session.post(
self.url,
data=req.http_body,
headers=req.http_headers,
timeout=self.timeout,
)
res = core.Response.from_http(res.status_code, res.content,
res.headers)
self.log.debug("Response: %r", res)
self.timer.stop()
self.context.update_metadata(res.metadata)
except requests.Timeout as exc:
self.log.debug("Got timeout error: %s", exc)
res = errors.Timeout(self.url)
except Exception as exc:
self.log.info(
"%r failed: %s",
self,
exc,
exc_info=True,
stack_info=True,
)
res = exc
res.metadata = self.context.metadata
self._response = res
def wait(self, timeout=None):
if self._response is None:
self._thread.join(timeout=timeout)
if timeout is not None:
if self._thread.is_alive():
raise errors.Timeout(self.url)
if isinstance(self._response, Exception):
raise self._response
result = self._response.value
if isinstance(result, Exception):
raise result
return result, self._response.metadata
@property
def result(self):
r, _ = self.wait()
return r
@property
def metadata(self):
_, m = self.wait()
return m
def __repr__(self):
return "Result(%r, %r)" % (
self.url,
self.args,
)
@property
def default_timeout(self):
if self.__class__._default_timeout is None:
self.__class__._default_timeout = get_default_timeout()
return self.__class__._default_timeout
def main():
logutils.configure_logging()
cmdline_config = cmdline.parse_args(
"worker.autoreload",
"worker.hostname",
"worker.load_workers",
"worker.num_processes",
"worker.num_threads",
"worker.port",
"worker.services_dir",
)
for k, v in cmdline_config.items():
if isinstance(v, list):
v = json.dumps(v)
else:
v = str(v)
os.environ[env_var(k)] = v
cmd = ["uwsgi", "--req-logger", "file:/dev/null"]
autoreload = int(config.get("worker.autoreload", "0"))
if autoreload > 0:
cmd.extend(["--py-autoreload",
"{}".format(autoreload)]) # pragma: no cover
serve_results = config.get("worker.serve_results", default=None)
if serve_results is not None:
for dname in serve_results.split(":"):
cmd.extend(["--static-map", "{}={}".format(dname, dname)])
swagger_yaml = Path(
config.get("worker.services_dir", default="/code/services"),
"swagger.yaml")
if swagger_yaml.exists():
cmd.extend(
["--static-map", "/services/swagger.yaml={}".format(swagger_yaml)])
swagger_ui = Path(
config.get("worker.swagger_ui_path", default="/usr/share/nginx/html"))
if swagger_yaml.exists():
cmd.extend(["--static-map", "/docs={}".format(swagger_ui)])
cmd.extend(["--static-index", "index.html"])
static_assets = config.get("worker.static_map", default=None)
if static_assets is not None:
cmd.extend(["--static-map", static_assets])
cmd.append(
config.get(
"worker.uwsgi_config_file",
default=str(Path(logutils.__file__, "..", "uwsgi.ini").resolve()),
))
os.environ.setdefault(
env_var("worker.num_processes"),
config.get("worker.num_processes", str(psutil.cpu_count())),
)
os.environ.setdefault(env_var("worker.num_threads"),
str(config.get("worker.num_threads", 1)))
os.environ.setdefault(env_var("worker.port"),
str(config.get("worker.port", 8000)))
log = logutils.get_logger("servicelib-worker")
log.info("Environment: %s", os.environ)
log.info("Running: %s", " ".join(cmd))
# If we're running under `pytest-cov`, call `pytest_cov.embed.cleanup()`
# before exec of uWSGI, so that we do not lose coverage info for this
# Python module.
if os.environ.get("COV_CORE_DATAFILE"):
from pytest_cov.embed import cleanup
cleanup()
os.execlp(cmd[0], *cmd[0:])
class Metadata(object):
log = logutils.get_logger(__name__)
def __init__(self, name=None):
self._name = name
self._timers = {}
self._extra = {}
self._kids = []
self._notes = {}
self._host = HOSTNAME
self._start = time.time()
self._pid = os.getpid()
self._stop = 0.0
@property
def name(self):
return self._name
def annotate(self, key, value):
if isinstance(value, compat.string_types) or isinstance(
value, (int, float, list, dict, tuple)):
self._notes[key] = value
def update_metadata(self, other):
if self != other:
self._kids.append(other)
def timer(self, name):
if name not in self._timers:
self._timers[name] = Timer()
return self._timers[name]
def add_timer(self, *args):
pass
def start(self):
self._start = time.time()
def stop(self):
self._stop = time.time()
def as_http_headers(self):
ret = {
"task": self._name,
"host": self._host,
"pid": str(self._pid),
"start": str(self._start),
"stop": str(self._stop),
"kids": json.dumps([k.as_dict() for k in self._kids]),
}
ret.update({
"note-{}".format(k): json.dumps(v)
for (k, v) in self._notes.items()
})
timers = dict((k, v.as_dict()) for k, v in self._timers.items())
timers.update(self._extra)
ret["timers"] = json.dumps(timers)
return ret
@classmethod
def from_http_headers(cls, h):
ret = cls(h["task"])
ret._timers = {
k: Timer.from_dict(v)
for k, v in json.loads(h["timers"]).items()
}
ret._kids = [cls.from_dict(k) for k in json.loads(h["kids"])]
ret._host = h["host"]
ret._pid = int(h["pid"])
ret._start = float(h["start"])
ret._stop = float(h["stop"])
ret._notes = {
k[len("note-"):]: json.loads(v)
for (k, v) in h.items() if k.startswith("note-")
}
return ret
def as_dict(self):
r = {
"task": self._name,
"host": self._host,
"pid": self._pid,
}
r["kids"] = [k.as_dict() for k in self._kids]
r["timers"] = dict((k, v.as_dict()) for k, v in self._timers.items())
r["timers"].update(self._extra)
if self._start:
r["start"] = self._start
if self._stop:
r["stop"] = self._stop
r["notes"] = self._notes
r.update(self._notes)
return r
@classmethod
def from_dict(cls, d):
ret = cls(d["task"])
ret._timers = {k: Timer.from_dict(v) for k, v in d["timers"].items()}
ret._kids = [cls.from_dict(k) for k in d["kids"]]
ret._notes = d["notes"]
ret._host = d["host"]
ret._pid = d["pid"]
if "start" in d:
ret._start = d["start"]
if "stop" in d:
ret._stop = d["stop"]
return ret
def clear_timers(self):
self._start = 0
self._stop = 0
self._timers = {}
self._extra = {}
def update_timers(self, timers):
self._extra.update(timers.get("timers", {}))
if "start" in timers:
self._start = timers["start"]
if "stop" in timers:
self._stop = timers["stop"]
@property
def tracker(self):
try:
return self._notes["tracker"]
except KeyError:
return self._kids[0].tracker
def __repr__(self):
return ("Metadata(name={!r}, "
"timers={!r}, "
"extra={!r}, "
"kids={!r}, "
"notes={!r}, "
"host={!r}, "
"start={!r}, "
"pid={!r}, "
"stop={!r})").format(
self._name,
self._timers,
self._extra,
self._kids,
self._notes,
self._host,
self._start,
self._pid,
self._stop,
)
def __eq__(self, other):
if isinstance(other, Metadata):
return (self._name == other._name and self._timers == other._timers
and self._extra == other._extra
and self._kids == other._kids
and self._notes == other._notes
and self._host == other._host and self._pid == other._pid
and abs(self._start - other._start) < 0.01
and abs(self._stop - other._stop) < 0.01)
return False
class ConfigServer(object):
log = logutils.get_logger(__name__)
def __init__(self, fname):
self.fname = fname
self.config = types.ConfigDict()
with open(fname, "rb") as f:
self.config.reset(yaml.safe_load(f))
self.read_only = os.environ.get("SERVICELIB_CONFIG_SERVER_READ_ONLY") == "true"
self.lock = threading.Lock()
def on_get(self, req, resp, key):
"""Returns the whole configuration encoded in JSON.
"""
resp.data = json.dumps(self.config.as_dict()).encode("utf-8")
resp.status = falcon.HTTP_200
return
def on_post(self, req, resp, key):
"""Updates the value for the setting specified in the HTTP request
path.
"""
if self.read_only:
resp.status = falcon.HTTP_403
resp.data = json.dumps("Config server in read-only mode").encode("utf-8")
return
resp.status = falcon.HTTP_200
self.log.debug("on_post(req.path=%s, key=%s): Entering", req.path, key)
if req.content_length:
value = json.load(req.stream)
self.log.debug("Setting '%s' to: %s", key, value)
old_config = copy.deepcopy(self.config)
self.config.set(key, value)
try:
self.save_config()
except Exception as exc:
msg = "Cannot save config: {}".format(exc)
self.log.error(msg, exc_info=True, stack_info=True)
self.config = old_config
resp.status = falcon.HTTP_500
resp.data = json.dumps(msg).encode("utf-8")
def on_delete(self, req, resp, key=None):
"""Removes the setting specified in the HTTP request path.
Returns an HTTP 200 response on success, or an HTTP 404 response if
the setting was not found.
"""
if self.read_only:
resp.status = falcon.HTTP_403
resp.data = json.dumps("Config server in read-only mode").encode("utf-8")
return
old_config = copy.deepcopy(self.config)
try:
self.config.delete(key)
except KeyError:
resp.status = falcon.HTTP_404
return
resp.status = falcon.HTTP_200
try:
self.save_config()
except Exception as exc:
msg = "Cannot save config: {}".format(exc)
self.log.error(msg, exc_info=True, stack_info=True)
self.config = old_config
resp.status = falcon.HTTP_500
resp.data = json.dumps(msg).encode("utf-8")
def save_config(self):
fname_new = "{}.new".format(self.fname)
config = json.dumps(self.config.as_dict())
with self.lock:
with open(fname_new, "wb") as f:
yaml.safe_dump(config, f, encoding="utf-8", allow_unicode=True)
f.flush()
os.fsync(f.fileno())
os.rename(fname_new, self.fname)
"redis": RedisRegistry,
}
def instance():
class_name = config.get("registry.class", default="no-op")
try:
ret = _INSTANCE_MAP[class_name]
except KeyError:
raise Exception("Invalid value for `registry.class`: {}".format(class_name))
if isinstance(ret, type):
_INSTANCE_MAP[class_name] = ret = ret()
return ret
LOG = logutils.get_logger(__name__)
# class Cache(object):
# def __init__(self, ttl):
# self.ttl = ttl
# self._data = {}
# def get(self, k):
# v, expires = self._data[k]
# now = time.time()
# if now > expires:
# try:
# del self._data[k]
# except KeyError:
# pass
class Result(object):
log = logutils.get_logger(__name__)
def __init__(self, content_type):
self.content_type = content_type
self._is_open = False
self._length = 0
self._metadata = {}
def as_dict(self):
ret = {
"location": self.location,
"contentLength": self.length,
"contentType": self.content_type,
}
ret.update(self._metadata)
# TODO: Add `bytes` field, so that we may specify byte ranges.
return ret
def __setitem__(self, k, v):
"""Annotates this file with a ``(k, v)`` pair, which will be
included in its JSON serialized form.
"""
if k in {"location", "contentType", "contentLength", "metadata"}:
raise ValueError("Invalid key '{}'".format(k))
self._metadata[k] = v
def __enter__(self):
self._open()
self._is_open = True
return self
def __exit__(self, *exc_info):
close_exc = None
try:
self._close()
except Exception as exc:
self.log.warn("Error closing %r: %s",
self,
exc,
exc_info=True,
stack_info=True)
close_exc = exc
self._is_open = False
if exc_info == (None, None, None) and close_exc:
raise close_exc
def write(self, b):
if not self._is_open:
raise Exception("{!r}: Not open".format(self))
n = self._write(b)
# XXX No way to test this case for both Python 2 an Python 3 until
# we have results implementations which are based on things other than
# local files.
if n is None: # pragma: no cover
return None
self._length += n
return n
@property
def length(self):
return self._length
@property
def location(self):
raise NotImplementedError
@property
def path(self):
raise NotImplementedError
def _open(self):
raise NotImplementedError
def _close(self):
raise NotImplementedError
def _write(self, b):
raise NotImplementedError
def __repr__(self):
return "{}({})".format(self.__class__.__name__, self.as_dict())
class RedisRegistry(Registry):
_redis_key_prefix = "servicelib.url.".encode("utf-8")
log = logutils.get_logger(__name__)
def __init__(self):
super(RedisRegistry, self).__init__()
self._pool = RedisPool()
def register(self, services):
p = self._pool.connection().pipeline()
for (name, url) in services:
k = self.redis_key(name)
self.log.info("Registering service %s at %s", name, url)
p.sadd(k, url.encode("utf-8"))
p.execute()
def unregister(self, services):
p = self._pool.connection().pipeline()
for (name, url) in services:
k = self.redis_key(name)
self.log.info("Unregistering service %s at %s", name, url)
p.srem(k, url)
p.execute()
# def service_url(self, name, local_only=False):
def service_url(self, name):
# TODO: Cache results.
c = self._pool.connection()
k = self.redis_key(name)
# url = None
# if local_only:
# for parsed, unparsed in [(urlparse(u), u) for u in c.smembers(k)]:
# if parsed.netloc.split(":")[0] == HOSTNAME:
# url = unparsed
# break
# else:
# url = c.srandmember(k)
url = c.srandmember(k)
if url is None:
# raise Exception(
# "No URL for service {} (local-only: {})".format(name, local_only)
# )
raise Exception("No URL for service {}".format(name))
return url.decode("utf-8")
def services_by_name(self):
ret = {}
c = self._pool.connection()
cur = 0
while True:
cur, keys = c.scan(cur)
self.log.debug("services_by_name(): keys: %s", keys)
for k in keys:
self.log.debug("services_by_name(): %s ?", k)
if k.startswith(self._redis_key_prefix):
self.log.debug("services_by_name(): Yep: %s", k)
ret.setdefault(
k[len(self._redis_key_prefix) :].decode("utf-8"), set()
)
if cur == 0:
break
self.log.debug("services_by_name(): ret: %s", ret)
for k, urls in ret.items():
for url in c.smembers(self.redis_key(k)):
urls.add(url.decode("utf-8"))
return ret
def redis_key(self, service_name):
return self._redis_key_prefix + service_name.encode("utf-8")
class ConfigClient(object):
"""A caching client for the config source."""
log = logutils.get_logger(__name__)
def __init__(
self,
url,
group=None,
name=None,
poll_interval=DEFAULT_SETTINGS_POLL_INTERVAL,
):
"""Constructor.
This client will talk to the config source at `url`.
Arguments `group` and `name` are used to build the prefix for key
searches. When a user wants to retrieve the value of setting
`foo`, the following keys are looked up in his order:
1. `<group>.<name>.foo`
2. `<group>.foo`
3. `foo`
This client will poll the settings server every `poll_interval`
seconds, in order to get changes to the values stored in the config
source.
"""
self.url = url
self.group = group
self.name = name
self.__values = None
self.__old_values = None
self.poll_interval = poll_interval
self._pid = None
self._lock = threading.Lock()
self._poll_thread_active = True
def lookup(self,
key,
default=NO_DEFAULT,
exact=False,
name=None,
group=None):
"""Get the value associated with `key`.
If `exact` is false, the following keys will be looked up:
1. `<group>.<name>.<key>`
2. `<group>.<key>`
3. `<key>`
Othewise (if `exact` is true), `key` will be looked up.
If no value is found, and `default` is given, the value of `default` is
returned. Otherwise `KeyError` is raised.
"""
self._ensure_poller_thread()
if exact or group is None:
keys = [key]
else:
if name:
keys = [
".".join([group, name, key]), ".".join([group, key]), key
]
else:
keys = [".".join([group, key]), key]
for k in keys:
env_k = env_var(k)
try:
ret = os.environ[env_k].strip()
except KeyError:
try:
self.log.debug("config(%s): Key %s not in environment",
key, env_k)
except Exception:
pass
else:
ret_lower = ret.lower()
if ret_lower == "true":
ret = True
elif ret_lower == "false":
ret = False
elif ret[0] in {"{", "["}:
try:
ret = json.loads(ret)
except Exception:
pass
else:
try:
ret = int(ret)
except ValueError:
try:
ret = float(ret)
except ValueError:
pass
try:
self.log.debug(
"config(%s): Returning %s (from environment %s)",
key,
ret,
env_k,
)
except Exception:
pass
return ret
exc_fetching_source = None
try:
ret = self._values.get(k)
try:
self.log.debug(
"config(%s): Returning %s (from config %s)",
key,
ret,
self.url,
)
except Exception:
pass
return ret
except KeyError:
pass
except Exception as exc:
self.log.warn(
"config(%s): Error fetching config key %s from %s: %s",
key,
k,
self.url,
exc,
exc_info=True,
stack_info=True,
)
exc_fetching_source = exc
if default is not NO_DEFAULT:
self.log.debug("config(%s): Returning %s (from default)", key,
default)
return default
if exc_fetching_source is not None:
raise exc_fetching_source
raise Exception(
"No config value for `{}` (tried {}, exact={}, group={})".format(
key,
keys,
exact,
group,
))
def get(self, key, default=NO_DEFAULT, exact=False):
return self.lookup(key,
default=default,
exact=exact,
name=self.name,
group=self.group)
def set(self, key, value):
"""Sets the given setting.
`key` is expected to have the following form:
foo.bar.baz
"""
if key in {"", None}:
raise ValueError("Invalid key `{}`".format(key))
self._key_set(key, value)
# Ensure we get updated values on next read.
self.clear()
def dump(self):
"""Returns a dictionary containing all settings.
The returned value is JSON-serialisable.
"""
self._ensure_poller_thread()
return self._values.as_dict()
def delete(self, key):
"""Removes the given setting.
`key` is expected to have the following form:
foo.bar.baz
Raises `KeyError` if `key` does not exist.
"""
self._key_deleted(key)
# Ensure we get updated values on next read.
self.clear()
def clear(self):
"""Clears cached settings.
"""
self.__values = None
refresh = clear
def url():
"""Normilized URL of the config source.
"""
def fget(self):
return self._url
def fset(self, url):
self._url = str(url.rstrip("/"))
return locals()
url = property(**url())
def _ensure_poller_thread(self):
try:
with self._lock:
pid = os.getpid()
if self._pid != pid:
t = threading.Thread(target=self._poll)
t.setDaemon(True)
t.start()
self._pid = pid
# Ensure the poller thread has run at least once
# before returning from this function (by doing
# explicitly what the poller thread does).
self._poll(only_once=True)
except Exception as exc:
self.log.warn(
"Call to _ensure_poller_thread() failed: %s",
exc,
exc_info=True,
stack_info=True,
)
def _poll(self, only_once=False):
while True:
if self._poll_thread_active:
self.clear()
if only_once:
return
time.sleep(self.poll_interval)
@property
def _values(self):
"""Returns all settings, retrieving them from the settings source if
necessary.
"""
v = copy.deepcopy(self.__values)
if v is None:
try:
values = self._read_values()
self.__old_values = values
except Exception as exc:
if self.__old_values is not None:
self.log.warn(
"Cannot fetch config from %s, reusing previous values: %s",
self.url,
exc,
exc_info=True,
stack_info=True,
)
values = self.__old_values
else:
raise
self.__values = v = ConfigDict(values)
return v
def _read_values(self):
raise NotImplementedError("Implement in subclass!")
def __repr__(self):
return "{}(url={})".format(self.__class__.__name__, self.url)
