def set_escalation_close_ctx(self):
current_context, current_span = get_current_span()
if current_context or self.escalation_close_ctx:
self.escalation_close_ctx = current_context
self._get_collection().update(
{"_id": self.id}, {"$set": {"escalation_close_ctx": current_context}}
)
def submit(self, fn, *args, **kwargs):
if self.to_shutdown:
raise RuntimeError("Cannot schedule new task after shutdown")
future = Future()
span_ctx, span = get_current_span()
# Fetch span label
if "_in_label" in kwargs:
in_label = kwargs.pop("_in_label")
else:
in_label = None
# Put to the working queue
self._put((future, fn, args, kwargs, span_ctx, span, in_label))
return future
def __getattr__(self, item):
@tornado.gen.coroutine
def _call(method, *args, **kwargs):
@tornado.gen.coroutine
def make_call(url, body, limit=3):
req_headers = {
"X-NOC-Calling-Service": self._service.name,
"Content-Type": "text/json"
}
sample = 1 if span_ctx and span_id else 0
with Span(server=self._service_name,
service=method,
sample=sample,
context=span_ctx,
parent=span_id) as span:
if sample:
req_headers["X-NOC-Span-Ctx"] = span.span_context
req_headers["X-NOC-Span"] = span.span_id
code, headers, data = yield fetch(
url,
method="POST",
headers=req_headers,
body=body,
connect_timeout=CONNECT_TIMEOUT,
request_timeout=REQUEST_TIMEOUT)
# Process response
if code == 200:
raise tornado.gen.Return(data)
elif code == 307:
# Process redirect
if not limit:
raise RPCException("Redirects limit exceeded")
url = headers.get("location")
self._logger.debug("Redirecting to %s", url)
r = yield make_call(url, data, limit - 1)
raise tornado.gen.Return(r)
elif code in (598, 599):
span.error_code = code
self._logger.debug("Timed out")
raise tornado.gen.Return(None)
else:
span.error_code = code
raise RPCHTTPError("HTTP Error %s: %s" % (code, body))
t0 = time.time()
self._logger.debug("[%sCALL>] %s.%s(%s, %s)",
"SYNC " if self._sync else "",
self._service_name, method, args, kwargs)
metrics["rpc_call", ("called_service", self._service_name),
("method", method)] += 1
tid = next(self._tid)
msg = {"method": method, "params": list(args)}
is_notify = "_notify" in kwargs
if not is_notify:
msg["id"] = tid
body = ujson.dumps(msg)
# Get services
response = None
for t in self._service.iter_rpc_retry_timeout():
# Resolve service against service catalog
if self._hints:
svc = random.choice(self._hints)
else:
svc = yield self._service.dcs.resolve(self._service_name)
response = yield make_call(
"http://%s/api/%s/" % (svc, self._api), body)
if response:
break
else:
yield tornado.gen.sleep(t)
t = time.time() - t0
self._logger.debug("[CALL<] %s.%s (%.2fms)", self._service_name,
method, t * 1000)
if response:
if not is_notify:
try:
result = ujson.loads(response)
except ValueError as e:
raise RPCHTTPError("Cannot decode json: %s" % e)
if result.get("error"):
self._logger.error("RPC call failed: %s",
result["error"])
raise RPCRemoteError(
"RPC call failed: %s" % result["error"],
remote_code=result.get("code", None))
else:
raise tornado.gen.Return(result["result"])
else:
# Notifications return None
raise tornado.gen.Return()
else:
raise RPCNoService("No active service %s found" %
self._service_name)
@tornado.gen.coroutine
def async_wrapper(*args, **kwargs):
result = yield _call(item, *args, **kwargs)
raise tornado.gen.Return(result)
def sync_wrapper(*args, **kwargs):
@tornado.gen.coroutine
def _sync_call():
try:
r = yield _call(item, *args, **kwargs)
result.append(r)
except tornado.gen.Return as e:
result.append(e.value)
except Exception:
error.append(sys.exc_info())
finally:
ev.set()
ev = threading.Event()
result = []
error = []
self._service.ioloop.add_callback(_sync_call)
ev.wait()
if error:
six.reraise(*error[0])
else:
return result[0]
if item.startswith("_"):
return self.__dict__[item]
span_ctx, span_id = get_current_span()
if self._sync:
return sync_wrapper
else:
return async_wrapper
def __getattr__(self, item):
async def _call(method, *args, **kwargs):
async def make_call(url, body, limit=3):
req_headers = {
"X-NOC-Calling-Service": self._service.name,
"Content-Type": "text/json",
}
sample = 1 if span_ctx and span_id else 0
with Span(
server=self._service_name,
service=method,
sample=sample,
context=span_ctx,
parent=span_id,
) as span:
if sample:
req_headers["X-NOC-Span-Ctx"] = span.span_context
req_headers["X-NOC-Span"] = span.span_id
code, headers, data = await fetch(
url,
method="POST",
headers=req_headers,
body=body,
connect_timeout=CONNECT_TIMEOUT,
request_timeout=REQUEST_TIMEOUT,
)
# Process response
if code == 200:
return data
elif code == 307:
# Process redirect
if not limit:
raise RPCException("Redirects limit exceeded")
url = headers.get("location")
self._logger.debug("Redirecting to %s", url)
r = await make_call(url, data, limit - 1)
return r
elif code in (598, 599):
span.set_error(code)
self._logger.debug("Timed out")
return None
else:
span.set_error(code)
raise RPCHTTPError("HTTP Error %s: %s" % (code, body))
t0 = perf_counter()
self._logger.debug(
"[%sCALL>] %s.%s(%s, %s)",
"SYNC " if self._sync else "",
self._service_name,
method,
args,
kwargs,
)
metrics["rpc_call", ("called_service", self._service_name),
("method", method)] += 1
tid = next(self._tid)
msg = {"method": method, "params": list(args)}
is_notify = "_notify" in kwargs
if not is_notify:
msg["id"] = tid
body = smart_text(orjson.dumps(msg))
# Get services
response = None
for t in self._service.iter_rpc_retry_timeout():
# Resolve service against service catalog
if self._hints:
svc = random.choice(self._hints)
else:
svc = await self._service.dcs.resolve(self._service_name)
response = await make_call(
"http://%s/api/%s/" % (svc, self._api), body)
if response:
break
else:
await asyncio.sleep(t)
t = perf_counter() - t0
self._logger.debug("[CALL<] %s.%s (%.2fms)", self._service_name,
method, t * 1000)
if response:
if not is_notify:
try:
with _orjson_crash_lock:
result = orjson.loads(response)
except ValueError as e:
raise RPCHTTPError("Cannot decode json: %s" % e)
if result.get("error"):
self._logger.error("RPC call failed: %s",
result["error"])
raise RPCRemoteError(
"RPC call failed: %s" % result["error"],
remote_code=result.get("code", None),
)
else:
return result["result"]
else:
# Notifications return None
return
else:
raise RPCNoService("No active service %s found" %
self._service_name)
async def async_wrapper(*args, **kwargs):
return await _call(item, *args, **kwargs)
def sync_wrapper(*args, **kwargs):
async def wrapper():
return await _call(item, *args, **kwargs)
return run_sync(wrapper)
if item.startswith("_"):
return self.__dict__[item]
span_ctx, span_id = get_current_span()
if self._sync:
return sync_wrapper
else:
return async_wrapper
