def set_escalation_close_ctx(self): current_context, current_span = get_current_span() if current_context or self.escalation_close_ctx: self.escalation_close_ctx = current_context self._get_collection().update( {"_id": self.id}, {"$set": {"escalation_close_ctx": current_context}} )
def submit(self, fn, *args, **kwargs): if self.to_shutdown: raise RuntimeError("Cannot schedule new task after shutdown") future = Future() span_ctx, span = get_current_span() # Fetch span label if "_in_label" in kwargs: in_label = kwargs.pop("_in_label") else: in_label = None # Put to the working queue self._put((future, fn, args, kwargs, span_ctx, span, in_label)) return future
def __getattr__(self, item): @tornado.gen.coroutine def _call(method, *args, **kwargs): @tornado.gen.coroutine def make_call(url, body, limit=3): req_headers = { "X-NOC-Calling-Service": self._service.name, "Content-Type": "text/json" } sample = 1 if span_ctx and span_id else 0 with Span(server=self._service_name, service=method, sample=sample, context=span_ctx, parent=span_id) as span: if sample: req_headers["X-NOC-Span-Ctx"] = span.span_context req_headers["X-NOC-Span"] = span.span_id code, headers, data = yield fetch( url, method="POST", headers=req_headers, body=body, connect_timeout=CONNECT_TIMEOUT, request_timeout=REQUEST_TIMEOUT) # Process response if code == 200: raise tornado.gen.Return(data) elif code == 307: # Process redirect if not limit: raise RPCException("Redirects limit exceeded") url = headers.get("location") self._logger.debug("Redirecting to %s", url) r = yield make_call(url, data, limit - 1) raise tornado.gen.Return(r) elif code in (598, 599): span.error_code = code self._logger.debug("Timed out") raise tornado.gen.Return(None) else: span.error_code = code raise RPCHTTPError("HTTP Error %s: %s" % (code, body)) t0 = time.time() self._logger.debug("[%sCALL>] %s.%s(%s, %s)", "SYNC " if self._sync else "", self._service_name, method, args, kwargs) metrics["rpc_call", ("called_service", self._service_name), ("method", method)] += 1 tid = next(self._tid) msg = {"method": method, "params": list(args)} is_notify = "_notify" in kwargs if not is_notify: msg["id"] = tid body = ujson.dumps(msg) # Get services response = None for t in self._service.iter_rpc_retry_timeout(): # Resolve service against service catalog if self._hints: svc = random.choice(self._hints) else: svc = yield self._service.dcs.resolve(self._service_name) response = yield make_call( "http://%s/api/%s/" % (svc, self._api), body) if response: break else: yield tornado.gen.sleep(t) t = time.time() - t0 self._logger.debug("[CALL<] %s.%s (%.2fms)", self._service_name, method, t * 1000) if response: if not is_notify: try: result = ujson.loads(response) except ValueError as e: raise RPCHTTPError("Cannot decode json: %s" % e) if result.get("error"): self._logger.error("RPC call failed: %s", result["error"]) raise RPCRemoteError( "RPC call failed: %s" % result["error"], remote_code=result.get("code", None)) else: raise tornado.gen.Return(result["result"]) else: # Notifications return None raise tornado.gen.Return() else: raise RPCNoService("No active service %s found" % self._service_name) @tornado.gen.coroutine def async_wrapper(*args, **kwargs): result = yield _call(item, *args, **kwargs) raise tornado.gen.Return(result) def sync_wrapper(*args, **kwargs): @tornado.gen.coroutine def _sync_call(): try: r = yield _call(item, *args, **kwargs) result.append(r) except tornado.gen.Return as e: result.append(e.value) except Exception: error.append(sys.exc_info()) finally: ev.set() ev = threading.Event() result = [] error = [] self._service.ioloop.add_callback(_sync_call) ev.wait() if error: six.reraise(*error[0]) else: return result[0] if item.startswith("_"): return self.__dict__[item] span_ctx, span_id = get_current_span() if self._sync: return sync_wrapper else: return async_wrapper
def __getattr__(self, item): async def _call(method, *args, **kwargs): async def make_call(url, body, limit=3): req_headers = { "X-NOC-Calling-Service": self._service.name, "Content-Type": "text/json", } sample = 1 if span_ctx and span_id else 0 with Span( server=self._service_name, service=method, sample=sample, context=span_ctx, parent=span_id, ) as span: if sample: req_headers["X-NOC-Span-Ctx"] = span.span_context req_headers["X-NOC-Span"] = span.span_id code, headers, data = await fetch( url, method="POST", headers=req_headers, body=body, connect_timeout=CONNECT_TIMEOUT, request_timeout=REQUEST_TIMEOUT, ) # Process response if code == 200: return data elif code == 307: # Process redirect if not limit: raise RPCException("Redirects limit exceeded") url = headers.get("location") self._logger.debug("Redirecting to %s", url) r = await make_call(url, data, limit - 1) return r elif code in (598, 599): span.set_error(code) self._logger.debug("Timed out") return None else: span.set_error(code) raise RPCHTTPError("HTTP Error %s: %s" % (code, body)) t0 = perf_counter() self._logger.debug( "[%sCALL>] %s.%s(%s, %s)", "SYNC " if self._sync else "", self._service_name, method, args, kwargs, ) metrics["rpc_call", ("called_service", self._service_name), ("method", method)] += 1 tid = next(self._tid) msg = {"method": method, "params": list(args)} is_notify = "_notify" in kwargs if not is_notify: msg["id"] = tid body = smart_text(orjson.dumps(msg)) # Get services response = None for t in self._service.iter_rpc_retry_timeout(): # Resolve service against service catalog if self._hints: svc = random.choice(self._hints) else: svc = await self._service.dcs.resolve(self._service_name) response = await make_call( "http://%s/api/%s/" % (svc, self._api), body) if response: break else: await asyncio.sleep(t) t = perf_counter() - t0 self._logger.debug("[CALL<] %s.%s (%.2fms)", self._service_name, method, t * 1000) if response: if not is_notify: try: with _orjson_crash_lock: result = orjson.loads(response) except ValueError as e: raise RPCHTTPError("Cannot decode json: %s" % e) if result.get("error"): self._logger.error("RPC call failed: %s", result["error"]) raise RPCRemoteError( "RPC call failed: %s" % result["error"], remote_code=result.get("code", None), ) else: return result["result"] else: # Notifications return None return else: raise RPCNoService("No active service %s found" % self._service_name) async def async_wrapper(*args, **kwargs): return await _call(item, *args, **kwargs) def sync_wrapper(*args, **kwargs): async def wrapper(): return await _call(item, *args, **kwargs) return run_sync(wrapper) if item.startswith("_"): return self.__dict__[item] span_ctx, span_id = get_current_span() if self._sync: return sync_wrapper else: return async_wrapper