async def rpc_handler(self, request: web.Request) -> web.Response:
try:
result = await self._rpc.call(
await request.read(), request.charset
)
if result.method is not None:
span.tag(SPAN_TAG_RPC_METHOD, result.method)
if result.error is not None:
span.error(result.error)
if isinstance(result.error, InternalError):
self.app.log_err(result.error)
resp = self._err_resp(result.error)
if result.result is not None:
resp['result'] = result.result
else:
resp = {"code": 0, "message": 'OK', 'result': result.result}
span.tag(SPAN_TAG_RPC_CODE, resp['code'])
span.name = 'rpc::in (%s)' % result.method
span.set_name4adapter(self.app.logger.ADAPTER_PROMETHEUS, 'rpc_in')
body = self._json_encode(resp).encode()
return web.Response(body=body, content_type='application/json')
except Exception as err:
span.error(err)
self.app.log_err(err)
return web.Response(
body=self._json_encode(
self._err_resp(InternalError(parent=err))
).encode(),
content_type='application/json',
)
def _set_span_method(self, method: Optional[str]) -> None:
if not span:
return
if method is not None:
span.name = 'rpc::in (%s)' % method
span.tag(SPAN_TAG_JSONRPC_METHOD, method)
else:
span.name = 'rpc::in::error'
async def _exec_in_executor(self, method: str, kwargs: Dict[str,
Any]) -> Any:
try:
return await self._ex.exec(name=method, args=None, kwargs=kwargs)
except Exception as err:
if app and span and hasattr(err, 'code'):
span.tag(SPAN_TAG_RPC_CODE, getattr(err, 'code'))
raise
async def _exec_in_executor(self, method: str, args: Tuple[Any, ...],
kwargs: Dict[str, Any]) -> Any:
try:
if self._discover_enabled and method == 'rpc.discover':
if len(args) or len(kwargs):
raise _InvalidArguments()
return self._discover()
return await self._ex.exec(method, args, kwargs)
except Exception as err:
if app and span and hasattr(err, 'jsonrpc_error_code'):
span.tag(SPAN_TAG_RPC_CODE, getattr(err, 'jsonrpc_error_code'))
raise
async def __call__(
self,
scope: dict,
receive: Callable[..., Awaitable],
send: Callable[..., Awaitable],
) -> None:
span_token: Optional[Token] = None
headers: Dict[str, str] = {}
url: Optional[URL] = None
host: Optional[str] = None
method: Optional[str] = None
path: Optional[str] = None
if scope['type'] == 'http':
headers = {h[0].decode(): h[1].decode() for h in scope['headers']}
host = ':'.join([str(s) for s in scope['server']])
path = scope['raw_path'].decode()
if scope['query_string']:
path += '?' + scope['query_string'].decode()
url = URL('%s://%s%s' % (scope['scheme'], host, path))
try:
span: HttpSpan = self.uvicorn.app.logger.span_from_headers( # type: ignore
headers, cls=ServerHttpSpan
)
span.name, span_tags = self._get_route_details(scope)
for tag_name, tag_value in span_tags.items():
span.tag(tag_name, tag_value)
span_token = ctx_span_set(span)
with span:
span.kind = HttpSpan.KIND_SERVER
if host is not None:
span.tag(HttpSpan.TAG_HTTP_HOST, host)
if path is not None:
span.tag(HttpSpan.TAG_HTTP_PATH, path)
if method is not None:
span.tag(HttpSpan.TAG_HTTP_METHOD, method)
if url is not None:
span.tag(HttpSpan.TAG_HTTP_URL, self._mask_url(url))
try:
await self.app(
scope,
partial(self.receive_wraper, receive),
partial(self.send_wraper, send),
)
except Exception as err:
self.uvicorn.app.log_err(err)
span.error(err)
except Exception as err:
self.uvicorn.app.log_err(err)
raise
finally:
if span_token:
ctx_span_reset(span_token)
async def _exec_batch(
self, req: rpc.JSONRPCBatchRequest
) -> Optional[rpc.RPCBatchResponse]:
if span:
span.name = 'rpc::in::batch'
span.tag(SPAN_TAG_JSONRPC_IS_BATCH, 'true')
span.set_name4adapter(
self._app.logger.ADAPTER_PROMETHEUS, 'rpc_in_batch'
)
resp = req.create_batch_response()
batch = []
for req_item in req:
if isinstance(req_item, rpc.InvalidRequestError):
batch.append(self._exec_err(req_item))
else:
batch.append(
self._exec(
req_item.method,
req_item.args,
req_item.kwargs,
req_item.one_way,
)
)
results = await asyncio.gather(*batch, return_exceptions=True)
if resp is None:
return None
for i in range(len(req)):
if isinstance(req[i], rpc.InvalidRequestError):
err_resp = req[i].error_respond()
if hasattr(err_resp, 'data'):
err_resp.data = self.cast2dump(err_resp.data)
resp.append(err_resp)
elif req[i].one_way:
pass
elif isinstance(results[i], BaseException):
err_resp = req[i].error_respond(results[i])
if hasattr(err_resp, 'data'):
err_resp.data = self.cast2dump(err_resp.data)
resp.append(err_resp)
else:
resp.append(req[i].respond(self.cast2dump(results[i])))
return resp
def _set_span_err(self, err: Exception) -> None:
if not span:
return
span.tag('error', 'true')
span.annotate(span.ANN_TRACEBACK, traceback.format_exc())
if hasattr(err, 'jsonrpc_error_code'):
span.tag(
SPAN_TAG_JSONRPC_CODE,
str(err.jsonrpc_error_code), # type: ignore
)
else:
code, _, _ = _get_code_message_and_data(err)
span.tag(SPAN_TAG_JSONRPC_CODE, str(code))
async def _exec(self, parent_trace_id: str, task: Task) -> None:
with wrap2span(
name=TaskManagerSpan.NAME_EXEC,
kind=Span.KIND_SERVER,
ignore_ctx=True,
cls=TaskManagerSpan,
app=self.app,
) as span:
if task.trace_id is not None:
span.trace_id = task.trace_id
span.parent_id = task.trace_span_id
if self._db is None or self._executor is None: # pragma: no cover
raise UserWarning
span.name = '%s::%s' % (TaskManagerSpan.NAME_EXEC, task.name)
span.tag(TaskManagerSpan.TAG_PARENT_TRACE_ID, parent_trace_id)
span.tag(TaskManagerSpan.TAG_TASK_ID, task.id)
span.tag(TaskManagerSpan.TAG_TASK_NAME, task.name)
try:
err: Optional[Exception] = None
err_str: Optional[str] = None
err_trace: Optional[str] = None
res: Any = None
time_begin = time.time()
try:
res = await self._executor.exec(task.name,
kwargs=task.params)
except Exception as e:
err = e
if isinstance(err, Retry):
err_str = str(err.err)
else:
err_str = str(err)
err_trace = traceback.format_exc()
span.error(err)
self.app.log_err(err)
time_finish = time.time()
await self._db.task_log_add(
task.id,
task.eta,
time_begin,
time_finish,
res,
err_str,
err_trace,
lock=True,
)
if task.retries is None:
retries = 0
else:
retries = task.retries + 1
if err is not None:
if isinstance(err, Retry):
if retries >= task.max_retries:
await self._db.task_move_arch(
task.id,
STATUS_ERROR,
retries,
lock=True,
with_trace_id=task.trace_id is not None,
)
else:
await self._db.task_retry(
task.id,
retries,
task.retry_delay.total_seconds(),
lock=True,
)
else:
await self._db.task_move_arch(
task.id,
STATUS_ERROR,
retries,
lock=True,
with_trace_id=task.trace_id is not None,
)
else:
await self._db.task_move_arch(
task.id,
STATUS_SUCCESSFUL,
retries,
lock=True,
with_trace_id=task.trace_id is not None,
)
except Exception as err:
span.error(err)
self.app.log_err(err)
raise
