async def rpc_handler(self, request: web.Request) -> web.Response:
try:
result = await self._rpc.call(
await request.read(), request.charset
)
if result.method is not None:
span.tag(SPAN_TAG_RPC_METHOD, result.method)
if result.error is not None:
span.error(result.error)
if isinstance(result.error, InternalError):
self.app.log_err(result.error)
resp = self._err_resp(result.error)
if result.result is not None:
resp['result'] = result.result
else:
resp = {"code": 0, "message": 'OK', 'result': result.result}
span.tag(SPAN_TAG_RPC_CODE, resp['code'])
span.name = 'rpc::in (%s)' % result.method
span.set_name4adapter(self.app.logger.ADAPTER_PROMETHEUS, 'rpc_in')
body = self._json_encode(resp).encode()
return web.Response(body=body, content_type='application/json')
except Exception as err:
span.error(err)
self.app.log_err(err)
return web.Response(
body=self._json_encode(
self._err_resp(InternalError(parent=err))
).encode(),
content_type='application/json',
)
async def __call__(
self,
scope: dict,
receive: Callable[..., Awaitable],
send: Callable[..., Awaitable],
) -> None:
span_token: Optional[Token] = None
headers: Dict[str, str] = {}
url: Optional[URL] = None
host: Optional[str] = None
method: Optional[str] = None
path: Optional[str] = None
if scope['type'] == 'http':
headers = {h[0].decode(): h[1].decode() for h in scope['headers']}
host = ':'.join([str(s) for s in scope['server']])
path = scope['raw_path'].decode()
if scope['query_string']:
path += '?' + scope['query_string'].decode()
url = URL('%s://%s%s' % (scope['scheme'], host, path))
try:
span: HttpSpan = self.uvicorn.app.logger.span_from_headers( # type: ignore
headers, cls=ServerHttpSpan
)
span.name, span_tags = self._get_route_details(scope)
for tag_name, tag_value in span_tags.items():
span.tag(tag_name, tag_value)
span_token = ctx_span_set(span)
with span:
span.kind = HttpSpan.KIND_SERVER
if host is not None:
span.tag(HttpSpan.TAG_HTTP_HOST, host)
if path is not None:
span.tag(HttpSpan.TAG_HTTP_PATH, path)
if method is not None:
span.tag(HttpSpan.TAG_HTTP_METHOD, method)
if url is not None:
span.tag(HttpSpan.TAG_HTTP_URL, self._mask_url(url))
try:
await self.app(
scope,
partial(self.receive_wraper, receive),
partial(self.send_wraper, send),
)
except Exception as err:
self.uvicorn.app.log_err(err)
span.error(err)
except Exception as err:
self.uvicorn.app.log_err(err)
raise
finally:
if span_token:
ctx_span_reset(span_token)
async def _scan(self) -> None:
if self.app is None or self._lock is None: # pragma: no cover
raise UserWarning
while True:
if self._stopping:
return
async with self._lock:
delay = 1.0 # default: 1 second
try:
with wrap2span(
name=TaskManagerSpan.NAME_SCAN,
kind=Span.KIND_SERVER,
# ignore_ctx=True,
cls=TaskManagerSpan,
app=self.app,
) as span:
try:
tasks, delay = await self._search_and_exec()
if len(tasks) == 0:
span.skip()
except Exception as err:
span.error(err)
self.app.log_err(err)
finally:
if not self._stopping:
span.annotate(
TaskManagerSpan.ANN_NEXT_SCAN,
'next: %s' % delay,
)
finally:
if not self._stopping:
self._scan_fut = None
eta = time() + delay
self.stamp_early = eta
sleep = self.stamp_early - time()
if sleep > 0:
try:
self._scan_sleep_fut = asyncio.create_task(
asyncio.sleep(sleep))
await self._scan_sleep_fut
except asyncio.CancelledError:
pass
self._scan_sleep_fut = None
async def _scan(self) -> List[int]:
if self.app is None or self._lock is None: # pragma: no cover
raise UserWarning
if self._stopping:
return []
async with self._lock:
delay = 1.0 # default: 1 second
try:
with wrap2span(
name=TaskManagerSpan.NAME_SCAN,
kind=Span.KIND_SERVER,
# ignore_ctx=True,
cls=TaskManagerSpan,
app=self.app,
) as span:
try:
tasks, delay = await self._search_and_exec()
if len(tasks) == 0:
span.skip()
return [task.id for task in tasks]
except Exception as err:
span.error(err)
self.app.log_err(err)
finally:
if not self._stopping:
span.annotate(
TaskManagerSpan.ANN_NEXT_SCAN,
'next: %s' % delay,
)
return []
finally:
if not self._stopping:
self._scan_fut = None
eta = self.loop.time() + delay
self.stamp_early = eta
self.loop.call_at(eta, self._scan_later, eta)
async def _exec(self, parent_trace_id: str, task: Task) -> None:
with wrap2span(
name=TaskManagerSpan.NAME_EXEC,
kind=Span.KIND_SERVER,
ignore_ctx=True,
cls=TaskManagerSpan,
app=self.app,
) as span:
if task.trace_id is not None:
span.trace_id = task.trace_id
span.parent_id = task.trace_span_id
if self._db is None or self._executor is None: # pragma: no cover
raise UserWarning
span.name = '%s::%s' % (TaskManagerSpan.NAME_EXEC, task.name)
span.tag(TaskManagerSpan.TAG_PARENT_TRACE_ID, parent_trace_id)
span.tag(TaskManagerSpan.TAG_TASK_ID, task.id)
span.tag(TaskManagerSpan.TAG_TASK_NAME, task.name)
try:
err: Optional[Exception] = None
err_str: Optional[str] = None
err_trace: Optional[str] = None
res: Any = None
time_begin = time.time()
try:
res = await self._executor.exec(task.name,
kwargs=task.params)
except Exception as e:
err = e
if isinstance(err, Retry):
err_str = str(err.err)
else:
err_str = str(err)
err_trace = traceback.format_exc()
span.error(err)
self.app.log_err(err)
time_finish = time.time()
await self._db.task_log_add(
task.id,
task.eta,
time_begin,
time_finish,
res,
err_str,
err_trace,
lock=True,
)
if task.retries is None:
retries = 0
else:
retries = task.retries + 1
if err is not None:
if isinstance(err, Retry):
if retries >= task.max_retries:
await self._db.task_move_arch(
task.id,
STATUS_ERROR,
retries,
lock=True,
with_trace_id=task.trace_id is not None,
)
else:
await self._db.task_retry(
task.id,
retries,
task.retry_delay.total_seconds(),
lock=True,
)
else:
await self._db.task_move_arch(
task.id,
STATUS_ERROR,
retries,
lock=True,
with_trace_id=task.trace_id is not None,
)
else:
await self._db.task_move_arch(
task.id,
STATUS_SUCCESSFUL,
retries,
lock=True,
with_trace_id=task.trace_id is not None,
)
except Exception as err:
span.error(err)
self.app.log_err(err)
raise