def task_prerun(task=None, **kwargs):
trace_info = None
request_state = None
if task.request.headers is not None:
trace_info = task.request.headers.get(TRACE_INFO)
request_state = task.request.headers.get(REQUEST_STATE)
if trace_info is None and hasattr(task.request, TRACE_INFO):
trace_info = getattr(task.request, TRACE_INFO)
if request_state is None and hasattr(task.request, REQUEST_STATE):
request_state = getattr(task.request, REQUEST_STATE)
if trace_info is not None:
trace_info = call_info_pb2.TraceInfo().FromString(
b64decode(trace_info))
if request_state is not None:
request_state = specialization.create_state_proto().FromString(
b64decode(request_state))
servicer.set_current_context(
CeleryTaskContext(request_state=request_state, trace_info=trace_info))
def setup_context(request, context):
task_context = GrpcTaskContext(context)
servicer.set_current_context(task_context)
start_time = time.time()
SERVER_FREE_WORKERS.labels(servicer.name(), 'grpc').dec()
SERVER_IN_PROGRESS.labels(servicer.name(), 'grpc').inc()
try:
response = handler.unary_unary(request, context)
response_metadata = task_context.response_metadata()
if response_metadata:
context.set_trailing_metadata(response_metadata)
finally:
servicer.set_current_context(None)
SERVER_FREE_WORKERS.labels(servicer.name(), 'grpc').inc()
SERVER_IN_PROGRESS.labels(servicer.name(), 'grpc').dec()
elapsed = time.time() - start_time
SERVER_LATENCY.labels(servicer.name(),
handler_call_details.method,
'grpc').observe(elapsed)
return response
def _before_request(request):
state = headers.get_state(request.headers)
trace_info = headers.get_trace_info(request.headers)
servicer.set_current_context(SanicTaskContext(state, trace_info))
SERVER_FREE_WORKERS.labels(servicer.name(), 'sanic').dec()
SERVER_IN_PROGRESS.labels(servicer.name(), 'sanic').inc()
def process_request(self, request):
SERVER_FREE_WORKERS.labels(servicer.name(), 'django').dec()
SERVER_IN_PROGRESS.labels(servicer.name(), 'django').inc()
request.counted_request = True
servicer.set_current_context(DjangoTaskContext())
def _on_message(self, queue_name, body, message):
if not message.channel.connection:
return
retry_cnt = int(message.headers.get('retry_cnt', '0'))
service_context = RabbitMQServicerContext(30)
start_time = None
try:
method_name = message.headers.get('method')
execute_at = message.headers.get('execute_at')
rabbitmq_request = rabbitmq_pb2.RabbitMqRpcMessage()
job_timestamp = None
caller_service_name = None
rpc_info = None
if execute_at and not RabbitMQServer._should_execute(
int(execute_at)):
self._msg_opr_queue.put(
(self._requeue,
RabbitMQServer._retry_queue_name(queue_name), body,
message, None, None))
self._msg_opr_queue.put((self._ack_message, message))
return
if method_name:
request_body = body
method_name = '/' + method_name
state_bin = message.headers.get(REQUEST_STATE)
if state_bin is not None:
rabbitmq_request.experiments_data = state_bin.encode()
else:
rabbitmq_request.ParseFromString(body)
method_name = rabbitmq_request.rpc_info.method
request_body = rabbitmq_request.request
job_timestamp = time.mktime(
rabbitmq_request.rpc_info.ts.ToDatetime().timetuple())
caller_service_name = rabbitmq_request.rpc_info.caller_service
rpc_info = rabbitmq_request.rpc_info
if retry_cnt == 0 and rabbitmq_request.rpc_info.HasField(
"target_ts"):
target_ts = rabbitmq_request.rpc_info.target_ts.ToDatetime(
)
else:
target_ts = datetime.utcnow()
if datetime.utcnow() < target_ts:
self._msg_opr_queue.put(
(self._requeue,
RabbitMQServer._retry_queue_name(queue_name), body,
message, retry_cnt,
rabbitmq_request.rpc_info.target_ts.ToMilliseconds()))
self._msg_opr_queue.put((self._ack_message, message))
return
task_context = RabbitMQTaskContext(rabbitmq_request,
service_context)
servicer.set_current_context(task_context)
start_time = time.time()
input_msg = self._server._method_by_method_fullname[
method_name].input_type._concrete_class()
input_msg.ParseFromString(request_body)
servicer_func = self._server._service_func_by_method_fullname[
method_name]
servicer_func(input_msg, service_context)
except RabbitMQServicerContext.Finished:
pass
except Exception:
logging.exception("Exception while handling method %s",
method_name)
service_context.set_code(grpc.StatusCode.INTERNAL)
canonical_method_name = '/'.join(method_name.rsplit('.', 1))
if start_time is not None:
elapsed = time.time() - start_time
SERVER_LATENCY.labels(servicer.name(), canonical_method_name,
'rabbitmq').observe(elapsed)
status_code = service_context.get_code() or grpc.StatusCode.OK
need_message_ack = True
job_done = False
if status_code == grpc.StatusCode.INTERNAL:
max_retries = self._server._retry_coordinator.max_retries(
rpc_info, status_code)
if retry_cnt < max_retries:
try:
if message.channel.connection:
backoff = self._server._retry_coordinator.backoff(
rpc_info, status_code, retry_cnt)
self._msg_opr_queue.put(
(self._requeue,
RabbitMQServer._retry_queue_name(queue_name),
body, message, retry_cnt + 1, backoff))
else:
logging.warning(
'Failed to requeue rabbitmq message')
need_message_ack = False
job_done = True
except Exception:
logging.exception("Could not requeue message")
need_message_ack = False
job_done = True
self._msg_opr_queue.put(
(self._reject_message, message))
else:
need_message_ack = False
job_done = True
logging.error("Rejecting message after %s retries",
retry_cnt)
self._msg_opr_queue.put((self._reject_message, message))
else:
job_done = True
if need_message_ack:
self._msg_opr_queue.put((self._ack_message, message))
SERVER_STATUS_CODE.labels(servicer.name(), canonical_method_name,
status_code.name, 'rabbitmq').inc()
if job_done:
now = time.mktime(datetime.utcnow().timetuple())
if job_timestamp:
JOB_LATENCY.labels(caller_service_name, servicer.name(),
canonical_method_name,
status_code.name).observe(now -
job_timestamp)
JOB_RETRIES.labels(caller_service_name, servicer.name(),
canonical_method_name,
status_code.name).observe(retry_cnt)