def set_result(self, retry_cnt, f, exception=None):
exception = exception or f.exception()
success = exception is None
do_retry = False
if success:
self.result.set_result(f.result())
else:
if issubclass(type(exception), grpc.Call):
do_retry = retry_cnt < (
retry.RetryCoordinator.max_retries_by_code.get(
exception.code()) or 0)
if do_retry:
asyncio.ensure_future(self.timeout_retry(retry_cnt),
loop=self.sanic_loop)
else:
self.result.set_exception(exception)
if not do_retry:
CLIENT_RETRIES.labels(
servicer.name(), self.remote_service_name, self.method._method,
get_response_conde(exception)).observe(retry_cnt)
elapsed = time.time() - self.start_time
CLIENT_LATENCY.labels(
servicer.name(),
self.remote_service_name,
self.method._method,
).observe(elapsed)
def process_response(response, exception=None):
exception = exception or response.exception()
elapsed = time.time() - start_time
CLIENT_ATTEMPT_LATENCY.labels(
servicer.name(), self._remote_service_name,
self._remote_instance_name, call_details.method,
self._get_response_conde(exception)).observe(elapsed)
CLIENT_IN_PROGRESS_REQUESTS.labels(servicer.name(),
self._remote_service_name,
call_details.method).dec()
def before_request():
CLIENT_IN_PROGRESS_REQUESTS.labels(servicer.name(),
self._remote_service_name,
call_details.method).inc()
CLIENT_ATTEMPT_DEADLINE.labels(
servicer.name(),
self._remote_service_name,
self._remote_instance_name,
call_details.method,
).observe(timeout or 1000)
def _try_on_message(self, *args):
try:
SERVER_FREE_WORKERS.labels(servicer.name(), 'rabbitmq').dec()
SERVER_IN_PROGRESS.labels(servicer.name(), 'rabbitmq').inc()
self._in_progress += 1
self._on_message(*args)
except:
logging.exception(
"Unrecoverable exception while handling message")
finally:
SERVER_FREE_WORKERS.labels(servicer.name(), 'rabbitmq').inc()
SERVER_IN_PROGRESS.labels(servicer.name(), 'rabbitmq').dec()
self._in_progress -= 1
def do(request, timeout=None, delay=None):
if timeout is not None:
logging.warning("Ignoring timeout for rabbitmq stub")
rabbitmq_request = rabbitmq_pb2.RabbitMqRpcMessage()
rabbitmq_request.request = method.input_type._concrete_class.SerializeToString(
request)
rabbitmq_request.rpc_info.method = '/' + method.full_name
rabbitmq_request.rpc_info.caller_service = servicer.name()
rabbitmq_request.rpc_info.ts.FromDatetime(
datetime.datetime.utcnow())
if delay is not None:
rabbitmq_request.rpc_info.target_ts.FromDatetime(
datetime.datetime.utcnow() + delay)
context = servicer.current_context()
if context is not None and context.request_state is not None:
rabbitmq_request.experiments_data = context.request_state.SerializeToString(
)
if context is not None and context.trace_info is not None:
rabbitmq_request.trace_info.CopyFrom(context.trace_info)
routing_key = "rpc.%s.%s" % (self._service_name,
method.full_name)
self._rabbitmq_client._publish_with_retry(
3, self._exchange, routing_key, {},
rabbitmq_request.SerializeToString())
def setup_context(request, context):
task_context = GrpcTaskContext(context)
servicer.set_current_context(task_context)
start_time = time.time()
SERVER_FREE_WORKERS.labels(servicer.name(), 'grpc').dec()
SERVER_IN_PROGRESS.labels(servicer.name(), 'grpc').inc()
try:
response = handler.unary_unary(request, context)
response_metadata = task_context.response_metadata()
if response_metadata:
context.set_trailing_metadata(response_metadata)
finally:
servicer.set_current_context(None)
SERVER_FREE_WORKERS.labels(servicer.name(), 'grpc').inc()
SERVER_IN_PROGRESS.labels(servicer.name(), 'grpc').dec()
elapsed = time.time() - start_time
SERVER_LATENCY.labels(servicer.name(),
handler_call_details.method,
'grpc').observe(elapsed)
return response
def _before_request(request):
state = headers.get_state(request.headers)
trace_info = headers.get_trace_info(request.headers)
servicer.set_current_context(SanicTaskContext(state, trace_info))
SERVER_FREE_WORKERS.labels(servicer.name(), 'sanic').dec()
SERVER_IN_PROGRESS.labels(servicer.name(), 'sanic').inc()
def _after_request(request, response):
SERVER_FREE_WORKERS.labels(servicer.name(), 'sanic').inc()
SERVER_IN_PROGRESS.labels(servicer.name(), 'sanic').dec()
def _after_exception(sender, exception, **extra):
SERVER_FREE_WORKERS.labels(servicer.name(), 'flask').inc()
SERVER_IN_PROGRESS.labels(servicer.name(), 'flask').dec()
def _after_request(sender, response, **extra):
SERVER_FREE_WORKERS.labels(servicer.name(), 'flask').inc()
SERVER_IN_PROGRESS.labels(servicer.name(), 'flask').dec()
def process_exception(self, request, exception):
if getattr(request, 'counted_request', False):
SERVER_FREE_WORKERS.labels(servicer.name(), 'django').inc()
SERVER_IN_PROGRESS.labels(servicer.name(), 'django').dec()
request.counted_request = False
def process_request(self, request):
SERVER_FREE_WORKERS.labels(servicer.name(), 'django').dec()
SERVER_IN_PROGRESS.labels(servicer.name(), 'django').inc()
request.counted_request = True
servicer.set_current_context(DjangoTaskContext())
def _on_message(self, queue_name, body, message):
if not message.channel.connection:
return
retry_cnt = int(message.headers.get('retry_cnt', '0'))
service_context = RabbitMQServicerContext(30)
start_time = None
try:
method_name = message.headers.get('method')
execute_at = message.headers.get('execute_at')
rabbitmq_request = rabbitmq_pb2.RabbitMqRpcMessage()
job_timestamp = None
caller_service_name = None
rpc_info = None
if execute_at and not RabbitMQServer._should_execute(
int(execute_at)):
self._msg_opr_queue.put(
(self._requeue,
RabbitMQServer._retry_queue_name(queue_name), body,
message, None, None))
self._msg_opr_queue.put((self._ack_message, message))
return
if method_name:
request_body = body
method_name = '/' + method_name
state_bin = message.headers.get(REQUEST_STATE)
if state_bin is not None:
rabbitmq_request.experiments_data = state_bin.encode()
else:
rabbitmq_request.ParseFromString(body)
method_name = rabbitmq_request.rpc_info.method
request_body = rabbitmq_request.request
job_timestamp = time.mktime(
rabbitmq_request.rpc_info.ts.ToDatetime().timetuple())
caller_service_name = rabbitmq_request.rpc_info.caller_service
rpc_info = rabbitmq_request.rpc_info
if retry_cnt == 0 and rabbitmq_request.rpc_info.HasField(
"target_ts"):
target_ts = rabbitmq_request.rpc_info.target_ts.ToDatetime(
)
else:
target_ts = datetime.utcnow()
if datetime.utcnow() < target_ts:
self._msg_opr_queue.put(
(self._requeue,
RabbitMQServer._retry_queue_name(queue_name), body,
message, retry_cnt,
rabbitmq_request.rpc_info.target_ts.ToMilliseconds()))
self._msg_opr_queue.put((self._ack_message, message))
return
task_context = RabbitMQTaskContext(rabbitmq_request,
service_context)
servicer.set_current_context(task_context)
start_time = time.time()
input_msg = self._server._method_by_method_fullname[
method_name].input_type._concrete_class()
input_msg.ParseFromString(request_body)
servicer_func = self._server._service_func_by_method_fullname[
method_name]
servicer_func(input_msg, service_context)
except RabbitMQServicerContext.Finished:
pass
except Exception:
logging.exception("Exception while handling method %s",
method_name)
service_context.set_code(grpc.StatusCode.INTERNAL)
canonical_method_name = '/'.join(method_name.rsplit('.', 1))
if start_time is not None:
elapsed = time.time() - start_time
SERVER_LATENCY.labels(servicer.name(), canonical_method_name,
'rabbitmq').observe(elapsed)
status_code = service_context.get_code() or grpc.StatusCode.OK
need_message_ack = True
job_done = False
if status_code == grpc.StatusCode.INTERNAL:
max_retries = self._server._retry_coordinator.max_retries(
rpc_info, status_code)
if retry_cnt < max_retries:
try:
if message.channel.connection:
backoff = self._server._retry_coordinator.backoff(
rpc_info, status_code, retry_cnt)
self._msg_opr_queue.put(
(self._requeue,
RabbitMQServer._retry_queue_name(queue_name),
body, message, retry_cnt + 1, backoff))
else:
logging.warning(
'Failed to requeue rabbitmq message')
need_message_ack = False
job_done = True
except Exception:
logging.exception("Could not requeue message")
need_message_ack = False
job_done = True
self._msg_opr_queue.put(
(self._reject_message, message))
else:
need_message_ack = False
job_done = True
logging.error("Rejecting message after %s retries",
retry_cnt)
self._msg_opr_queue.put((self._reject_message, message))
else:
job_done = True
if need_message_ack:
self._msg_opr_queue.put((self._ack_message, message))
SERVER_STATUS_CODE.labels(servicer.name(), canonical_method_name,
status_code.name, 'rabbitmq').inc()
if job_done:
now = time.mktime(datetime.utcnow().timetuple())
if job_timestamp:
JOB_LATENCY.labels(caller_service_name, servicer.name(),
canonical_method_name,
status_code.name).observe(now -
job_timestamp)
JOB_RETRIES.labels(caller_service_name, servicer.name(),
canonical_method_name,
status_code.name).observe(retry_cnt)