def _format_request(self,
rpc,
to_topic,
reply_topic,
**kwargs):
"""
Format a request to send over kafka
:param rpc: Requested remote API
:param to_topic: Topic to send the request
:param reply_topic: Topic to receive the resulting response, if any
:param kwargs: Dictionary of key-value pairs to pass as arguments to
the remote rpc API.
:return: A InterContainerMessage message type on success or None on
failure
"""
try:
transaction_id = uuid4().hex
request = InterContainerMessage()
request_body = InterContainerRequestBody()
request.header.id = transaction_id
request.header.type = MessageType.Value("REQUEST")
request.header.from_topic = reply_topic
request.header.to_topic = to_topic
response_required = False
if reply_topic:
request_body.reply_to_topic = reply_topic
request_body.response_required = True
response_required = True
request.header.timestamp = int(round(time.time() * 1000))
request_body.rpc = rpc
for a, b in kwargs.iteritems():
arg = Argument()
arg.key = a
try:
arg.value.Pack(b)
request_body.args.extend([arg])
except Exception as e:
log.exception("Failed-parsing-value", e=e)
request.body.Pack(request_body)
return request, transaction_id, response_required
except Exception as e:
log.exception("formatting-request-failed",
rpc=rpc,
to_topic=to_topic,
reply_topic=reply_topic,
args=kwargs)
return None, None, None
def _handle_message(self, msg, start_time): # pylint: disable=too-many-locals, too-many-branches
"""
Default internal method invoked for every batch of messages received
from Kafka.
"""
if self.stopped:
returnValue(None)
def _augment_args_with_from_topic(args, from_topic):
arg = Argument(key=ARG_FROM_TOPIC)
arg.value.Pack(StrType(val=from_topic))
args.extend([arg])
return args
def _to_dict(args):
"""
Convert a repeatable Argument type into a python dictionary
:param args: Repeatable core_adapter.Argument type
:return: a python dictionary
"""
return {arg.key: arg.value
for arg in args} if args is not None else None
try:
val = msg.value()
log.debug("rx-msg", message=msg)
# Go over customized callbacks first
m_topic = msg.topic()
if m_topic in self.topic_callback_map:
for topic_cb in self.topic_callback_map[m_topic]:
yield topic_cb(val)
# Check whether we need to process request/response scenario
if m_topic not in self.topic_target_cls_map:
returnValue(None)
# Process request/response scenario
message = InterContainerMessage()
message.ParseFromString(val)
if message.header.type == MessageType.Value("REQUEST"):
# Get the target class for that specific topic
targetted_topic = self._to_string(message.header.to_topic)
msg_body = InterContainerRequestBody()
if message.body.Is(InterContainerRequestBody.DESCRIPTOR):
message.body.Unpack(msg_body)
else:
log.debug("unsupported-msg", msg_type=type(message.body))
returnValue(None)
# Extract opentrace span from the message
with self.enrich_context_with_span(msg_body.rpc,
msg_body.args) as scope:
# log.debug('rx-span')
span = scope.span if scope is not None else None
if targetted_topic in self.topic_target_cls_map:
# // let the callee unpack the arguments as its the only one that knows the real proto type
# // Augment the requestBody with the message Id as it will be used in scenarios where cores
# // are set in pairs and competing
# requestBody.Args = kp.addTransactionId(msg.Header.Id, requestBody.Args)
# Augment the request arguments with the from_topic
augmented_args = _augment_args_with_from_topic(
msg_body.args, msg_body.reply_to_topic)
try:
rpc = msg_body.rpc
rpc_str = self._to_string(rpc)
if augmented_args:
log.debug(
"message-body-args-present",
rpc=rpc_str,
response_required=msg_body.
response_required,
reply_to_topic=msg_body.reply_to_topic)
(status, res) = yield getattr(
self.topic_target_cls_map[targetted_topic],
rpc_str)(**_to_dict(augmented_args))
else:
log.debug(
"message-body-args-absent",
rpc=rpc_str,
response_required=msg_body.
response_required,
reply_to_topic=msg_body.reply_to_topic)
(status, res) = yield getattr(
self.topic_target_cls_map[targetted_topic],
rpc_str)()
if self.rx_stats:
self.rx_stats.requests.increment(
time.monotonic() - start_time, rpc_str)
if msg_body.response_required:
response = self._format_response(
msg_header=message.header,
msg_body=res,
status=status)
if response is not None:
res_topic = self._to_string(
response.header.to_topic)
res_span, span = span, None
self.send_kafka_message(
res_topic, response, res_span)
if self.rx_stats:
self.rx_stats.responses.increment(
time.monotonic() - start_time,
rpc_str)
log.debug("response-sent",
to_topic=res_topic,
rpc=rpc_str,
delta=time.monotonic() -
start_time)
else:
log.debug("no-response",
rpc=rpc_str,
delta=time.monotonic() -
start_time)
except Exception as _e:
# TODO: set error in span
log.exception('request-failure', e=_e)
finally:
if span is not None:
span.finish()
elif message.header.type == MessageType.Value("RESPONSE"):
trns_id = self._to_string(message.header.id)
log.debug('received-response', transaction_id=trns_id)
wait_for_response = self.transaction_id_deferred_map.pop(
trns_id, None)
if wait_for_response:
resp = self._parse_response(val)
wait_for_response.callback(resp)
else:
log.error("INVALID-TRANSACTION-TYPE")
except Exception as e:
log.exception("Failed-to-process-message", message=msg, e=e)
def _process_message(self, m):
"""
Default internal method invoked for every batch of messages received
from Kafka.
"""
def _augment_args_with_FromTopic(args, from_topic):
arg = Argument(key=ARG_FROM_TOPIC)
t = StrType(val=from_topic)
arg.value.Pack(t)
args.extend([arg])
return args
def _toDict(args):
"""
Convert a repeatable Argument type into a python dictionary
:param args: Repeatable core_adapter.Argument type
:return: a python dictionary
"""
if args is None:
return None
result = {}
for arg in args:
assert isinstance(arg, Argument)
result[arg.key] = arg.value
return result
current_time = int(time.time() * 1000)
# log.debug("Got Message", message=m)
try:
val = m.value()
# val = m.message.value
# print m.topic
# Go over customized callbacks first
m_topic = m.topic()
if m_topic in self.topic_callback_map:
for c in self.topic_callback_map[m_topic]:
yield c(val)
# Check whether we need to process request/response scenario
if m_topic not in self.topic_target_cls_map:
return
# Process request/response scenario
message = InterContainerMessage()
message.ParseFromString(val)
if message.header.type == MessageType.Value("REQUEST"):
# Get the target class for that specific topic
targetted_topic = self._to_string(message.header.to_topic)
msg_body = InterContainerRequestBody()
if message.body.Is(InterContainerRequestBody.DESCRIPTOR):
message.body.Unpack(msg_body)
else:
log.debug("unsupported-msg", msg_type=type(message.body))
return
if targetted_topic in self.topic_target_cls_map:
# Augment the request arguments with the from_topic
augmented_args = _augment_args_with_FromTopic(
msg_body.args, msg_body.reply_to_topic)
if augmented_args:
log.debug("message-body-args-present",
rpc=msg_body.rpc,
response_required=msg_body.response_required,
reply_to_topic=msg_body.reply_to_topic)
(status, res) = yield getattr(
self.topic_target_cls_map[targetted_topic],
self._to_string(
msg_body.rpc))(**_toDict(augmented_args))
else:
log.debug(
"message-body-args-absent",
rpc=msg_body.rpc,
response_required=msg_body.response_required,
reply_to_topic=msg_body.reply_to_topic,
)
(status, res) = yield getattr(
self.topic_target_cls_map[targetted_topic],
self._to_string(msg_body.rpc))()
if msg_body.response_required:
response = self._format_response(
msg_header=message.header,
msg_body=res,
status=status,
)
if response is not None:
res_topic = self._to_string(
response.header.to_topic)
self._send_kafka_message(res_topic, response)
log.debug("Response-sent", to_topic=res_topic)
elif message.header.type == MessageType.Value("RESPONSE"):
trns_id = self._to_string(message.header.id)
if trns_id in self.transaction_id_deferred_map:
resp = self._parse_response(val)
self.transaction_id_deferred_map[trns_id].callback(resp)
else:
log.error("!!INVALID-TRANSACTION-TYPE!!")
except Exception as e:
log.exception("Failed-to-process-message", message=m, e=e)