def __push_events_to_kafka(topic, events):
call_start = datetime.datetime.now()
if __get_uid() not in config.UIDS_TO_POST_EVENT:
logging.info('[#OAUTH_401] Received uuid is not valid for posting: %s', flask.request.token_info.get("uid"))
return {'detail': 'Not Authorized. You are not allowed to use this endpoint'}, 401
if not __topic_exists(topic):
return {'detail': 'Topic does not exist'}, 422
failed = 0
for event in events:
if 'partitioning_key' in event:
key = event['partitioning_key']
else:
key = event['ordering_key']
try:
retry_if_failed(__produce_kafka_message, topic.encode('utf-8'), key.encode('utf-8'), json.dumps(event).encode('utf-8'))
except:
failed += 1
ms_elapsed = monitoring.stop_time_measure(call_start)
logging.info('[#KAFKA_PUSH_TIME] Time spent total for pushing to kafka: %s ms', ms_elapsed)
metrics_writer.log_events(__get_uid(), topic, "-", len(events) - failed, 0)
if failed > 0:
return {'detail': 'Failed to write %s event(s) to kafka' % failed}, 503
else:
return {}, 201
def process_batch(latest_offsets, current_batch):
for partition in partitions:
topic_partition = (topic.encode('UTF-8'), partition)
# send the messages we could read so far
if len(current_batch[partition]) > 0:
stream_message = __create_stream_message(partition, latest_offsets[topic_partition],
current_batch[partition])
with __measure_time(current_batch[partition], stream_message):
yield stream_message
metrics_writer.log_events(uid, topic, partition, 0, len(current_batch[partition]))
# just send the keep alive
else:
yield __create_stream_message(partition, latest_offsets[topic_partition])
def process_batch(latest_offsets, current_batch):
for partition in partitions:
topic_partition = (topic.encode('UTF-8'), partition)
# send the messages we could read so far
if len(current_batch[partition]) > 0:
stream_message = __create_stream_message(
partition, latest_offsets[topic_partition],
current_batch[partition])
with __measure_time(current_batch[partition], stream_message):
yield stream_message
metrics_writer.log_events(uid, topic, partition, 0,
len(current_batch[partition]))
# just send the keep alive
else:
yield __create_stream_message(partition,
latest_offsets[topic_partition])
