def __push_events_to_kafka(topic, events):
call_start = datetime.datetime.now()
if __get_uid() not in config.UIDS_TO_POST_EVENT:
logging.info('[#OAUTH_401] Received uuid is not valid for posting: %s', flask.request.token_info.get("uid"))
return {'detail': 'Not Authorized. You are not allowed to use this endpoint'}, 401
if not __topic_exists(topic):
return {'detail': 'Topic does not exist'}, 422
failed = 0
for event in events:
if 'partitioning_key' in event:
key = event['partitioning_key']
else:
key = event['ordering_key']
try:
retry_if_failed(__produce_kafka_message, topic.encode('utf-8'), key.encode('utf-8'), json.dumps(event).encode('utf-8'))
except:
failed += 1
ms_elapsed = monitoring.stop_time_measure(call_start)
logging.info('[#KAFKA_PUSH_TIME] Time spent total for pushing to kafka: %s ms', ms_elapsed)
metrics_writer.log_events(__get_uid(), topic, "-", len(events) - failed, 0)
if failed > 0:
return {'detail': 'Failed to write %s event(s) to kafka' % failed}, 503
else:
return {}, 201
def post_event(topic):
call_start = datetime.datetime.now()
if not __uid_is_valid_to_post():
logging.info('[#OAUTH_401] Received uuid is not valid for posting: %s', flask.request.token_info.get("uid"))
return {'detail': 'Not Authorized. You are not allowed to use this endpoint'}, 401
if not __topic_exists(topic):
return {'detail': 'Topic does not exist'}, 422
event = flask.request.json
logging.info('[#GOTEVENT] Received event:\n%s', event)
key = event['partitioning_key']
logging.debug('Using key %s', key)
try:
retry_if_failed(__produce_kafka_message, topic.encode('utf-8'), key.encode('utf-8'), json.dumps(event).encode('utf-8'))
except:
return {'detail': 'Failed to write event to kafka'}, 503
ms_elapsed = monitoring.stop_time_measure(call_start)
logging.info('[#POST_TIME_TOTAL] Time spent total %s ms', ms_elapsed)
return {}, 201
def post_events(topic):
call_start = datetime.datetime.now()
encoding = flask.request.headers.get('Content-Encoding')
if encoding == 'gzip':
try:
logging.info('Received compressed body. Uncompressing...')
fake_file = BytesIO(flask.request.data)
uncompressed = gzip.GzipFile(fileobj=fake_file, mode='r')
json_bytes = uncompressed.read()
json_data = flask.json.loads(json_bytes)
except:
return {'detail': 'Body decompression failed'}, 422
else:
json_data = flask.request.json
ms_elapsed = monitoring.stop_time_measure(call_start)
logging.info('Received batch of %s events', len(json_data))
logging.info('[#DECMP_DESRL_TIME] Time spent on uncompression/deserialization: %s ms', ms_elapsed)
return __push_events_to_kafka(topic, json_data)
