def __push_events_to_kafka(topic, events): call_start = datetime.datetime.now() if __get_uid() not in config.UIDS_TO_POST_EVENT: logging.info('[#OAUTH_401] Received uuid is not valid for posting: %s', flask.request.token_info.get("uid")) return {'detail': 'Not Authorized. You are not allowed to use this endpoint'}, 401 if not __topic_exists(topic): return {'detail': 'Topic does not exist'}, 422 failed = 0 for event in events: if 'partitioning_key' in event: key = event['partitioning_key'] else: key = event['ordering_key'] try: retry_if_failed(__produce_kafka_message, topic.encode('utf-8'), key.encode('utf-8'), json.dumps(event).encode('utf-8')) except: failed += 1 ms_elapsed = monitoring.stop_time_measure(call_start) logging.info('[#KAFKA_PUSH_TIME] Time spent total for pushing to kafka: %s ms', ms_elapsed) metrics_writer.log_events(__get_uid(), topic, "-", len(events) - failed, 0) if failed > 0: return {'detail': 'Failed to write %s event(s) to kafka' % failed}, 503 else: return {}, 201
def post_event(topic): call_start = datetime.datetime.now() if not __uid_is_valid_to_post(): logging.info('[#OAUTH_401] Received uuid is not valid for posting: %s', flask.request.token_info.get("uid")) return {'detail': 'Not Authorized. You are not allowed to use this endpoint'}, 401 if not __topic_exists(topic): return {'detail': 'Topic does not exist'}, 422 event = flask.request.json logging.info('[#GOTEVENT] Received event:\n%s', event) key = event['partitioning_key'] logging.debug('Using key %s', key) try: retry_if_failed(__produce_kafka_message, topic.encode('utf-8'), key.encode('utf-8'), json.dumps(event).encode('utf-8')) except: return {'detail': 'Failed to write event to kafka'}, 503 ms_elapsed = monitoring.stop_time_measure(call_start) logging.info('[#POST_TIME_TOTAL] Time spent total %s ms', ms_elapsed) return {}, 201
def post_events(topic): call_start = datetime.datetime.now() encoding = flask.request.headers.get('Content-Encoding') if encoding == 'gzip': try: logging.info('Received compressed body. Uncompressing...') fake_file = BytesIO(flask.request.data) uncompressed = gzip.GzipFile(fileobj=fake_file, mode='r') json_bytes = uncompressed.read() json_data = flask.json.loads(json_bytes) except: return {'detail': 'Body decompression failed'}, 422 else: json_data = flask.request.json ms_elapsed = monitoring.stop_time_measure(call_start) logging.info('Received batch of %s events', len(json_data)) logging.info('[#DECMP_DESRL_TIME] Time spent on uncompression/deserialization: %s ms', ms_elapsed) return __push_events_to_kafka(topic, json_data)