async def sync_query_monitor_data(query_monitor: QueryMonitor):
# print(query_monitor)
topic_event = TopicEvent(code="raw_query_monitor",
data=query_monitor.dict())
# payload = {'code': "raw_query_monitor", "data": query_monitor.json()}
# print(settings.HOST_URL+"/topic/data")
# print(topic_event.json())
asyncio.ensure_future(import_raw_topic_data(topic_event))
def sync_alarm_message(alarm: AlarmMessage):
topic_event = TopicEvent(code="alarm", data=alarm.dict())
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
trigger_pipeline(topic_event.code, {
pipeline_constants.NEW: topic_event.data,
pipeline_constants.OLD: None
}, TriggerType.insert)
asyncio.ensure_future(send_notifier(alarm))
def sync_pipeline_monitor_data(pipeline_monitor: PipelineRunStatus):
topic_event = TopicEvent(code="raw_pipeline_monitor",
data=pipeline_monitor.dict())
# asyncio.ensure_future(import_raw_topic_data(topic_event))
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
watchmen.pipeline.index.trigger_pipeline(
topic_event.code, {
pipeline_constants.NEW: topic_event.data,
pipeline_constants.OLD: None
}, TriggerType.insert)
async def consume():
consumer = AIOKafkaConsumer(
kafka_topics_list,
loop=loop,
bootstrap_servers=settings.KAFKA_BOOTSTRAP_SERVER,
value_deserializer=lambda m: json.loads(m.decode('utf-8')))
await consumer.start()
try:
async for msg in consumer:
topic_event = TopicEvent.parse_obj(msg.value)
await import_raw_topic_data(topic_event)
except:
log.error(traceback.format_exc())
await consume()
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
async def sync_query_monitor_data(query_monitor: QueryMonitor):
topic_event = TopicEvent(code="raw_query_monitor",
data=query_monitor.dict())
asyncio.ensure_future(import_raw_topic_data(topic_event))