async def import_raw_topic_data(topic_event):
topic = get_topic(topic_event.code)
if topic is None:
raise Exception(topic_event.code + " topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
await __trigger_pipeline(topic_event)
def insert_topic_data(topic_name, mapping_result, pipeline_uid):
collection_name = build_collection_name(topic_name)
codec_options = build_code_options()
collection = db.get_collection(collection_name, codec_options=codec_options)
add_audit_columns(mapping_result, INSERT)
add_trace_columns(mapping_result, "insert_row", pipeline_uid)
collection.insert(mapping_result)
trigger_pipeline(topic_name, {pipeline_constants.NEW: mapping_result, pipeline_constants.OLD: None},
TriggerType.insert)
def update_topic_data(topic_name, mapping_result, target_data, pipeline_uid):
collection_name = build_collection_name(topic_name)
codec_options = build_code_options()
collection = db.get_collection(collection_name, codec_options=codec_options)
old_data = find_topic_data_by_id(collection, target_data["_id"])
add_audit_columns(mapping_result, UPDATE)
add_trace_columns(mapping_result, "update_row", pipeline_uid)
collection.update_one({"_id": target_data["_id"]}, {"$set": mapping_result})
data = {**target_data, **mapping_result}
trigger_pipeline(topic_name, {pipeline_constants.NEW: data, pipeline_constants.OLD: old_data}, TriggerType.update)
async def save_topic_data(topic_event: TopicEvent):
# TODO user check URP
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
await __trigger_pipeline(topic_event)
return {"received": True}
def sync_alarm_message(alarm: AlarmMessage):
topic_event = TopicEvent(code="alarm", data=alarm.dict())
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
trigger_pipeline(topic_event.code, {
pipeline_constants.NEW: topic_event.data,
pipeline_constants.OLD: None
}, TriggerType.insert)
asyncio.ensure_future(send_notifier(alarm))
def sync_pipeline_monitor_data(pipeline_monitor: PipelineRunStatus):
topic_event = TopicEvent(code="raw_pipeline_monitor",
data=pipeline_monitor.dict())
# asyncio.ensure_future(import_raw_topic_data(topic_event))
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
watchmen.pipeline.index.trigger_pipeline(
topic_event.code, {
pipeline_constants.NEW: topic_event.data,
pipeline_constants.OLD: None
}, TriggerType.insert)
def update_topic_data(topic_name, mapping_result, target_data, pipeline_uid, mongo_query):
'''
collection_name = build_collection_name(topic_name)
codec_options = build_code_options()
collection = db.get_collection(collection_name, codec_options=codec_options)
old_data = find_topic_data_by_id(collection, target_data["_id"])
'''
old_data = topic_data_find_by_id(target_data[__get_key()], topic_name)
add_audit_columns(mapping_result, UPDATE)
add_trace_columns(mapping_result, "update_row", pipeline_uid)
'''
collection.update_one({"_id": target_data["_id"]}, {"$set": mapping_result})
'''
topic_data_update_(topic_name, mongo_query, mapping_result)
data = {**target_data, **mapping_result}
return __build_trigger_pipeline_data(topic_name,
{pipeline_constants.NEW: data, pipeline_constants.OLD: old_data},
TriggerType.update)
