async def import_raw_topic_data(topic_event):
topic = get_topic(topic_event.code)
if topic is None:
raise Exception(topic_event.code + " topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
await __trigger_pipeline(topic_event)
async def save_topic_data(topic, data, current_user):
add_audit_columns(data, INSERT)
if is_raw(topic):
flatten_fields = get_flatten_field(data["data_"], topic.factors)
data.update(flatten_fields)
else:
data = process_factor_format(topic,data)
save_topic_instance(topic, data, current_user)
async def save_topic_data(topic_event: TopicEvent):
# TODO user check URP
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
await __trigger_pipeline(topic_event)
return {"received": True}
def sync_alarm_message(alarm: AlarmMessage):
topic_event = TopicEvent(code="alarm", data=alarm.dict())
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
trigger_pipeline(topic_event.code, {
pipeline_constants.NEW: topic_event.data,
pipeline_constants.OLD: None
}, TriggerType.insert)
asyncio.ensure_future(send_notifier(alarm))
async def import_raw_topic_data(topic_event, current_user):
topic = get_topic(topic_event.code, current_user)
if topic is None:
raise Exception(topic_event.code + " topic name does not exist")
raw_data = await get_input_data(topic, topic_event)
add_audit_columns(raw_data, INSERT)
flatten_fields = get_flatten_field(topic_event.data, topic.factors)
raw_data.update(flatten_fields)
save_topic_instance(topic, raw_data, current_user)
__trigger_pipeline(topic_event, current_user)
def sync_pipeline_monitor_data(pipeline_monitor: PipelineRunStatus):
topic_event = TopicEvent(code="raw_pipeline_monitor",
data=pipeline_monitor.dict())
# asyncio.ensure_future(import_raw_topic_data(topic_event))
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
watchmen.pipeline.index.trigger_pipeline(
topic_event.code, {
pipeline_constants.NEW: topic_event.data,
pipeline_constants.OLD: None
}, TriggerType.insert)
async def patch_topic_instance(topic_name,
instance_id=None,
instance=Body(...),
current_user: User = Depends(
deps.get_current_user)):
topic = get_topic_by_name(topic_name, current_user)
if instance_id is None:
add_audit_columns(instance, INSERT)
return save_topic_instance(topic, instance, current_user)
else:
result = find_topic_data_by_id_and_topic_name(topic, instance_id)
if result is not None:
add_audit_columns(instance, UPDATE)
return update_topic_instance(topic, instance, instance_id)
else:
raise Exception(
"instance ID {0} could not find any data for update".format(
instance_id))