async def save_topic_data(topic, data, current_user):
add_audit_columns(data, INSERT)
if is_raw(topic):
flatten_fields = get_flatten_field(data["data_"], topic.factors)
data.update(flatten_fields)
else:
data = process_factor_format(topic,data)
save_topic_instance(topic, data, current_user)
def sync_alarm_message(alarm: AlarmMessage):
topic_event = TopicEvent(code="alarm", data=alarm.dict())
topic = get_topic(topic_event.code)
if topic is None:
raise Exception("topic name does not exist")
add_audit_columns(topic_event.data, INSERT)
save_topic_instance(topic_event.code, topic_event.data)
trigger_pipeline(topic_event.code, {
pipeline_constants.NEW: topic_event.data,
pipeline_constants.OLD: None
}, TriggerType.insert)
asyncio.ensure_future(send_notifier(alarm))
async def import_raw_topic_data(topic_event, current_user):
topic = get_topic(topic_event.code, current_user)
if topic is None:
raise Exception(topic_event.code + " topic name does not exist")
raw_data = await get_input_data(topic, topic_event)
add_audit_columns(raw_data, INSERT)
flatten_fields = get_flatten_field(topic_event.data, topic.factors)
raw_data.update(flatten_fields)
save_topic_instance(topic, raw_data, current_user)
__trigger_pipeline(topic_event, current_user)
def insert_topic_data(mapping_result, pipeline_uid, topic: Topic,
current_user):
check_current_user(current_user)
add_audit_columns(mapping_result, INSERT)
add_tenant_id_to_instance(mapping_result, current_user)
add_trace_columns(mapping_result, "insert_row", pipeline_uid)
if __need_encrypt():
__encrypt_value(
__find_encrypt_factor_in_mapping_result(mapping_result, topic),
mapping_result, current_user)
template = get_template_by_datasource_id(topic.dataSourceId)
template.topic_data_insert_one(mapping_result, topic.name)
return __build_trigger_pipeline_data(topic.name, {
pipeline_constants.NEW: mapping_result,
pipeline_constants.OLD: None
}, TriggerType.insert)
async def patch_topic_instance(topic_name,
instance_id=None,
instance=Body(...),
current_user: User = Depends(
deps.get_current_user)):
topic = get_topic_by_name(topic_name, current_user)
if instance_id is None:
add_audit_columns(instance, INSERT)
return save_topic_instance(topic, instance, current_user)
else:
result = find_topic_data_by_id_and_topic_name(topic, instance_id)
if result is not None:
add_audit_columns(instance, UPDATE)
return update_topic_instance(topic, instance, instance_id)
else:
raise Exception(
"instance ID {0} could not find any data for update".format(
instance_id))
def update_topic_data_one(mapping_result, target_data, pipeline_uid, id_,
topic: Topic, current_user):
check_current_user(current_user)
template = get_template_by_datasource_id(topic.dataSourceId)
old_data = template.topic_data_find_by_id(
target_data[get_id_name_by_datasource(
data_source_container.get_data_source_by_id(topic.dataSourceId))],
topic.name)
add_audit_columns(mapping_result, UPDATE)
add_trace_columns(mapping_result, "update_row", pipeline_uid)
if __need_encrypt():
__encrypt_value(
__find_encrypt_factor_in_mapping_result(mapping_result, topic),
mapping_result, current_user)
add_tenant_id_to_instance(mapping_result, current_user)
template.topic_data_update_one(id_, mapping_result, topic.name)
data = {**target_data, **mapping_result}
return __build_trigger_pipeline_data(topic.name, {
pipeline_constants.NEW: data,
pipeline_constants.OLD: old_data
}, TriggerType.update)
def sync_pipeline_monitor_data(pipeline_monitor: PipelineRunStatus):
code = "raw_pipeline_monitor"
data = pipeline_monitor.dict()
raw_data = {
"data_": data,
"tenant_id_": pipeline_monitor.tenantId,
"traceid": pipeline_monitor.traceId
}
topic = find_monitor_topic(code, pipeline_monitor.currentUser)
trace_id = get_surrogate_key()
if topic is None:
raise Exception(code + " topic name does not exist")
add_audit_columns(raw_data, INSERT)
flatten_fields = get_flatten_field(data, topic.factors)
raw_data.update(flatten_fields)
storage_template = get_template_by_datasource_id(topic.dataSourceId)
storage_template.topic_data_insert_one(raw_data, code)
watchmen.pipeline.index.trigger_pipeline(code, {
pipeline_constants.NEW: data,
pipeline_constants.OLD: None
}, TriggerType.insert, pipeline_monitor.currentUser, trace_id)