def update_retry_callback(mappings_results: dict, where_: dict,
target_topic: Topic, current_user: User):
target_data = query_topic_data(where_, target_topic, current_user)
if target_data is not None:
id_ = target_data.get(
get_id_name_by_datasource(
data_source_container.get_data_source_by_id(
target_topic.dataSourceId)), None)
version_ = target_data.get("version_", None)
if id_ is not None and version_ is not None:
mappings_results['version_'] = version_
template = get_template_by_datasource_id(target_topic.dataSourceId)
template.topic_data_update_one_with_version(
id_, version_, mappings_results, target_topic.name)
data = {**target_data, **mappings_results}
return TriggerData(topicName=target_topic.name,
triggerType="Update",
data={
"new": data,
"old": target_data
})
else:
raise RuntimeError(
"when do update topic {0}, the id_ {1} and version_ {2} should not be None"
.format(target_topic.name, id_, version_))
else:
raise RuntimeError(
"update topic {0} failed, the target record is not exist. where: {1}"
.format(target_topic.name, where_))
def update_recovery_callback(mappings_results: dict, where_: dict,
target_topic: Topic):
log.error(
"The maximum number of retry times (3) is exceeded, retry failed. Do recovery, "
"mappings_results: {0}, where: {1}, target_topic: {2}".format(
mappings_results, where_, target_topic))
target_data = query_topic_data(where_, target_topic.name)
if target_data is not None:
id_ = target_data.get(
get_id_name_by_datasource(
data_source_container.get_data_source_by_id(
target_topic.dataSourceId)), None)
if id_ is not None:
template = get_template_by_datasource_id(target_topic.dataSourceId)
template.topic_data_update_one(id_, mappings_results,
target_topic.name)
data = {**target_data, **mappings_results}
return TriggerData(topicName=target_topic.name,
triggerType="Update",
data={
"new": data,
"old": target_data
})
else:
raise RuntimeError(
"when do update topic {0}, the id_ {1} should not be None".
format(target_topic.name, id_))
else:
raise RuntimeError(
"target topic {0} recovery failed. the record is not exist. where: {1}"
.format(target_topic.name, where_))
def insert_topic_data(mapping_result, pipeline_uid, topic: Topic,
current_user):
check_current_user(current_user)
add_audit_columns(mapping_result, INSERT)
add_tenant_id_to_instance(mapping_result, current_user)
add_trace_columns(mapping_result, "insert_row", pipeline_uid)
if __need_encrypt():
__encrypt_value(
__find_encrypt_factor_in_mapping_result(mapping_result, topic),
mapping_result, current_user)
template = get_template_by_datasource_id(topic.dataSourceId)
template.topic_data_insert_one(mapping_result, topic.name)
return __build_trigger_pipeline_data(topic.name, {
pipeline_constants.NEW: mapping_result,
pipeline_constants.OLD: None
}, TriggerType.insert)
def update_topic_data_one(mapping_result, target_data, pipeline_uid, id_,
topic: Topic, current_user):
check_current_user(current_user)
template = get_template_by_datasource_id(topic.dataSourceId)
old_data = template.topic_data_find_by_id(
target_data[get_id_name_by_datasource(
data_source_container.get_data_source_by_id(topic.dataSourceId))],
topic.name)
add_audit_columns(mapping_result, UPDATE)
add_trace_columns(mapping_result, "update_row", pipeline_uid)
if __need_encrypt():
__encrypt_value(
__find_encrypt_factor_in_mapping_result(mapping_result, topic),
mapping_result, current_user)
add_tenant_id_to_instance(mapping_result, current_user)
template.topic_data_update_one(id_, mapping_result, topic.name)
data = {**target_data, **mapping_result}
return __build_trigger_pipeline_data(topic.name, {
pipeline_constants.NEW: data,
pipeline_constants.OLD: old_data
}, TriggerType.update)
def sync_pipeline_monitor_data(pipeline_monitor: PipelineRunStatus):
code = "raw_pipeline_monitor"
data = pipeline_monitor.dict()
raw_data = {
"data_": data,
"tenant_id_": pipeline_monitor.tenantId,
"traceid": pipeline_monitor.traceId
}
topic = find_monitor_topic(code, pipeline_monitor.currentUser)
trace_id = get_surrogate_key()
if topic is None:
raise Exception(code + " topic name does not exist")
add_audit_columns(raw_data, INSERT)
flatten_fields = get_flatten_field(data, topic.factors)
raw_data.update(flatten_fields)
storage_template = get_template_by_datasource_id(topic.dataSourceId)
storage_template.topic_data_insert_one(raw_data, code)
watchmen.pipeline.index.trigger_pipeline(code, {
pipeline_constants.NEW: data,
pipeline_constants.OLD: None
}, TriggerType.insert, pipeline_monitor.currentUser, trace_id)
def query_topic_data_aggregate(where_, aggregate, topic: Topic, current_user: User):
template = get_template_by_datasource_id(topic.dataSourceId)
return template.topic_data_find_with_aggregate(__merge_tenant_id_to_where_condition(where_, current_user),
topic.name, aggregate)
def query_multiple_topic_data(where_, topic: Topic, current_user: User):
template = get_template_by_datasource_id(topic.dataSourceId)
return template.topic_data_find_(__merge_tenant_id_to_where_condition(where_, current_user), topic.name)
def find_topic_data_by_id_and_topic_name(topic: Topic, object_id) -> Topic:
template = get_template_by_datasource_id(topic.dataSourceId)
return template.topic_data_find_by_id(object_id, topic.name)
def save_topic_instance(topic: Topic, instance, current_user=None):
template = get_template_by_datasource_id(topic.dataSourceId)
return template.topic_data_insert_one(
add_tenant_id_to_instance(instance, current_user), topic.name)
def get_topic_instances_all(topic: Topic):
template = get_template_by_datasource_id(topic.dataSourceId)
return template.topic_data_list_all(topic.name)
def get_topic_instances(topic: Topic, conditions):
template = get_template_by_datasource_id(topic.dataSourceId)
return template.topic_data_find_(conditions, topic.name)
def update_topic_instance(topic: Topic, instance, instance_id):
template = get_template_by_datasource_id(topic.dataSourceId)
return template.topic_data_update_one(instance_id, instance, topic.name)
def save_topic_instances(topic: Topic, instances):
template = get_template_by_datasource_id(topic.dataSourceId)
return template.topic_data_insert_(instances, topic.name)
def query_pipeline_monitor(topic_name, query, pagination, current_user=None):
topic = find_monitor_topic(topic_name, current_user)
storage_template = get_template_by_datasource_id(topic.dataSourceId)
result = storage_template.topic_data_page_(query, None, pagination, None,
topic_name)
return result