def merge_topic():
# begin time
start = time.time()
# create action status monitor
status = ActionStatus()
status.type = "merge-row"
status.uid = action_context.get_pipeline_id()
previous_data = action_context.previousOfTriggerData
current_data = action_context.currentOfTriggerData
action = action_context.action
if action.topicId is None:
raise ValueError("action.topicId is empty {0}".format(
action.topicId))
pipeline_topic = action_context.unitContext.stageContext.pipelineContext.pipelineTopic
target_topic = get_topic_by_id(action.topicId)
variables = get_variables(action_context)
# if there are aggregate functions, need lock the record to update
mappings_results, having_aggregate_functions = parse_mappings(
action.mapping, target_topic, previous_data, current_data,
variables)
status.value = mappings_results
where_ = parse_parameter_joint(action.by, current_data, variables,
pipeline_topic, target_topic)
status.by = where_
trigger_pipeline_data_list = []
target_data = query_topic_data(where_, target_topic,
action_context.get_current_user())
if target_data is None:
raise Exception("can't insert data in merge row action ")
else:
if target_topic.type == "aggregate":
args = [
mappings_results, where_, target_topic,
action_context.get_current_user()
]
retry_callback = (update_retry_callback, args)
recovery_callback = (update_recovery_callback, args)
execute_ = retry_template(retry_callback, recovery_callback,
RetryPolicy())
result = execute_()
trigger_pipeline_data_list.append(result)
else:
trigger_pipeline_data_list.append(
update_topic_data_one(mappings_results, target_data,
action_context.get_pipeline_id(),
target_data[get_id_name()],
target_topic,
action_context.get_current_user()))
status.updateCount = status.updateCount + 1
elapsed_time = time.time() - start
status.completeTime = elapsed_time
return status, trigger_pipeline_data_list
def exists():
# begin time
start = time.time()
# create action status monitor
status = ActionStatus()
status.type = "exists"
status.uid = action_context.get_pipeline_id()
previous_data = action_context.previousOfTriggerData
current_data = action_context.currentOfTriggerData
action = action_context.action
pipeline_topic = action_context.get_pipeline_context().pipelineTopic
target_topic = get_topic_by_id(action.topicId)
variables = get_variables(action_context)
where_ = parse_parameter_joint(action.by, current_data, variables,
pipeline_topic, target_topic)
status.by = where_
target_data = query_topic_data(where_, target_topic,
action_context.get_current_user())
if target_data is not None:
set_variable(action_context, action.variableName, 'true')
else:
set_variable(action_context, action.variableName, 'false')
elapsed_time = time.time() - start
status.completeTime = elapsed_time
return status, []
def update_retry_callback(mappings_results: dict, where_: dict,
target_topic: Topic, current_user: User):
target_data = query_topic_data(where_, target_topic, current_user)
if target_data is not None:
id_ = target_data.get(
get_id_name_by_datasource(
data_source_container.get_data_source_by_id(
target_topic.dataSourceId)), None)
version_ = target_data.get("version_", None)
if id_ is not None and version_ is not None:
mappings_results['version_'] = version_
template = get_template_by_datasource_id(target_topic.dataSourceId)
template.topic_data_update_one_with_version(
id_, version_, mappings_results, target_topic.name)
data = {**target_data, **mappings_results}
return TriggerData(topicName=target_topic.name,
triggerType="Update",
data={
"new": data,
"old": target_data
})
else:
raise RuntimeError(
"when do update topic {0}, the id_ {1} and version_ {2} should not be None"
.format(target_topic.name, id_, version_))
else:
raise RuntimeError(
"update topic {0} failed, the target record is not exist. where: {1}"
.format(target_topic.name, where_))
def update_recovery_callback(mappings_results: dict, where_: dict,
target_topic: Topic):
log.error(
"The maximum number of retry times (3) is exceeded, retry failed. Do recovery, "
"mappings_results: {0}, where: {1}, target_topic: {2}".format(
mappings_results, where_, target_topic))
target_data = query_topic_data(where_, target_topic.name)
if target_data is not None:
id_ = target_data.get(
get_id_name_by_datasource(
data_source_container.get_data_source_by_id(
target_topic.dataSourceId)), None)
if id_ is not None:
template = get_template_by_datasource_id(target_topic.dataSourceId)
template.topic_data_update_one(id_, mappings_results,
target_topic.name)
data = {**target_data, **mappings_results}
return TriggerData(topicName=target_topic.name,
triggerType="Update",
data={
"new": data,
"old": target_data
})
else:
raise RuntimeError(
"when do update topic {0}, the id_ {1} should not be None".
format(target_topic.name, id_))
else:
raise RuntimeError(
"target topic {0} recovery failed. the record is not exist. where: {1}"
.format(target_topic.name, where_))
def read_factor():
# begin time
start = time.time()
# create action status monitor
status = ActionStatus()
status.type = "read-factor"
status.uid = action_context.unitContext.stageContext.pipelineContext.pipeline.pipelineId
previous_data = action_context.previousOfTriggerData
current_data = action_context.currentOfTriggerData
action = action_context.action
pipeline_topic = action_context.unitContext.stageContext.pipelineContext.pipelineTopic
target_topic = get_topic_by_id(action.topicId)
variables = get_variables(action_context)
where_ = parse_parameter_joint(action.by, current_data, variables, pipeline_topic, target_topic)
status.by = where_
target_factor = get_factor(action.factorId, target_topic)
if action.arithmetic == "none" or action.arithmetic is None:
target_data = query_topic_data(where_, target_topic, action_context.get_current_user())
if target_data is not None:
if isinstance(target_data, list):
raise ValueError("read factor action should just get one factor record")
else:
read_value = target_data[target_factor.name]
set_variable(action_context, action.variableName, read_value)
status.value = read_value
else:
raise ValueError("read factor action must match one factor record")
else:
read_value = None
if action.arithmetic == "sum":
read_value = query_topic_data_aggregate(where_,
{target_factor.name: "sum"},
target_topic, action_context.get_current_user())
elif action.arithmetic == "count":
read_value = query_topic_data_aggregate(where_,
{target_factor.name: "count"},
target_topic, action_context.get_current_user())
elif action.arithmetic == "avg":
read_value = query_topic_data_aggregate(where_,
{target_factor.name: "avg"},
target_topic,
action_context.get_current_user())
if read_value is not None:
set_variable(action_context, action.variableName, read_value)
else:
raise ValueError("read factor action must match one factor record at least")
status.value = read_value
elapsed_time = time.time() - start
status.completeTime = elapsed_time
return status, []
def read_row():
# begin time
start = time.time()
# create action status monitor
status = ActionStatus()
status.type = "read-row"
status.uid = action_context.unitContext.stageContext.pipelineContext.pipeline.pipelineId
previous_data = action_context.previousOfTriggerData
current_data = action_context.currentOfTriggerData
action = action_context.action
target_topic = get_topic_by_id(action.topicId)
pipeline_topic = action_context.unitContext.stageContext.pipelineContext.pipelineTopic
variables = get_variables(action_context)
where_ = parse_parameter_joint(action.by, current_data, variables,
pipeline_topic, target_topic)
status.by = where_
# print(where_)
target_data = query_topic_data(where_, target_topic,
action_context.get_current_user())
if target_data is not None:
if isinstance(target_data, list):
raise ValueError(
"read row action should just get one row record")
else:
set_variable(action_context, action.variableName, target_data)
status.value = target_data
else:
raise ValueError("read row action must match one record at least")
elapsed_time = time.time() - start
status.completeTime = elapsed_time
return status, []
def write_factor():
start = time.time()
# create action status monitor
status = ActionStatus()
status.type = "write-factor"
status.uid = action_context.get_pipeline_id()
previous_data = action_context.previousOfTriggerData
current_data = action_context.currentOfTriggerData
action = action_context.action
if action.topicId is not None:
pipeline_topic = action_context.get_pipeline_context(
).pipelineTopic
target_topic = get_topic_by_id(action.topicId)
variables = get_variables(action_context)
where_ = parse_parameter_joint(action.by, current_data, variables,
pipeline_topic, target_topic)
status.by = where_
target_data = query_topic_data(where_, target_topic,
action_context.get_current_user())
target_factor = get_factor(action.factorId, target_topic)
source_ = action.source
arithmetic = action.arithmetic
result = None
current_value_ = check_and_convert_value_by_factor(
target_factor, parse_parameter(source_, current_data,
variables))
if arithmetic is None or arithmetic == "none": # mean AS IS
result = {target_factor.name: current_value_}
elif arithmetic == "sum":
previous_value_ = check_and_convert_value_by_factor(
target_factor,
parse_parameter(source_, previous_data, variables))
if previous_value_ is None:
previous_value_ = 0
value_ = Decimal(current_value_) - Decimal(previous_value_)
result = {target_factor.name: {"_sum": value_}}
elif arithmetic == "count":
if previous_data is None:
result = {target_factor.name: {"_count": 1}}
else:
result = {target_factor.name: {"_count": 0}}
elif arithmetic == "avg":
result = {target_factor.name: {"_avg": current_value_}}
updates_ = result
trigger_pipeline_data_list = []
if target_data is not None:
if target_topic.type == "aggregate":
args = [
updates_, where_, target_topic,
action_context.get_current_user()
]
retry_callback = (update_retry_callback, args)
recovery_callback = (update_recovery_callback, args)
execute_ = retry_template(retry_callback,
recovery_callback, RetryPolicy())
result = execute_()
trigger_pipeline_data_list.append(result)
else:
trigger_pipeline_data_list.append(
update_topic_data_one(
updates_, target_data,
action_context.get_pipeline_id(),
target_data[get_id_name_by_datasource(
data_source_container.get_data_source_by_id(
target_topic.dataSourceId))], target_topic,
action_context.get_current_user()))
else:
raise Exception("can't insert data in write factor action ")
status.updateCount = status.updateCount + 1
elapsed_time = time.time() - start
status.completeTime = elapsed_time
return status, trigger_pipeline_data_list