def merge_or_insert_topic(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[
pipeline_constants.OLD]
unit_action_status = UnitStatus(type=action.type)
start = time.time()
pipeline_uid = context[PIPELINE_UID]
unit_action_status.uid = pipeline_uid
if action.topicId is None:
raise ValueError("action.topicId is empty {0}".format(action.name))
target_topic = get_topic_by_id(action.topicId)
mapping_results, mapping_logs = run_mapping_rules(
action.mapping, target_topic, raw_data, pipeline_topic)
joint_type, where_condition = build_query_conditions(
action.by, pipeline_topic, raw_data, target_topic, context)
for index in range(len(mapping_results)):
mongo_query = __build_mongo_query(joint_type, where_condition)
target_data = query_topic_data(mongo_query, target_topic.name)
if target_data is None:
insert_topic_data(target_topic.name, mapping_results[index],
pipeline_uid)
unit_action_status.insertCount = unit_action_status.insertCount + 1
else:
update_topic_data(target_topic.name, mapping_results[index],
target_data, pipeline_uid)
unit_action_status.updateCount = unit_action_status.updateCount + 1
unit_action_status.mapping = mapping_logs
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status
def merge_topic(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD]
# unit_action_status = UnitActionStatus(type=action.type)
unit_action_status = MergeRowAction(type=action.type)
start = time.time()
pipeline_uid = context[PIPELINE_UID]
unit_action_status.uid = pipeline_uid
if action.topicId is None:
raise ValueError("action.topicId is empty {0}".format(action.name))
target_topic = get_topic_by_id(action.topicId)
mapping_results = run_mapping_rules(action.mapping, target_topic, raw_data, pipeline_topic, context)
joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, target_topic, context)
unit_action_status.whereConditions = where_condition
unit_action_status.mapping = mapping_results
trigger_pipeline_data_list = []
for index, mapping_result in enumerate(mapping_results):
mongo_query = __build_mongo_query(joint_type, index_conditions(where_condition, index))
target_data = query_topic_data(mongo_query, target_topic.name)
if target_data is None:
raise Exception("can't insert data in merge row action ")
else:
trigger_pipeline_data_list.append(
update_topic_data(target_topic.name, mapping_result, target_data, pipeline_uid, mongo_query))
unit_action_status.updateCount = unit_action_status.updateCount + 1
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status, trigger_pipeline_data_list
def read_factor(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD]
unit_action_status = ReadFactorAction(type=action.type)
start = time.time()
# print("context",context)
variable_type, context_target_name = process_variable(action.variableName)
topic = get_topic_by_id(action.topicId)
factor = get_factor(action.factorId, topic)
joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, topic, context)
mongo_query = __build_mongo_query(joint_type, where_condition)
target_data = query_topic_data(mongo_query, topic.name)
if target_data is not None:
if factor.name in target_data:
read_value = target_data[factor.name]
if factor.name in context:
log.warn("factor name {0} is already in context".format(factor.name))
context[context_target_name] = target_data[factor.name]
unit_action_status.value = read_value
else:
context[context_target_name] = convert_factor_type(factor.defaultValue, factor.type)
log.warn("target_data is empty ,conditions {0}".format(mongo_query))
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
# print("read context",context)
return context, unit_action_status, []
def write_factor(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[
pipeline_constants.OLD]
unit_action_status = WriteFactorAction(type=action.type)
start = time.time()
if action.topicId is not None:
target_topic = get_topic_by_id(action.topicId)
# todo for find factor
# factor_dict = build_factor_dict(target_topic)
conditions = action.by
joint_type, where_condition = build_query_conditions(
conditions, pipeline_topic, raw_data, target_topic, context)
target_factor = get_factor(action.factorId, target_topic)
# if action.source.kind
source_value_list = __run_arithmetic(
action.arithmetic,
get_source_value_list(pipeline_topic, raw_data, action.source,
target_factor, context))
update_data = {target_factor.name: source_value_list}
mongo_query = __build_mongo_query(joint_type, where_condition)
condition_factors = {
"$set":
get_condition_factor_value(raw_data, where_condition,
joint_type)
}
trigger_pipeline_data_list = []
target_data = query_topic_data(mongo_query, target_topic.name)
if old_value is not None:
old_value_list = get_source_value_list(pipeline_topic,
old_value,
action.source,
target_factor, context)
trigger_pipeline_data_list.append(
find_and_modify_topic_data(
target_topic.name, mongo_query,
__merge_condition_factor(
__build_mongo_update(update_data,
action.arithmetic,
target_factor,
old_value_list),
condition_factors), target_data))
else:
trigger_pipeline_data_list.append(
find_and_modify_topic_data(
target_topic.name, mongo_query,
__merge_condition_factor(
__build_mongo_update(update_data,
action.arithmetic,
target_factor),
condition_factors), target_data))
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status, trigger_pipeline_data_list
def write_factor(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[
pipeline_constants.OLD]
unit_action_status = UnitStatus(type=action.type)
start = time.time()
pipeline_uid = context[PIPELINE_UID]
# TODO action_log
# action_log = WriteFactorAction()
if action.topicId is not None:
target_topic = get_topic_by_id(action.topicId)
# todo for find factor
factor_dict = build_factor_dict(target_topic)
conditions = action.by
joint_type, where_condition = build_query_conditions(
conditions, pipeline_topic, raw_data, target_topic, context)
source_value_list = get_source_value_list(pipeline_topic, raw_data,
action.source)
target_factor = get_factor(action.factorId, target_topic)
update_data = {target_factor.name: source_value_list}
mongo_query = __build_mongo_query(joint_type, where_condition)
target_data = query_topic_data(mongo_query, target_topic.name)
if target_data is None:
condition_factors = get_condition_factor_value(
raw_data, where_condition, joint_type)
insert_data = {
**{
target_factor.name: source_value_list
},
**condition_factors
}
log.info("Insert data : {0}".format(insert_data))
insert_topic_data(target_topic.name, insert_data, pipeline_uid)
else:
if old_value is not None:
old_value_list = get_source_value_list(
pipeline_topic, old_value, action.source)
# def_value_list = source_value_list - old_value_list
# update_data = {target_factor.name: def_value_list}
find_and_modify_topic_data(
target_topic.name, mongo_query,
__build_mongo_update(update_data, action.arithmetic,
target_factor, old_value_list),
target_data)
else:
find_and_modify_topic_data(
target_topic.name, mongo_query,
__build_mongo_update(update_data, action.arithmetic,
target_factor, None), target_data)
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status
def read_row(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD]
unit_action_status = UnitActionStatus(type=action.type)
start = time.time()
variable_type, context_target_name = process_variable(action.variableName)
topic = get_topic_by_id(action.topicId)
joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, topic, context)
mongo_query = __build_mongo_query(joint_type, where_condition)
target_data = query_topic_data(mongo_query, topic.name)
if target_data is not None:
context[context_target_name] = target_data
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status, []
def read_factor(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD]
unit_action_status = UnitActionStatus(type=action.type)
start = time.time()
variable_type, context_target_name = process_variable(action.variableName)
topic = get_topic_by_id(action.topicId)
factor = get_factor(action.factorId, topic)
joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, topic, context)
mongo_query = __build_mongo_query(joint_type, where_condition)
target_data = query_topic_data(mongo_query, topic.name)
if factor.name in target_data:
read_value = target_data[factor.name]
context[context_target_name] = target_data[factor.name]
build_action_log(factor, read_value, topic, unit_action_status)
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status
