def merge_topic(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD]
# unit_action_status = UnitActionStatus(type=action.type)
unit_action_status = MergeRowAction(type=action.type)
start = time.time()
pipeline_uid = context[PIPELINE_UID]
unit_action_status.uid = pipeline_uid
if action.topicId is None:
raise ValueError("action.topicId is empty {0}".format(action.name))
target_topic = get_topic_by_id(action.topicId)
mapping_results = run_mapping_rules(action.mapping, target_topic, raw_data, pipeline_topic, context)
joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, target_topic, context)
unit_action_status.whereConditions = where_condition
unit_action_status.mapping = mapping_results
trigger_pipeline_data_list = []
for index, mapping_result in enumerate(mapping_results):
mongo_query = __build_mongo_query(joint_type, index_conditions(where_condition, index))
target_data = query_topic_data(mongo_query, target_topic.name)
if target_data is None:
raise Exception("can't insert data in merge row action ")
else:
trigger_pipeline_data_list.append(
update_topic_data(target_topic.name, mapping_result, target_data, pipeline_uid, mongo_query))
unit_action_status.updateCount = unit_action_status.updateCount + 1
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status, trigger_pipeline_data_list
def merge_or_insert_topic(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[
pipeline_constants.OLD]
unit_action_status = UnitStatus(type=action.type)
start = time.time()
pipeline_uid = context[PIPELINE_UID]
unit_action_status.uid = pipeline_uid
if action.topicId is None:
raise ValueError("action.topicId is empty {0}".format(action.name))
target_topic = get_topic_by_id(action.topicId)
mapping_results, mapping_logs = run_mapping_rules(
action.mapping, target_topic, raw_data, pipeline_topic)
joint_type, where_condition = build_query_conditions(
action.by, pipeline_topic, raw_data, target_topic, context)
for index in range(len(mapping_results)):
mongo_query = __build_mongo_query(joint_type, where_condition)
target_data = query_topic_data(mongo_query, target_topic.name)
if target_data is None:
insert_topic_data(target_topic.name, mapping_results[index],
pipeline_uid)
unit_action_status.insertCount = unit_action_status.insertCount + 1
else:
update_topic_data(target_topic.name, mapping_results[index],
target_data, pipeline_uid)
unit_action_status.updateCount = unit_action_status.updateCount + 1
unit_action_status.mapping = mapping_logs
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status
def insert_topic(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[
pipeline_constants.OLD]
unit_action_status = InsertAction(type=action.type)
start = time.time()
pipeline_uid = context[PIPELINE_UID]
unit_action_status.uid = pipeline_uid
# print(json.loads(raw_data))
if action.topicId is None:
raise ValueError("action.topicId is empty {0}".format(action.name))
target_topic = get_topic_by_id(action.topicId)
log.info("run target_topic {0}".format(target_topic.name))
mapping_results = run_mapping_rules(action.mapping, target_topic,
raw_data, pipeline_topic, context)
log.info("mapping_results:{0}".format(mapping_results))
unit_action_status.mapping = mapping_results
trigger_pipeline_data_list = []
for index, item in enumerate(mapping_results):
trigger_pipeline_data_list.append(
insert_topic_data(target_topic.name, item, pipeline_uid))
unit_action_status.insertCount = unit_action_status.insertCount + 1
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
return context, unit_action_status, trigger_pipeline_data_list
def insert_topic(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[
pipeline_constants.OLD]
unit_action_status = UnitStatus(type=action.type)
start = time.time()
pipeline_uid = context[PIPELINE_UID]
unit_action_status.uid = pipeline_uid
if action.topicId is None:
raise ValueError("action.topicId is empty {0}".format(action.name))
insert_action = InsertAction()
target_topic = get_topic_by_id(action.topicId)
mapping_results, mapping_logs = run_mapping_rules(
action.mapping, target_topic, raw_data, pipeline_topic)
# for i enumerate(mapping_results)
for count, item in enumerate(mapping_results):
# print(count, item)
insert_topic_data(target_topic.name, item, pipeline_uid)
unit_action_status.insertCount = unit_action_status.insertCount + 1
# for index in range(len(mapping_results)):
insert_action.mapping = mapping_logs
# unit_action_status.mapping = mapping_logs
elapsed_time = time.time() - start
unit_action_status.action = insert_action
unit_action_status.complete_time = elapsed_time
return context, unit_action_status
