def merge_topic(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD] # unit_action_status = UnitActionStatus(type=action.type) unit_action_status = MergeRowAction(type=action.type) start = time.time() pipeline_uid = context[PIPELINE_UID] unit_action_status.uid = pipeline_uid if action.topicId is None: raise ValueError("action.topicId is empty {0}".format(action.name)) target_topic = get_topic_by_id(action.topicId) mapping_results = run_mapping_rules(action.mapping, target_topic, raw_data, pipeline_topic, context) joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, target_topic, context) unit_action_status.whereConditions = where_condition unit_action_status.mapping = mapping_results trigger_pipeline_data_list = [] for index, mapping_result in enumerate(mapping_results): mongo_query = __build_mongo_query(joint_type, index_conditions(where_condition, index)) target_data = query_topic_data(mongo_query, target_topic.name) if target_data is None: raise Exception("can't insert data in merge row action ") else: trigger_pipeline_data_list.append( update_topic_data(target_topic.name, mapping_result, target_data, pipeline_uid, mongo_query)) unit_action_status.updateCount = unit_action_status.updateCount + 1 elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status, trigger_pipeline_data_list
def merge_or_insert_topic(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[ pipeline_constants.OLD] unit_action_status = UnitStatus(type=action.type) start = time.time() pipeline_uid = context[PIPELINE_UID] unit_action_status.uid = pipeline_uid if action.topicId is None: raise ValueError("action.topicId is empty {0}".format(action.name)) target_topic = get_topic_by_id(action.topicId) mapping_results, mapping_logs = run_mapping_rules( action.mapping, target_topic, raw_data, pipeline_topic) joint_type, where_condition = build_query_conditions( action.by, pipeline_topic, raw_data, target_topic, context) for index in range(len(mapping_results)): mongo_query = __build_mongo_query(joint_type, where_condition) target_data = query_topic_data(mongo_query, target_topic.name) if target_data is None: insert_topic_data(target_topic.name, mapping_results[index], pipeline_uid) unit_action_status.insertCount = unit_action_status.insertCount + 1 else: update_topic_data(target_topic.name, mapping_results[index], target_data, pipeline_uid) unit_action_status.updateCount = unit_action_status.updateCount + 1 unit_action_status.mapping = mapping_logs elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status
def insert_topic(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[ pipeline_constants.OLD] unit_action_status = InsertAction(type=action.type) start = time.time() pipeline_uid = context[PIPELINE_UID] unit_action_status.uid = pipeline_uid # print(json.loads(raw_data)) if action.topicId is None: raise ValueError("action.topicId is empty {0}".format(action.name)) target_topic = get_topic_by_id(action.topicId) log.info("run target_topic {0}".format(target_topic.name)) mapping_results = run_mapping_rules(action.mapping, target_topic, raw_data, pipeline_topic, context) log.info("mapping_results:{0}".format(mapping_results)) unit_action_status.mapping = mapping_results trigger_pipeline_data_list = [] for index, item in enumerate(mapping_results): trigger_pipeline_data_list.append( insert_topic_data(target_topic.name, item, pipeline_uid)) unit_action_status.insertCount = unit_action_status.insertCount + 1 elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status, trigger_pipeline_data_list
def insert_topic(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[ pipeline_constants.OLD] unit_action_status = UnitStatus(type=action.type) start = time.time() pipeline_uid = context[PIPELINE_UID] unit_action_status.uid = pipeline_uid if action.topicId is None: raise ValueError("action.topicId is empty {0}".format(action.name)) insert_action = InsertAction() target_topic = get_topic_by_id(action.topicId) mapping_results, mapping_logs = run_mapping_rules( action.mapping, target_topic, raw_data, pipeline_topic) # for i enumerate(mapping_results) for count, item in enumerate(mapping_results): # print(count, item) insert_topic_data(target_topic.name, item, pipeline_uid) unit_action_status.insertCount = unit_action_status.insertCount + 1 # for index in range(len(mapping_results)): insert_action.mapping = mapping_logs # unit_action_status.mapping = mapping_logs elapsed_time = time.time() - start unit_action_status.action = insert_action unit_action_status.complete_time = elapsed_time return context, unit_action_status