def merge_or_insert_topic(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[ pipeline_constants.OLD] unit_action_status = UnitStatus(type=action.type) start = time.time() pipeline_uid = context[PIPELINE_UID] unit_action_status.uid = pipeline_uid if action.topicId is None: raise ValueError("action.topicId is empty {0}".format(action.name)) target_topic = get_topic_by_id(action.topicId) mapping_results, mapping_logs = run_mapping_rules( action.mapping, target_topic, raw_data, pipeline_topic) joint_type, where_condition = build_query_conditions( action.by, pipeline_topic, raw_data, target_topic, context) for index in range(len(mapping_results)): mongo_query = __build_mongo_query(joint_type, where_condition) target_data = query_topic_data(mongo_query, target_topic.name) if target_data is None: insert_topic_data(target_topic.name, mapping_results[index], pipeline_uid) unit_action_status.insertCount = unit_action_status.insertCount + 1 else: update_topic_data(target_topic.name, mapping_results[index], target_data, pipeline_uid) unit_action_status.updateCount = unit_action_status.updateCount + 1 unit_action_status.mapping = mapping_logs elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status
def merge_topic(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD] # unit_action_status = UnitActionStatus(type=action.type) unit_action_status = MergeRowAction(type=action.type) start = time.time() pipeline_uid = context[PIPELINE_UID] unit_action_status.uid = pipeline_uid if action.topicId is None: raise ValueError("action.topicId is empty {0}".format(action.name)) target_topic = get_topic_by_id(action.topicId) mapping_results = run_mapping_rules(action.mapping, target_topic, raw_data, pipeline_topic, context) joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, target_topic, context) unit_action_status.whereConditions = where_condition unit_action_status.mapping = mapping_results trigger_pipeline_data_list = [] for index, mapping_result in enumerate(mapping_results): mongo_query = __build_mongo_query(joint_type, index_conditions(where_condition, index)) target_data = query_topic_data(mongo_query, target_topic.name) if target_data is None: raise Exception("can't insert data in merge row action ") else: trigger_pipeline_data_list.append( update_topic_data(target_topic.name, mapping_result, target_data, pipeline_uid, mongo_query)) unit_action_status.updateCount = unit_action_status.updateCount + 1 elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status, trigger_pipeline_data_list
def read_factor(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD] unit_action_status = ReadFactorAction(type=action.type) start = time.time() # print("context",context) variable_type, context_target_name = process_variable(action.variableName) topic = get_topic_by_id(action.topicId) factor = get_factor(action.factorId, topic) joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, topic, context) mongo_query = __build_mongo_query(joint_type, where_condition) target_data = query_topic_data(mongo_query, topic.name) if target_data is not None: if factor.name in target_data: read_value = target_data[factor.name] if factor.name in context: log.warn("factor name {0} is already in context".format(factor.name)) context[context_target_name] = target_data[factor.name] unit_action_status.value = read_value else: context[context_target_name] = convert_factor_type(factor.defaultValue, factor.type) log.warn("target_data is empty ,conditions {0}".format(mongo_query)) elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time # print("read context",context) return context, unit_action_status, []
def write_factor(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[ pipeline_constants.OLD] unit_action_status = WriteFactorAction(type=action.type) start = time.time() if action.topicId is not None: target_topic = get_topic_by_id(action.topicId) # todo for find factor # factor_dict = build_factor_dict(target_topic) conditions = action.by joint_type, where_condition = build_query_conditions( conditions, pipeline_topic, raw_data, target_topic, context) target_factor = get_factor(action.factorId, target_topic) # if action.source.kind source_value_list = __run_arithmetic( action.arithmetic, get_source_value_list(pipeline_topic, raw_data, action.source, target_factor, context)) update_data = {target_factor.name: source_value_list} mongo_query = __build_mongo_query(joint_type, where_condition) condition_factors = { "$set": get_condition_factor_value(raw_data, where_condition, joint_type) } trigger_pipeline_data_list = [] target_data = query_topic_data(mongo_query, target_topic.name) if old_value is not None: old_value_list = get_source_value_list(pipeline_topic, old_value, action.source, target_factor, context) trigger_pipeline_data_list.append( find_and_modify_topic_data( target_topic.name, mongo_query, __merge_condition_factor( __build_mongo_update(update_data, action.arithmetic, target_factor, old_value_list), condition_factors), target_data)) else: trigger_pipeline_data_list.append( find_and_modify_topic_data( target_topic.name, mongo_query, __merge_condition_factor( __build_mongo_update(update_data, action.arithmetic, target_factor), condition_factors), target_data)) elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status, trigger_pipeline_data_list
def write_factor(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[ pipeline_constants.OLD] unit_action_status = UnitStatus(type=action.type) start = time.time() pipeline_uid = context[PIPELINE_UID] # TODO action_log # action_log = WriteFactorAction() if action.topicId is not None: target_topic = get_topic_by_id(action.topicId) # todo for find factor factor_dict = build_factor_dict(target_topic) conditions = action.by joint_type, where_condition = build_query_conditions( conditions, pipeline_topic, raw_data, target_topic, context) source_value_list = get_source_value_list(pipeline_topic, raw_data, action.source) target_factor = get_factor(action.factorId, target_topic) update_data = {target_factor.name: source_value_list} mongo_query = __build_mongo_query(joint_type, where_condition) target_data = query_topic_data(mongo_query, target_topic.name) if target_data is None: condition_factors = get_condition_factor_value( raw_data, where_condition, joint_type) insert_data = { **{ target_factor.name: source_value_list }, **condition_factors } log.info("Insert data : {0}".format(insert_data)) insert_topic_data(target_topic.name, insert_data, pipeline_uid) else: if old_value is not None: old_value_list = get_source_value_list( pipeline_topic, old_value, action.source) # def_value_list = source_value_list - old_value_list # update_data = {target_factor.name: def_value_list} find_and_modify_topic_data( target_topic.name, mongo_query, __build_mongo_update(update_data, action.arithmetic, target_factor, old_value_list), target_data) else: find_and_modify_topic_data( target_topic.name, mongo_query, __build_mongo_update(update_data, action.arithmetic, target_factor, None), target_data) elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status
def read_row(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD] unit_action_status = UnitActionStatus(type=action.type) start = time.time() variable_type, context_target_name = process_variable(action.variableName) topic = get_topic_by_id(action.topicId) joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, topic, context) mongo_query = __build_mongo_query(joint_type, where_condition) target_data = query_topic_data(mongo_query, topic.name) if target_data is not None: context[context_target_name] = target_data elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status, []
def read_factor(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD] unit_action_status = UnitActionStatus(type=action.type) start = time.time() variable_type, context_target_name = process_variable(action.variableName) topic = get_topic_by_id(action.topicId) factor = get_factor(action.factorId, topic) joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, topic, context) mongo_query = __build_mongo_query(joint_type, where_condition) target_data = query_topic_data(mongo_query, topic.name) if factor.name in target_data: read_value = target_data[factor.name] context[context_target_name] = target_data[factor.name] build_action_log(factor, read_value, topic, unit_action_status) elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time return context, unit_action_status