def __process_parameter_constants(parameter: Parameter, context, target_factor=None): variable_type, context_target_name = process_variable(parameter.value) if variable_type == parameter_constants.CONSTANT: if target_factor is not None: if SPLIT_FLAG in parameter.value: value_list = __split_value(parameter.value) result = [] for value in value_list: result.append( convert_factor_type(value, target_factor.type)) return result else: return convert_factor_type(parameter.value, target_factor.type) else: return parameter.value elif variable_type == parameter_constants.MEMORY: if context_target_name in context: return context[context_target_name] else: raise ValueError( "no variable {0} in context".format(context_target_name)) else: raise ValueError("variable_type is invalid")
def __convert_to_target_value_list(target_factor, source_value_list): if isinstance(source_value_list, list): target_value_list = [] for source_value in source_value_list: target_value_list.append(convert_factor_type(source_value, target_factor.type)) return target_value_list else: return convert_factor_type(source_value_list, target_factor.type)
def get_source_value_list(pipeline_topic, raw_data, parameter: Parameter, target_factor: Factor = None, context=None): if parameter.kind == parameter_constants.TOPIC: source_factor: Factor = get_factor(parameter.factorId, pipeline_topic) return get_source_factor_value(raw_data, source_factor) elif parameter.kind == parameter_constants.CONSTANT: if parameter.value is None: return None elif parameter.value == '': return '' elif not parameter.value: return None else: variable_type, context_target_name = process_variable( parameter.value) if variable_type == MEMORY: # print("context_target_name",context_target_name) # print(context) if context_target_name in context: result = context[context_target_name] if result is None: return __check_default_value(target_factor) else: return result else: return __check_default_value(target_factor) elif variable_type == SNOWFLAKE: return context_target_name else: if target_factor is not None: if SPLIT_FLAG in parameter.value: value_list = __split_value(parameter.value) result = [] for value in value_list: result.append( convert_factor_type(value, target_factor.type)) return result else: return convert_factor_type(parameter.value, target_factor.type) else: return parameter.value # else: # elif parameter.kind == parameter_constants.COMPUTED: # print(target_factor.name) return __process_compute_kind(parameter, raw_data, pipeline_topic, target_factor, context) else: raise Exception("Unknown source kind {0}".format(parameter.kind))
def read_factor(instance, context): raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD] unit_action_status = ReadFactorAction(type=action.type) start = time.time() # print("context",context) variable_type, context_target_name = process_variable(action.variableName) topic = get_topic_by_id(action.topicId) factor = get_factor(action.factorId, topic) joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, topic, context) mongo_query = __build_mongo_query(joint_type, where_condition) target_data = query_topic_data(mongo_query, topic.name) if target_data is not None: if factor.name in target_data: read_value = target_data[factor.name] if factor.name in context: log.warn("factor name {0} is already in context".format(factor.name)) context[context_target_name] = target_data[factor.name] unit_action_status.value = read_value else: context[context_target_name] = convert_factor_type(factor.defaultValue, factor.type) log.warn("target_data is empty ,conditions {0}".format(mongo_query)) elapsed_time = time.time() - start unit_action_status.complete_time = elapsed_time # print("read context",context) return context, unit_action_status, []
def get_source_value_list(pipeline_topic, raw_data, parameter: Parameter, target_factor=None, context=None): if parameter.kind == parameter_constants.TOPIC: source_factor: Factor = get_factor(parameter.factorId, pipeline_topic) return get_source_factor_value(raw_data, source_factor) elif parameter.kind == parameter_constants.CONSTANT: if parameter.value is None or not parameter.value: return None else: variable_type, context_target_name = process_variable( parameter.value) if variable_type == "memory": return context[context_target_name] else: if target_factor is not None: if SPLIT_FLAG in parameter.value: value_list = __split_value(parameter.value) result = [] for value in value_list: result.append( convert_factor_type(value, target_factor.type)) return result else: return convert_factor_type(parameter.value, target_factor.type) else: return parameter.value # else: # elif parameter.kind == parameter_constants.COMPUTED: return __process_compute_kind(parameter, raw_data, pipeline_topic, target_factor) else: raise Exception("Unknown source kind {0}".format(parameter.kind))
def get_source_value_list(pipeline_topic, raw_data, parameter: Parameter, target_factor=None): if parameter.kind == parameter_constants.TOPIC: source_factor: Factor = get_factor(parameter.factorId, pipeline_topic) return get_source_factor_value(raw_data, source_factor) elif parameter.kind == parameter_constants.CONSTANT: if parameter.value is None or not parameter.value: return None else: # print("target_factor",target_factor.type) if target_factor is not None: return convert_factor_type(parameter.value, target_factor.type) else: return parameter.value elif parameter.kind == parameter_constants.COMPUTED: return __process_compute_kind(parameter, raw_data, pipeline_topic, target_factor) else: raise Exception("Unknown source kind {0}".format(parameter.kind))
def get_factor_value(index, factor_list, raw_data, result): # results=[] factor = factor_list[index] data = get_value(factor, raw_data) if type(data) is list: for raw in data: get_factor_value(index + 1, factor_list, raw, result) elif type(data) is dict: get_factor_value(index + 1, factor_list, data, result) else: if data is None and factor.defaultValue is not None: result.append(convert_factor_type(factor.defaultValue, factor.type)) else: result.append(data) return result
def __check_default_value(target_factor): if target_factor is not None and target_factor.defaultValue is not None: return convert_factor_type(target_factor.defaultValue, target_factor.type) else: return None