def __process_parameter_constants(parameter: Parameter,
context,
target_factor=None):
variable_type, context_target_name = process_variable(parameter.value)
if variable_type == parameter_constants.CONSTANT:
if target_factor is not None:
if SPLIT_FLAG in parameter.value:
value_list = __split_value(parameter.value)
result = []
for value in value_list:
result.append(
convert_factor_type(value, target_factor.type))
return result
else:
return convert_factor_type(parameter.value, target_factor.type)
else:
return parameter.value
elif variable_type == parameter_constants.MEMORY:
if context_target_name in context:
return context[context_target_name]
else:
raise ValueError(
"no variable {0} in context".format(context_target_name))
else:
raise ValueError("variable_type is invalid")
def __convert_to_target_value_list(target_factor, source_value_list):
if isinstance(source_value_list, list):
target_value_list = []
for source_value in source_value_list:
target_value_list.append(convert_factor_type(source_value, target_factor.type))
return target_value_list
else:
return convert_factor_type(source_value_list, target_factor.type)
def get_source_value_list(pipeline_topic,
raw_data,
parameter: Parameter,
target_factor: Factor = None,
context=None):
if parameter.kind == parameter_constants.TOPIC:
source_factor: Factor = get_factor(parameter.factorId, pipeline_topic)
return get_source_factor_value(raw_data, source_factor)
elif parameter.kind == parameter_constants.CONSTANT:
if parameter.value is None:
return None
elif parameter.value == '':
return ''
elif not parameter.value:
return None
else:
variable_type, context_target_name = process_variable(
parameter.value)
if variable_type == MEMORY:
# print("context_target_name",context_target_name)
# print(context)
if context_target_name in context:
result = context[context_target_name]
if result is None:
return __check_default_value(target_factor)
else:
return result
else:
return __check_default_value(target_factor)
elif variable_type == SNOWFLAKE:
return context_target_name
else:
if target_factor is not None:
if SPLIT_FLAG in parameter.value:
value_list = __split_value(parameter.value)
result = []
for value in value_list:
result.append(
convert_factor_type(value, target_factor.type))
return result
else:
return convert_factor_type(parameter.value,
target_factor.type)
else:
return parameter.value
# else:
#
elif parameter.kind == parameter_constants.COMPUTED:
# print(target_factor.name)
return __process_compute_kind(parameter, raw_data, pipeline_topic,
target_factor, context)
else:
raise Exception("Unknown source kind {0}".format(parameter.kind))
def read_factor(instance, context):
raw_data, old_value = instance[pipeline_constants.NEW], instance[pipeline_constants.OLD]
unit_action_status = ReadFactorAction(type=action.type)
start = time.time()
# print("context",context)
variable_type, context_target_name = process_variable(action.variableName)
topic = get_topic_by_id(action.topicId)
factor = get_factor(action.factorId, topic)
joint_type, where_condition = build_query_conditions(action.by, pipeline_topic, raw_data, topic, context)
mongo_query = __build_mongo_query(joint_type, where_condition)
target_data = query_topic_data(mongo_query, topic.name)
if target_data is not None:
if factor.name in target_data:
read_value = target_data[factor.name]
if factor.name in context:
log.warn("factor name {0} is already in context".format(factor.name))
context[context_target_name] = target_data[factor.name]
unit_action_status.value = read_value
else:
context[context_target_name] = convert_factor_type(factor.defaultValue, factor.type)
log.warn("target_data is empty ,conditions {0}".format(mongo_query))
elapsed_time = time.time() - start
unit_action_status.complete_time = elapsed_time
# print("read context",context)
return context, unit_action_status, []
def get_source_value_list(pipeline_topic,
raw_data,
parameter: Parameter,
target_factor=None,
context=None):
if parameter.kind == parameter_constants.TOPIC:
source_factor: Factor = get_factor(parameter.factorId, pipeline_topic)
return get_source_factor_value(raw_data, source_factor)
elif parameter.kind == parameter_constants.CONSTANT:
if parameter.value is None or not parameter.value:
return None
else:
variable_type, context_target_name = process_variable(
parameter.value)
if variable_type == "memory":
return context[context_target_name]
else:
if target_factor is not None:
if SPLIT_FLAG in parameter.value:
value_list = __split_value(parameter.value)
result = []
for value in value_list:
result.append(
convert_factor_type(value, target_factor.type))
return result
else:
return convert_factor_type(parameter.value,
target_factor.type)
else:
return parameter.value
# else:
#
elif parameter.kind == parameter_constants.COMPUTED:
return __process_compute_kind(parameter, raw_data, pipeline_topic,
target_factor)
else:
raise Exception("Unknown source kind {0}".format(parameter.kind))
def get_source_value_list(pipeline_topic, raw_data, parameter: Parameter, target_factor=None):
if parameter.kind == parameter_constants.TOPIC:
source_factor: Factor = get_factor(parameter.factorId, pipeline_topic)
return get_source_factor_value(raw_data, source_factor)
elif parameter.kind == parameter_constants.CONSTANT:
if parameter.value is None or not parameter.value:
return None
else:
# print("target_factor",target_factor.type)
if target_factor is not None:
return convert_factor_type(parameter.value, target_factor.type)
else:
return parameter.value
elif parameter.kind == parameter_constants.COMPUTED:
return __process_compute_kind(parameter, raw_data, pipeline_topic, target_factor)
else:
raise Exception("Unknown source kind {0}".format(parameter.kind))
def get_factor_value(index, factor_list, raw_data, result):
# results=[]
factor = factor_list[index]
data = get_value(factor, raw_data)
if type(data) is list:
for raw in data:
get_factor_value(index + 1, factor_list, raw, result)
elif type(data) is dict:
get_factor_value(index + 1, factor_list, data, result)
else:
if data is None and factor.defaultValue is not None:
result.append(convert_factor_type(factor.defaultValue,
factor.type))
else:
result.append(data)
return result
def __check_default_value(target_factor):
if target_factor is not None and target_factor.defaultValue is not None:
return convert_factor_type(target_factor.defaultValue,
target_factor.type)
else:
return None
