def second_evaluation_core(spark_session, condition, operator_id):
"""
二分类评估核心函数
:param spark_session:
:param condition:
:param operator_id:
:return:
"""
# 读模型
# 当前节点（评估节点）一个父节点
operator = OperatorDao.get_operator_by_id(operator_id)
# 父节点(预测节点) 两个父节点
father_id = operator.father_operator_ids
father_operator = OperatorDao.get_operator_by_id(father_id)
# 祖节点（模型节点和读预测数据节点）
grand_father_ids = father_operator.father_operator_ids.split(',')
print("**********祖节点（模型节点和读预测数据源节点）:", grand_father_ids)
# 读数据
def get_predict_data(operator_config_):
for grand_father_file_ in operator_config_:
grand_father_id_ = list(grand_father_file_.keys())[0]
grand_father_ = OperatorDao.get_operator_by_id(grand_father_id_)
if grand_father_.operator_type_id == 5001 or grand_father_.operator_type_id < 3000:
print("***************评估函数，预测数据：",
grand_father_.operator_type_id)
pre_data_file_url = grand_father_.operator_output_url.split(
'*,')[grand_father_file_[grand_father_id_]]
print("***************评估函数，预测数据url：", pre_data_file_url)
return read_data(spark_session, pre_data_file_url)
print("**********预测节点:", father_operator.operator_config)
df = get_predict_data(
json.loads(father_operator.operator_config)['fileUrl'])
# 评估
for grand_father_id in grand_father_ids:
grand_father = OperatorDao.get_operator_by_id(grand_father_id)
grand_father_operator_type = grand_father.operator_type_id
# 模型加载节点
if grand_father_operator_type == 8000:
grand_father_operator_type = json.loads(
grand_father.operator_config)['parameter']['modelTypeId']
if grand_father_operator_type == 6001: # svm二分类节点
print("***************评估函数，训练模型", grand_father.operator_type_id)
evaluation_df = svm_second_evaluation(
spark_session, grand_father.operator_output_url, df,
json.loads(father_operator.operator_config)['parameter'],
condition)
return evaluation_df
elif grand_father_operator_type == 6003: # lr二分类节点
print("***************评估函数，训练模型", grand_father.operator_type_id)
evaluation_df = lr_second_evaluation(
spark_session, grand_father.operator_output_url, df,
json.loads(father_operator.operator_config)['parameter'],
condition)
return evaluation_df
def save_ml_model(operator_id, user_id, name):
"""
保存训练模型
:param operator_id:
:param user_id:
:param name:
:return:
"""
# 查看算子
operator = OperatorDao.get_operator_by_id(operator_id)
if operator.operator_type_id > 7000 or operator.operator_type_id < 6001:
return "所选择的节点并不是模型算子节点"
if operator.status != "success":
return "请执行该节点"
if operator.operator_output_url is not None:
operator_output_url = operator.operator_output_url.split('*,')
else:
return "没有运行结果"
model_url = operator_output_url[0]
operator_type_id = operator.operator_type_id
model_id = operator.model_id
# 查看执行流程model
model = ModelDao.get_model_by_id(model_id)
project_id = model.project_id
ml_model = MLModel(user_id=user_id,
project_id=project_id,
model_id=model_id,
status='save',
name=name,
operator_type_id=operator_type_id,
model_url=model_url)
return MLModelDao.create_ml_model(ml_model)
def get_predict_data(operator_config_):
for grand_father_file_ in operator_config_:
grand_father_id_ = list(grand_father_file_.keys())[0]
grand_father_ = OperatorDao.get_operator_by_id(grand_father_id_)
if grand_father_.operator_type_id == 5001 or grand_father_.operator_type_id < 3000:
print("***************评估函数，预测数据：",
grand_father_.operator_type_id)
pre_data_file_url = grand_father_.operator_output_url.split(
'*,')[grand_father_file_[grand_father_id_]]
print("***************评估函数，预测数据url：", pre_data_file_url)
return read_data(spark_session, pre_data_file_url)
def ml_predict_core(spark_session, operator_id, df, model_url, condition):
"""
路由控制加载哪种模型进行预测
:param spark_session:
:param operator_id:
:param df:
:param model_url:
:param condition:
:return: 预测结果 sparkframe
"""
# 父节点是什么组件
operator = OperatorDao.get_operator_by_id(operator_id)
father_ids = operator.father_operator_ids.split(',')
print("**********", operator.father_operator_ids)
for father_id in father_ids:
father = OperatorDao.get_operator_by_id(father_id)
print("***************", father.operator_type_id)
print("---------------", father.operator_type_id == 6001)
operator_type_flag = father.operator_type_id
# 模型加载节点
if operator_type_flag == 8000:
operator_type_flag = json.loads(
father.operator_config)['parameter']['modelTypeId']
if operator_type_flag == 6001: # svm二分类
prediction_df = svm_second_predict(spark_session, model_url, df,
condition)
elif operator_type_flag == 6002: # gbdt二分类
prediction_df = gbdt_second_predict(model_url, df, condition)
elif operator_type_flag == 6003: # lr二分类
prediction_df = lr_second_predict(model_url, df, condition)
elif operator_type_flag == 6004: # lr多分类
prediction_df = lr_multiple_predict(model_url, df, condition)
elif operator_type_flag == 6005: # mpc多分类
prediction_df = mpc_multiple_predict(model_url, df, condition)
# 根据父组件的类型决定加载哪种模型
return prediction_df
def get_operate_result_data():
"""
查看算子运行结果数据
:return:
"""
operator_id = request.form.get('operatorId')
start = int(request.form.get('start'))
end = int(request.form.get('end'))
print(operator_id, start, end)
operator = OperatorDao.get_operator_by_id(operator_id)
if operator.status != "success":
return "请执行该节点"
if operator.operator_output_url is not None:
operator_output_url = operator.operator_output_url.split('*,')
else:
return "没有运行结果"
result_arr = []
try:
for i in range(len(operator_output_url)):
data = pd.read_csv(operator_output_url[i], encoding='utf-8')
if len(data) < end:
end = len(data)
if start > end:
result_arr.append({
'length': len(data),
'data': "请输入合法参数",
'position': i
})
else:
data2 = data[int(start):int(end)].to_json(orient='records',
force_ascii=False)
result_arr.append({
'length': len(data),
'data': json.loads(data2),
'position': i
})
return jsonify(result_arr)
except:
traceback.print_exc()
return "Error，please contact the administrator "
def initial_execute_status(execute_user_id, start_nodes):
"""
每次执行model时，初始化执行状态
:param execute_user_id:
:param start_nodes: []
:return:
"""
# 查找参与运行的 operator
operator_list = []
operator_id_queue = []
for x in start_nodes:
operator_id_queue.append(x)
while len(operator_id_queue) > 0:
operator_id = operator_id_queue.pop(0)
if operator_id is None or operator_id == "":
continue
operator = OperatorDao.get_operator_by_id(operator_id)
operator_list.append(operator)
for x in operator.child_operator_ids.split(','):
operator_id_queue.append(x)
# 每个operator 状态初始化为initial
for operator in operator_list:
OperatorDao.update_operator_by_id(operator.id, "initial")
# 追加执行记录
model_execute = ModelExecute(start_nodes=','.join(start_nodes),
status='initial',
execute_user_id=execute_user_id,
create_time=time.strftime(
"%Y-%m-%d %H:%M:%S", time.localtime()))
model_execute = ModelExecuteDao.create_model_execute(model_execute)
if model_execute is False:
return False
else:
return model_execute.id
def operator_execute(spark_session, operator_id):
"""
执行算子
:param spark_session:
:param operator_id:
:return:
"""
try:
# 查算子
operator = OperatorDao.get_operator_by_id(operator_id)
print("------执行算子------", "operator_id：", operator_id,
operator.operator_type_id)
# 获取input_url
config = json.loads(operator.operator_config)
file_url_list = config['fileUrl']
# 获取输入地址
url_arr = []
for file_url_dict in file_url_list:
key = ''
for ikey in file_url_dict.keys():
key = ikey
if operator_id == key:
url_arr.append(file_url_dict[key])
else:
father = OperatorDao.get_operator_by_id(key)
# 检查父节点是否准备就绪
if father.status != 'success':
return []
# TODO:暂定从0 开始
father_output_url_index = file_url_dict[key]
father_url_arr = father.operator_output_url.split('*,')
url_arr.append(father_url_arr[father_output_url_index])
# 算子函数
if operator.operator_type_id == 1001:
preprocessService.filter_multi_conditions(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 1002:
preprocessService.sort(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 1003:
preprocessService.column_split(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 1005:
preprocessService.columns_merge(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 1006:
preprocessService.replace(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 1007:
preprocessService.fill_null_value(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 1008:
preprocessService.column_map(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 1009:
preprocessService.random_split(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 2001:
FEService.quantile_discretization(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 2002:
FEService.vector_indexer(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 2003:
FEService.standard_scaler(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 2004:
FEService.pca(spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 2005:
FEService.string_indexer(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 2006:
FEService.one_hot_encoder(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 2007:
FEService.polynomial_expansion(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 2008:
FEService.chiSqSelector(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 3001:
ExplorationService.full_table_statistics(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 3002:
ExplorationService.frequency_statistics(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 3003:
ExplorationService.correlation_coefficient(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 5001:
preprocessService.read_data_with_update_record(
spark_session, operator_id, url_arr[0])
elif operator.operator_type_id == 6000:
PredictService.ml_predict(
spark_session, operator_id, url_arr,
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 6001:
SecondClassification.svm(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 6002:
SecondClassification.gbdt(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 6003:
SecondClassification.lr(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 6004:
MultipleClassifition.lr(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 6005:
MultipleClassifition.mpc(
spark_session, operator_id, url_arr[0],
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 7001:
Evaluation.second_evaluation(
spark_session, operator_id,
json.loads(operator.operator_config)['parameter'])
elif operator.operator_type_id == 8000:
ModelService.model_operator(
operator_id,
json.loads(operator.operator_config)['parameter'])
return operator.child_operator_ids.split(',')
except:
traceback.print_exc()
return False