def train_model_parallel(args, data_df, data_df_log, sc):
""" 对需要分组的数据进行并行处理
note: 并行有很多问题，先暂时串行
"""
distinct_group_name = {i for i in data_df[args.group_field_name].array}
# parallel_num = len(distinct_group_name) if len(distinct_group_name) < 10 else 10
# p = Pool(parallel_num)
# for group_name in distinct_group_name:
# group_data_df = data_df[data_df[args.group_field_name] == group_name].drop(args.group_field_name, axis=1)
# group_data_df_log = data_df_log[data_df_log[args.group_field_name] == group_name].drop(args.group_field_name, axis=1)
# p.apply_async(train_model, args=(args, group_data_df, group_data_df_log, sc, "-" + group_name))
#
# p.close()
# p.join()
l = []
for group_name in distinct_group_name:
group_data_df = data_df[data_df[args.group_field_name] ==
group_name].drop(args.group_field_name, axis=1)
group_data_df_log = data_df_log[data_df_log[args.group_field_name] ==
group_name].drop(args.group_field_name,
axis=1)
# 对数据做校验
check_data_num(sc, group_data_df, args)
l.append((group_name, group_data_df, group_data_df_log))
for group_name, group_data_df, group_data_df_log in l:
train_model(args, group_data_df, group_data_df_log, sc,
str(group_name) + "-")
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep +
constants.EVALUATION_MODEL_NAME,
str(res_trend),
is_text_file=True)
def check_data_num(sc, data, args):
""" 对数据进行异常校验 """
# 计算预测秒数
predict_seconds = 0
if "d" == args.freq.lower():
predict_seconds = args.periods * 24 * 60 * 60
elif "y" == args.freq.lower():
predict_seconds = args.periods * 24 * 60 * 60 * 365
elif "m" == args.freq.lower():
predict_seconds = args.periods * 24 * 60 * 60 * 30
elif "min" == args.freq.lower():
predict_seconds = args.periods * 60
elif "h" == args.freq.lower():
predict_seconds = args.periods * 60 * 60
else:
predict_seconds = args.periods * 24 * 60 * 60
if (data["ds"].max().timestamp() - data["ds"].min().timestamp()
) < args.base_multiple * predict_seconds:
print("======> 数据异常，预测数据过少 ")
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep +
constants.ERROR_LOG,
"数据异常-预测数据过少",
is_text_file=True)
exit(0)
def gen_pmml_to_hdfs(sc, model, args):
""" 生成 pmml,pkl 到 hdfs """
# 先将pmml文件生成到driver的tmp目录下，然后再将其上传至HDFS
# 目录名称 /tmp/时间戳/文件名
print("===========> 保存模型文件到 HDFS")
pmml_model_name = constants.PMML_NAME
pkl_model_name = constants.PKL_NAME
dir_name = "/tmp/" + str(time.time())
args.tmp_dir = dir_name
os.mkdir(dir_name)
# 保存为pmml文件
pipeline = PMMLPipeline([("classifier", model)])
sklearn2pmml(pipeline, dir_name + os.sep + pmml_model_name, with_repr=True)
joblib.dump(model, dir_name + os.sep + pkl_model_name)
# 上传文件至HDFS
with open(dir_name + os.sep + pmml_model_name, "r") as f1, \
open(dir_name + os.sep + pkl_model_name, "rb") as f2:
data1 = f1.read()
data2 = f2.read()
save_data.write_data_to_cluster(sc, args.export_dir + os.sep + pmml_model_name, data1)
save_data.write_data_to_cluster(sc, args.model_dir + os.sep + pkl_model_name, data2, is_text_file=False)
# 删除临时文件
os.remove(dir_name + os.sep + pmml_model_name)
os.remove(dir_name + os.sep + pkl_model_name)
def train_model_by_all_data(args, data_df, sc, suffix=""):
""" 使用全量数据训练模型并保存结果到hdfs """
m = get_prophet_model(args)
m.fit(data_df)
future = m.make_future_dataframe(periods=args.periods, freq=args.freq)
future_tmp = m.make_future_dataframe(periods=args.periods,
freq=args.freq,
include_history=False)
forecast = m.predict(future)
forecast_tmp = m.predict(future_tmp)
# res_fig = m.plot(forecast)
# component_fig = m.plot_components(forecast)
splitFlag = forecast["ds"].count() - data_df["ds"].count()
res_fig = drawing_pic.plot(m, forecast, splitFlag)
component_fig = drawing_pic.plot_components(m, forecast)
dir_name = args.tmp_dir
# 预测结果
res_path = dir_name + os.sep + "000000"
res_df = pd.DataFrame(forecast, columns=["ds", "yhat"])
res_df_tmp = pd.DataFrame(forecast_tmp, columns=["ds", "yhat"])
if suffix != "":
res_df_tmp[args.group_field_name] = suffix[:-1]
res_df[args.group_field_name] = suffix[:-1]
tail_sum = res_df[-args.periods:]["yhat"].sum()
pre_sum = res_df[-args.periods * 2:-args.periods]["yhat"].sum()
global res_trend
res_trend[suffix[:-1]] = {
"difference": abs(tail_sum - pre_sum),
"amplitude": abs(tail_sum - pre_sum) / pre_sum
}
# res_df.to_csv(res_path, index=False, mode="a", header=False)
res_df_tmp.to_csv(res_path, index=False, mode="a", header=False)
# 预测结果图
res_pic_path = dir_name + os.sep + suffix + constants.PREDICT_PIC
# 成分分析图
component_pic_path = dir_name + os.sep + suffix + constants.COMPONENT_PIC
standard_fig(res_fig)
standard_fig_component(component_fig)
res_fig.savefig(res_pic_path)
component_fig.savefig(component_pic_path)
with open(res_pic_path, "rb") as f1, open(component_pic_path, "rb") as f2:
data1 = f1.read()
data2 = f2.read()
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep + suffix +
constants.PREDICT_PIC,
data1,
is_text_file=False)
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep + suffix +
constants.COMPONENT_PIC,
data2,
is_text_file=False)
os.remove(res_pic_path)
os.remove(component_pic_path)
def run():
args = parse_args()
sc, spark, num_executors = init_spark_session(args)
# 拿到训练数据
dataDF, label_type, features_name = get_train_data(spark, args,
num_executors)
trainDF, testDF = dataDF.randomSplit(
[1 - float(args.sample_ratio),
float(args.sample_ratio)],
seed=args.seed)
trainDF.cache()
testDF.cache() # 后面用于预测
import math
trainDFCount = trainDF.count()
args.epochs = math.ceil(args.steps * args.batch_size / trainDFCount)
print("迭代步数：" + str(args.steps) + "=======" + " 迭代轮次：" + str(args.epochs))
# 将路径转换为 hdfs 路径
args.export_dir = common_utils.get_abs_path(sc, args.export_dir)
args.model_dir = common_utils.get_abs_path(sc, args.model_dir)
# 训练
print("{0} ===== Estimator.fit()".format(datetime.now().isoformat()))
estimator = get_tf_estimator(args)
model = estimator.fit(trainDF)
trainDF.unpersist()
# 推理 [y_,y_pre]
set_model_param(model, args)
print("{0} ===== Model.transform()".format(datetime.now().isoformat()))
test_preds_tmp = model.transform(testDF)
testDF.unpersist()
test_preds_tmp = save_res_to_hdfs(test_preds_tmp, args, label_type,
features_name, spark)
# 将预测结果和label拿出来，评估模型
test_preds = test_preds_tmp.map(lambda row: [row[1], row[2]]).cache()
test_preds_tmp.unpersist()
# 模型评估
final_result = evaluate_model(test_preds, label_type)
save_data.write_data_to_cluster(
sc,
args.evaluate + constants.PATH_SEP + constants.EVALUATION_MODEL_NAME,
str(final_result))
print("{0} ===== Stop".format(datetime.now().isoformat()))
def save_res_to_hdfs(sc, args):
# 将预测结果写入hdfs
res_path = args.tmp_dir + os.sep + "000000"
if args.group_field_name != "null":
format_res(res_path, args.tmp_dir + os.sep + "000001")
res_path = args.tmp_dir + os.sep + "000001"
with open(res_path, "r") as f:
data = f.read()
save_data.write_data_to_cluster(sc,
args.output + os.sep + "000000",
data,
is_text_file=True)
os.remove(res_path)
def cross_validation(args, train_data_df, test_data_df, sc):
""" 交叉验证求 mape """
m = get_prophet_model(args)
m.fit(train_data_df)
res = pd.merge(pd.DataFrame(m.predict(test_data_df),
columns=["ds", "yhat"]),
test_data_df,
how="inner",
on="ds")
mape = ((res["y"] - res["yhat"]) /
res["y"]).apply(abs).sum() / test_data_df.count().array[0]
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep +
constants.EVALUATION_MODEL_NAME,
str({"mape": str(mape)}),
is_text_file=True)
def get_standard_data(sc, data_df, args):
""" 对数据进行处理 """
schema_list = data_df.columns.to_list()
# assert len(schema_list) == 2, "目前只支持两列输入"
# 将列名修改为 ds,y
try:
schema_list.remove(args.label_name)
if args.no_validation:
schema_list.remove(args.group_field_name)
if len(schema_list) > 1:
data_df = data_df.drop(schema_list[0], axis=1)
schema_list.remove(schema_list[0])
print("=======> 删除序号以及分组列之后的表头信息 " + str(schema_list))
data_df = data_df.rename(columns={
args.label_name: "y",
schema_list[0]: "ds"
})
data_df["ds"] = pd.to_datetime(data_df["ds"])
except:
print("======> 数据格式异常 ")
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep +
constants.ERROR_LOG,
"数据格式异常",
is_text_file=True)
exit(0)
# 对数处理
data_df_log = copy.deepcopy(data_df)
data_df_log["y"] = data_df_log["y"].apply(math.log)
if not args.no_validation:
# 交叉验证数据
# data_df["ds"] = pd.to_datetime(data_df["ds"])
count = data_df.count().array[0]
data_df = data_df.sort_values(by="ds")
split_flag = math.floor(count * (1 - args.sample_ratio))
train_data_df = data_df.iloc[:split_flag]
test_data_df = data_df.iloc[split_flag:]
return data_df, data_df_log, train_data_df, test_data_df
else:
return data_df, data_df_log, None, None
def train_model_by_log_data(args, data_df_log, sc, suffix=""):
""" 使用log处理的y进行重新训练 """
m = get_prophet_model(args)
m.fit(data_df_log)
future = m.make_future_dataframe(periods=args.periods, freq=args.freq)
forecast = m.predict(future)
png_path = args.tmp_dir + os.sep + suffix + constants.PREDICT_LOG_PIC
# res_fig = m.plot(forecast)
splitFlag = forecast["ds"].count() - data_df_log["ds"].count()
res_fig = drawing_pic.plot(m, forecast, splitFlag)
standard_fig(res_fig)
res_fig.savefig(png_path)
with open(png_path, "rb") as f:
data = f.read()
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep + suffix +
constants.PREDICT_LOG_PIC,
data,
is_text_file=False)
os.remove(png_path)
def check_data_null(sc, data, args):
flag = False
try:
count = data[args.label_name].count()
null_ratio_group_field = sum(pd.isna(
data[args.group_field_name])) / count
null_ratio_label = sum(pd.isna(data[args.label_name])) / count
schema_list = data.columns.to_list()
schema_list.remove(args.label_name)
schema_list.remove(args.group_field_name)
null_ratio_date = 0
if len(schema_list) > 1:
null_ratio_date = sum(pd.isna(data[schema_list[1]])) / count
elif len(schema_list) == 1:
null_ratio_date = sum(pd.isna(data[schema_list[0]])) / count
print("========> cellid空值占比 " + str(null_ratio_group_field))
print("========> x空值占比 " + str(null_ratio_date))
print("========> y空值占比 " + str(null_ratio_label))
if null_ratio_group_field > 0.05 or null_ratio_label > 0.05 or null_ratio_date > 0.05:
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep +
constants.ERROR_LOG,
"数据异常-预测数据空值过多",
is_text_file=True)
flag = True
exit(0)
except:
if not flag:
print("======> 数据格式异常 ")
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep +
constants.ERROR_LOG,
"数据格式异常",
is_text_file=True)
exit(0)
def evaluate_model(y, y_, args, class_s, sc, model, x):
""" 评估模型 """
print("=======> 开始进行模型评估")
# 保存报告文件到HDFS
report = classification_report(y, y_, labels=class_s)
report_dict = classification_report(y, y_, labels=class_s, output_dict=True)
res_dict = {}
accuracy_score(y, y_)
res_dict["total_accuracy"] = accuracy_score(y, y_)
# 存放类别字典
class_list = []
res_dict["other_indicators"] = class_list
for i in [str(c) for c in class_s]:
d = {"recall": report_dict[i]["recall"],
"precise": report_dict[i]["precision"],
"f1-score": report_dict[i]["f1-score"],
"type": i
}
class_list.append(d)
res_dict["n_class"] = len(class_s)
save_data.write_data_to_cluster(sc, args.evaluate + os.sep + constants.CLASSIFICATION_REPORT, report)
# 保存报告图片
report_path = args.tmp_dir + os.sep + constants.CLASSIFICATION_REPORT + ".png"
drawing_pic.save_text_to_pic(report, report_path)
with open(report_path, "rb") as f:
data = f.read()
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep + constants.CLASSIFICATION_REPORT + ".png",
data,
is_text_file=False)
os.remove(report_path)
# 保存混淆矩阵信息到HDFS
matrix = confusion_matrix(y, y_, labels=class_s).tolist()
res = ""
for index, line in enumerate(matrix):
res += str(class_s[index]) + "," + ",".join([str(l) for l in line]) + "\n"
save_data.write_data_to_cluster(sc, args.evaluate + os.sep + constants.CONFUSION_MATRIX, res)
# 保存混淆矩阵图片到HDFS
matrix_path = args.tmp_dir + os.sep + constants.CONFUSION_MATRIX + ".png"
drawing_pic.save_confusion_matrix(matrix, class_s, matrix_path)
with open(matrix_path, "rb") as f:
data = f.read()
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep + constants.CONFUSION_MATRIX + ".png",
data,
is_text_file=False)
os.remove(matrix_path)
if len(class_s) == 2 :
# 如果是二分类，则需要生成 ROC曲线
# 计算出预测概率
y_proba = model.predict_proba(x)
# 取第一个类为 正样例
fpr, tpr, thresholds = roc_curve(y, y_proba[:, 0], pos_label=class_s[0])
roc_auc = auc(fpr, tpr)
roc_pic_path = args.tmp_dir + os.sep + constants.ROC_PIC
drawing_pic.save_roc_to_pic(fpr, tpr, roc_auc, roc_pic_path)
with open(roc_pic_path, "rb") as f:
data = f.read()
save_data.write_data_to_cluster(sc,
args.evaluate + os.sep + constants.ROC_PIC,
data,
is_text_file=False)
os.remove(roc_pic_path)
res_dict["auc"] = roc_auc
# 删除临时目录
os.rmdir(args.tmp_dir)
# 将评估结果写入HDFS
save_data.write_data_to_cluster(sc, args.evaluate + os.sep + constants.EVALUATION_MODEL_NAME, str(res_dict))