def run(self):
"""
| 処理の最後
"""
print("---" + __class__.__name__ + ": run")
with self.output().open("w") as target:
exp_data = pd.read_pickle(self.intermediate_folder +
mu.convert_date_to_str(self.end_date) +
'_exp_data.pkl')
# 予測を実施
pred_df = self.skproc.proc_predict_sk_model(exp_data)
print("End_baoz_predict run: pred_df", pred_df.shape)
import_df = self.skproc.create_import_data(pred_df)
if self.export_mode:
print("export data")
import_df.to_pickle(self.intermediate_folder +
self.skproc.version_str + '/' +
'export_data.pkl')
if self.skproc.version_str == "win":
analyze_df = self.skproc.eval_pred_data(import_df)
print(analyze_df)
self.skproc.import_data(import_df)
Output.post_slack_text(dt.now().strftime("%Y/%m/%d %H:%M:%S") +
" finish predict job:" +
self.skproc.version_str)
print(__class__.__name__ +
" says: task finished".format(task=self.__class__.__name__))
def test_11_proc_predict_sk_model(self):
""" 予測データの作成ができることを確認 """
import sys
import pandas as pd
import modules.util as mu
print("-- " + sys._getframe().f_code.co_name + " start --")
predict_file_name = self.intermediate_folder + mu.convert_date_to_str(
self.end_date) + '_exp_data.pkl'
exp_data = pd.read_pickle(predict_file_name)
print("test_11_proc_predict_sk_model: exp_data", exp_data.shape)
all_pred_df = pd.DataFrame()
class_list = self.skmodel.class_list
for cls_val in class_list:
val_list = self.skmodel.get_val_list(exp_data, cls_val)
for val in val_list:
# 対象の競馬場のデータを取得する
filter_df = self.skmodel.get_filter_df(exp_data, cls_val, val)
# 予測を実施
pred_df = self.skmodel.proc_predict_sk_model(
filter_df, cls_val, val)
all_pred_df = pd.concat([all_pred_df, pred_df])
break
import_df = self.skmodel.create_import_data(all_pred_df)
self.skmodel.eval_pred_data(import_df)
# not empty check
self.assertFalse(len(import_df.index) == 0)
# 必要な列があるかチェック
contain_columns_set = set(self.import_list)
contain_check = self.proc_test_contain_columns_check(
import_df, contain_columns_set)
self.assertTrue(contain_check)
def output(self):
"""
:return: MockのOutputを返す
"""
# return MockTarget("output")
return luigi.LocalTarget(format=luigi.format.Nop,
path=self.intermediate_folder +
mu.convert_date_to_str(self.end_date) + "_" +
__class__.__name__)
def run(self):
"""
| 処理の最後
"""
print("---" + __class__.__name__ + ": run")
with self.output().open("w") as target:
if os.path.exists(self.intermediate_folder +
mu.convert_date_to_str(self.end_date) +
'_exp_data.pkl'):
exp_data = pd.read_pickle(
self.intermediate_folder +
mu.convert_date_to_str(self.end_date) + '_exp_data.pkl')
# 予測を実施して予測結果ファイルを作成
all_df = self.skmodel.proc_predict_sk_model(exp_data)
if self.test_flag:
print("精度チェック")
self.skmodel.eval_pred_data(all_df)
else:
slack = OperationSlack()
slack.post_slack_text(
dt.now().strftime("%Y/%m/%d %H:%M:%S") +
" finish predict job:" + self.skmodel.version_str)
print(__class__.__name__ +
" says: task finished".format(task=self.__class__.__name__))
def run(self):
"""
渡されたexp_data_nameに基づいてSK_DATA_MODELから説明変数のデータを取得する処理を実施。pickelファイル形式でデータを保存
"""
print("----" + __class__.__name__ + ": run")
Output.post_slack_text(dt.now().strftime("%Y/%m/%d %H:%M:%S") +
" start predict job:" + self.skproc.version_str)
with self.output().open("w") as target:
print("------ モデル毎に予測データが違うので指定してデータ作成を実行")
predict_df = self.skproc.create_predict_data()
print("Sub_get_exp_data run: predict_df", predict_df.shape)
predict_df.to_pickle(self.intermediate_folder +
mu.convert_date_to_str(self.end_date) +
'_exp_data.pkl')
print(__class__.__name__ +
" says: task finished".format(task=self.__class__.__name__))
def run(self):
"""
渡されたexp_data_nameに基づいてSK_DATA_MODELから説明変数のデータを取得する処理を実施。pickelファイル形式でデータを保存
"""
print("----" + __class__.__name__ + ": run")
mu.create_folder(self.intermediate_folder)
if not self.test_flag:
slack = OperationSlack()
slack.post_slack_text(dt.now().strftime("%Y/%m/%d %H:%M:%S") +
" start predict job:" +
self.skmodel.version_str)
with self.output().open("w") as target:
predict_df = self.skmodel.create_predict_data()
if not predict_df.empty:
print("Sub_get_exp_data run: predict_df", predict_df.shape)
predict_df.to_pickle(self.intermediate_folder +
mu.convert_date_to_str(self.end_date) +
'_exp_data.pkl')
print(__class__.__name__ +
" says: task finished".format(task=self.__class__.__name__))
def run(self):
"""
| 処理の最後
"""
print("---" + __class__.__name__ + ": run")
with self.output().open("w") as target:
exp_data = pd.read_pickle(self.intermediate_folder +
mu.convert_date_to_str(self.end_date) +
'_exp_data.pkl')
print("------ 分類軸毎の学習モデルを作成")
all_pred_df = pd.DataFrame()
class_list = self.skmodel.class_list
for cls_val in class_list:
print("------ " + cls_val + "毎のデータを抽出して処理を実施")
val_list = self.skmodel.get_val_list(exp_data, cls_val)
for val in val_list:
# 対象の競馬場のデータを取得する
print("=============== cls_val:" + cls_val + " val:" +
val + " ===========================")
filter_df = self.skmodel.get_filter_df(
exp_data, cls_val, val)
# 予測を実施
pred_df = self.skmodel.proc_predict_sk_model(
filter_df, cls_val, val)
all_pred_df = pd.concat([all_pred_df, pred_df])
print("End_baoz_predict run: all_pred_df",
all_pred_df.shape)
import_df = self.skmodel.create_import_data(all_pred_df)
if self.export_mode:
print("export data")
import_df.to_pickle(self.intermediate_folder +
'export_data.pkl')
self.skmodel.eval_pred_data(import_df)
else:
self.skmodel.import_data(import_df)
slack = OperationSlack()
slack.post_slack_text(dt.now().strftime("%Y/%m/%d %H:%M:%S") +
" finish predict job:" +
self.skmodel.version_str)
print(__class__.__name__ +
" says: task finished".format(task=self.__class__.__name__))
def output(self):
return luigi.LocalTarget(format=luigi.format.Nop,
path=self.intermediate_folder + '/' +
mu.convert_date_to_str(self.end_date) + "_" +
__class__.__name__)
