def run(self): """ | 処理の最後 """ print("---" + __class__.__name__ + ": run") with self.output().open("w") as target: exp_data = pd.read_pickle(self.intermediate_folder + mu.convert_date_to_str(self.end_date) + '_exp_data.pkl') # 予測を実施 pred_df = self.skproc.proc_predict_sk_model(exp_data) print("End_baoz_predict run: pred_df", pred_df.shape) import_df = self.skproc.create_import_data(pred_df) if self.export_mode: print("export data") import_df.to_pickle(self.intermediate_folder + self.skproc.version_str + '/' + 'export_data.pkl') if self.skproc.version_str == "win": analyze_df = self.skproc.eval_pred_data(import_df) print(analyze_df) self.skproc.import_data(import_df) Output.post_slack_text(dt.now().strftime("%Y/%m/%d %H:%M:%S") + " finish predict job:" + self.skproc.version_str) print(__class__.__name__ + " says: task finished".format(task=self.__class__.__name__))
def test_11_proc_predict_sk_model(self): """ 予測データの作成ができることを確認 """ import sys import pandas as pd import modules.util as mu print("-- " + sys._getframe().f_code.co_name + " start --") predict_file_name = self.intermediate_folder + mu.convert_date_to_str( self.end_date) + '_exp_data.pkl' exp_data = pd.read_pickle(predict_file_name) print("test_11_proc_predict_sk_model: exp_data", exp_data.shape) all_pred_df = pd.DataFrame() class_list = self.skmodel.class_list for cls_val in class_list: val_list = self.skmodel.get_val_list(exp_data, cls_val) for val in val_list: # 対象の競馬場のデータを取得する filter_df = self.skmodel.get_filter_df(exp_data, cls_val, val) # 予測を実施 pred_df = self.skmodel.proc_predict_sk_model( filter_df, cls_val, val) all_pred_df = pd.concat([all_pred_df, pred_df]) break import_df = self.skmodel.create_import_data(all_pred_df) self.skmodel.eval_pred_data(import_df) # not empty check self.assertFalse(len(import_df.index) == 0) # 必要な列があるかチェック contain_columns_set = set(self.import_list) contain_check = self.proc_test_contain_columns_check( import_df, contain_columns_set) self.assertTrue(contain_check)
def output(self): """ :return: MockのOutputを返す """ # return MockTarget("output") return luigi.LocalTarget(format=luigi.format.Nop, path=self.intermediate_folder + mu.convert_date_to_str(self.end_date) + "_" + __class__.__name__)
def run(self): """ | 処理の最後 """ print("---" + __class__.__name__ + ": run") with self.output().open("w") as target: if os.path.exists(self.intermediate_folder + mu.convert_date_to_str(self.end_date) + '_exp_data.pkl'): exp_data = pd.read_pickle( self.intermediate_folder + mu.convert_date_to_str(self.end_date) + '_exp_data.pkl') # 予測を実施して予測結果ファイルを作成 all_df = self.skmodel.proc_predict_sk_model(exp_data) if self.test_flag: print("精度チェック") self.skmodel.eval_pred_data(all_df) else: slack = OperationSlack() slack.post_slack_text( dt.now().strftime("%Y/%m/%d %H:%M:%S") + " finish predict job:" + self.skmodel.version_str) print(__class__.__name__ + " says: task finished".format(task=self.__class__.__name__))
def run(self): """ 渡されたexp_data_nameに基づいてSK_DATA_MODELから説明変数のデータを取得する処理を実施。pickelファイル形式でデータを保存 """ print("----" + __class__.__name__ + ": run") Output.post_slack_text(dt.now().strftime("%Y/%m/%d %H:%M:%S") + " start predict job:" + self.skproc.version_str) with self.output().open("w") as target: print("------ モデル毎に予測データが違うので指定してデータ作成を実行") predict_df = self.skproc.create_predict_data() print("Sub_get_exp_data run: predict_df", predict_df.shape) predict_df.to_pickle(self.intermediate_folder + mu.convert_date_to_str(self.end_date) + '_exp_data.pkl') print(__class__.__name__ + " says: task finished".format(task=self.__class__.__name__))
def run(self): """ 渡されたexp_data_nameに基づいてSK_DATA_MODELから説明変数のデータを取得する処理を実施。pickelファイル形式でデータを保存 """ print("----" + __class__.__name__ + ": run") mu.create_folder(self.intermediate_folder) if not self.test_flag: slack = OperationSlack() slack.post_slack_text(dt.now().strftime("%Y/%m/%d %H:%M:%S") + " start predict job:" + self.skmodel.version_str) with self.output().open("w") as target: predict_df = self.skmodel.create_predict_data() if not predict_df.empty: print("Sub_get_exp_data run: predict_df", predict_df.shape) predict_df.to_pickle(self.intermediate_folder + mu.convert_date_to_str(self.end_date) + '_exp_data.pkl') print(__class__.__name__ + " says: task finished".format(task=self.__class__.__name__))
def run(self): """ | 処理の最後 """ print("---" + __class__.__name__ + ": run") with self.output().open("w") as target: exp_data = pd.read_pickle(self.intermediate_folder + mu.convert_date_to_str(self.end_date) + '_exp_data.pkl') print("------ 分類軸毎の学習モデルを作成") all_pred_df = pd.DataFrame() class_list = self.skmodel.class_list for cls_val in class_list: print("------ " + cls_val + "毎のデータを抽出して処理を実施") val_list = self.skmodel.get_val_list(exp_data, cls_val) for val in val_list: # 対象の競馬場のデータを取得する print("=============== cls_val:" + cls_val + " val:" + val + " ===========================") filter_df = self.skmodel.get_filter_df( exp_data, cls_val, val) # 予測を実施 pred_df = self.skmodel.proc_predict_sk_model( filter_df, cls_val, val) all_pred_df = pd.concat([all_pred_df, pred_df]) print("End_baoz_predict run: all_pred_df", all_pred_df.shape) import_df = self.skmodel.create_import_data(all_pred_df) if self.export_mode: print("export data") import_df.to_pickle(self.intermediate_folder + 'export_data.pkl') self.skmodel.eval_pred_data(import_df) else: self.skmodel.import_data(import_df) slack = OperationSlack() slack.post_slack_text(dt.now().strftime("%Y/%m/%d %H:%M:%S") + " finish predict job:" + self.skmodel.version_str) print(__class__.__name__ + " says: task finished".format(task=self.__class__.__name__))
def output(self): return luigi.LocalTarget(format=luigi.format.Nop, path=self.intermediate_folder + '/' + mu.convert_date_to_str(self.end_date) + "_" + __class__.__name__)