def join_data():
train_files = os.listdir(DataPath.raw.train_dir)
test_files = os.listdir(DataPath.raw.test_dir)
for files, is_train in zip([train_files, test_files], [True, False]):
dfs = []
for f in files:
if is_train:
path = f"{DataPath.raw.train_dir}/{f}"
else:
path = f"{DataPath.raw.test_dir}/{f}"
df_tmp = pd.read_csv(path)
stem = os.path.splitext(f)[0]
if int(stem) < 400:
df_tmp["is_screen_play"] = 1
else:
df_tmp["is_screen_play"] = 0
df_tmp["filename"] = stem
dfs.append(df_tmp)
df = pd.concat(dfs, axis=0, ignore_index=True)
df = df.sort_values(["filename", "frame"]).reset_index(drop=True)
if not is_train:
df = df.drop("is_screen_play", axis=1)
del dfs
gc.collect()
save_path = (f"{DataPath.interim.train}.jbl"
if is_train else f"{DataPath.interim.test}.jbl")
Jbl.save(df, save_path)
def save_model(self, model_path: str) -> None:
"""モデルの保存を行う
:param path: モデルの保存先パス
"""
model_path_dir = os.path.dirname(model_path)
mkdir(model_path_dir)
Jbl.save(self.model, model_path)
def preprocess(fe_cfg: Config):
fe_name = fe_cfg.basic.name
target_col = fe_cfg.column.target
train_path = f"{DataPath.interim.train}.jbl"
test_path = f"{DataPath.interim.test}.jbl"
for path, is_train in zip([train_path, test_path], [True, False]):
df = Jbl.load(path)
if "frame" in fe_cfg.__annotations__:
if "window" in fe_cfg.frame.__annotations__:
frame_column = fe_cfg.frame.column
frame_window = fe_cfg.frame.window
df = _filter_frame_window(df, frame_column, frame_window)
else:
frame_start_q = fe_cfg.frame.start
frame_end_q = fe_cfg.frame.end
df = _filter_frame(df, frame_start_q, frame_end_q)
df_processed = _build_features(df, is_train, fe_cfg)
if is_train:
X = df_processed.drop(target_col, axis=1)
y = df_processed[target_col]
else:
X = df_processed.copy()
y = None
X_save_path = (f"{DataPath.processed.X_train}_{fe_name}.jbl"
if is_train else
f"{DataPath.processed.X_test}_{fe_name}.jbl")
Jbl.save(X, X_save_path)
if is_train:
y_save_path = f"{DataPath.processed.y_train}_{fe_name}.jbl"
Jbl.save(y, y_save_path)
def run_predict_cv(self) -> None:
"""クロスバリデーションで学習した各foldのモデルの平均により、テストデータの予測を行う
あらかじめrun_train_cvを実行しておく必要がある
"""
logger.info(f"{self.run_name} - start prediction cv")
X_test = self.X_test
preds = []
show_feature_importance = "LGBM" in str(self.model_cls)
if show_feature_importance:
feature_importances = pd.DataFrame()
# 各foldのモデルで予測を行う
for i_fold in range(self.cv.n_splits):
logger.info(f"{self.run_name} - start prediction fold:{i_fold}")
model = self.build_model(i_fold)
model.load_model()
pred = model.predict(X_test)
preds.append(pred)
logger.info(f"{self.run_name} - end prediction fold:{i_fold}")
if show_feature_importance:
feature_importances = pd.concat(
[feature_importances,
model.feature_importance(X_test)],
axis=0)
# 予測の平均値を出力する
pred_avg = np.mean(preds, axis=0)
# 予測結果の保存
Jbl.save(pred_avg,
f"{PATH['prefix']['prediction']}/{self.run_name}-test.jbl")
logger.info(f"{self.run_name} - end prediction cv")
# 特徴量の重要度
if show_feature_importance:
aggs = (feature_importances.groupby("Feature").mean().sort_values(
by="importance", ascending=False))
cols = aggs[:200].index
pd.DataFrame(aggs.index).to_csv(
f"{PATH['prefix']['importance']}/{self.run_name}-fi.csv",
index=False)
best_features = feature_importances.loc[
feature_importances.Feature.isin(cols)]
plt.figure(figsize=(14, 26))
sns.barplot(
x="importance",
y="Feature",
data=best_features.sort_values(by="importance",
ascending=False),
)
plt.title("LightGBM Features (averaged over folds)")
plt.tight_layout()
plt.savefig(
f"{PATH['prefix']['importance']}/{self.run_name}-fi.png")
plt.show()
# mlflow
mlflow.start_run(run_id=self.run_id)
log_artifact(
f"{PATH['prefix']['importance']}/{self.run_name}-fi.png")
mlflow.end_run()
def run_train_cv(self) -> None:
"""クロスバリデーションでの学習・評価を行う
学習・評価とともに、各foldのモデルの保存、スコアのログ出力についても行う
"""
# mlflow
mlflow.set_experiment(self.exp_name)
mlflow.start_run(run_name=self.run_name)
logger.info(f"{self.run_name} - start training cv")
scores = []
va_idxes = []
preds = []
# Adversarial validation
if self.advanced and "adversarial_validation" in self.advanced:
X_train = self.X_train
X_test = self.X_test
X_train["target"] = 0
X_test["target"] = 1
X_train = pd.concat([X_train, X_test],
sort=False).reset_index(drop=True)
y_train = X_train["target"]
X_train.drop("target", axis=1, inplace=True)
X_test.drop("target", axis=1, inplace=True)
self.X_train = X_train
self.y_train = y_train
# 各foldで学習を行う
for i_fold in range(self.cv.n_splits):
# 学習を行う
logger.info(f"{self.run_name} fold {i_fold} - start training")
model, va_idx, va_pred, score = self.train_fold(i_fold)
logger.info(
f"{self.run_name} fold {i_fold} - end training - score {score}\tbest_iteration: {model.model.best_iteration}"
)
# モデルを保存する
model.save_model()
# 結果を保持する
va_idxes.append(va_idx)
scores.append(score)
preds.append(va_pred)
# 各foldの結果をまとめる
va_idxes = np.concatenate(va_idxes)
order = np.argsort(va_idxes)
preds = np.concatenate(preds, axis=0)
preds = preds[order]
cv_score = self.evaluate(self.y_train.values, preds)
logger.info(
f"{self.run_name} - end training cv - score {cv_score}\tbest_iteration: {model.model.best_iteration}"
)
# 予測結果の保存
Jbl.save(preds,
f"{PATH['prefix']['prediction']}/{self.run_name}-train.jbl")
# mlflow
self.run_id = mlflow.active_run().info.run_id
log_param("model_name", self.model_cls.__class__.__name__)
log_param("fe_name", self.fe_name)
log_param("train_params", self.params)
log_param("cv_strategy", str(self.cv))
log_param("evaluation_metric", self.evaluation_metric)
log_metric("cv_score", cv_score)
log_param(
"fold_scores",
dict(
zip(
[f"fold_{i}" for i in range(len(scores))],
[round(s, 4) for s in scores],
)),
)
log_param("cols_definition", self.cols_definition)
log_param("description", self.description)
mlflow.end_run()
def save_model(self, path: str = "models/model"):
model_path = os.path.join(path, f"{self.run_fold_name}.model")
os.makedirs(os.path.dirname(model_path), exist_ok=True)
Jbl.save(self.model, model_path)
def save_model(self):
model_path = os.path.join(f"{ModelPath.model}",
f"{self.run_fold_name}.model")
os.makedirs(os.path.dirname(model_path), exist_ok=True)
Jbl.save(self.model, model_path)
def save_model(self, path: str = "models/model"):
model_path = os.path.join(path, f"{self.run_fold_name}.model")
Jbl.save(self.model, model_path)
print(f"{model_path} is saved")
def run_predict_cv(self) -> None:
"""クロスバリデーションで学習した各foldのモデルの平均により、テストデータの予測を行う
あらかじめrun_train_cvを実行しておく必要がある
"""
self.logger.info(f"{self.run_name} - start prediction cv")
X_test = self.X_test.copy()
if self.advanced and self.advanced.Selector is not None:
selector = Jbl.load(
f"{ModelPath.selector}/{self.run_name}.selector")
X_test = selector.transform(X_test)
preds = []
show_feature_importance = "LGBM" in str(self.model_cls)
feature_importances = pd.DataFrame()
# 各foldのモデルで予測を行う
for i_fold in range(self.cv.n_splits):
self.logger.info(
f"{self.run_name} - start prediction fold:{i_fold}")
if self.pretrain is None:
model = self.build_model(i_fold)
else:
model = self.model_cls(f"{self.pretrain.run_name}-{i_fold}",
self.run_cfg, self.cat_cols)
model.load_model()
pred = model.predict(X_test)
# 後処理
pred = postprocess_prediction(pred)
preds.append(pred)
self.logger.info(f"{self.run_name} - end prediction fold:{i_fold}")
if show_feature_importance:
feature_importances = pd.concat(
[feature_importances,
model.feature_importance(X_test)],
axis=0)
# 予測の平均値を出力する
pred_avg = np.mean(preds, axis=0)
# 閾値で2値化
if self.pretrain is None:
best_threshold = Jbl.load(
f"{ModelPath.prediction}/{self.run_name}-best-threshold.jbl")
else:
best_threshold = Jbl.load(
f"{ModelPath.prediction}/{self.pretrain.run_name}-best-threshold.jbl"
)
pred_avg_binarized = np.where(pred_avg > best_threshold, 1, 0)
# 予測結果の保存
Jbl.save(pred_avg, f"{ModelPath.prediction}/{self.run_name}-test.jbl")
Jbl.save(
pred_avg_binarized,
f"{ModelPath.prediction}/{self.run_name}-test-binarized.jbl",
)
self.logger.info(f"{self.run_name} - end prediction cv")
# 特徴量の重要度
if show_feature_importance:
aggs = (feature_importances.groupby("Feature").mean().sort_values(
by="importance", ascending=False))
cols = aggs[:200].index
pd.DataFrame(aggs.index).to_csv(
f"{ModelPath.importance}/{self.run_name}-fi.csv", index=False)
best_features = feature_importances.loc[
feature_importances.Feature.isin(cols)]
plt.figure(figsize=(14, 26))
sns.barplot(
x="importance",
y="Feature",
data=best_features.sort_values(by="importance",
ascending=False),
)
plt.title("LightGBM Features (averaged over folds)")
plt.tight_layout()
plt.savefig(f"{ModelPath.importance}/{self.run_name}-fi.png")
plt.show()
# mlflow
mlflow.start_run(run_id=self.run_id)
log_artifact(f"{ModelPath.importance}/{self.run_name}-fi.png")
mlflow.end_run()
def run_train_cv(self) -> None:
"""クロスバリデーションでの学習・評価を行う
学習・評価とともに、各foldのモデルの保存、スコアのログ出力についても行う
"""
# mlflow
mlflow.set_experiment(self.exp_name)
mlflow.start_run(run_name=self.run_name)
self.logger.info(f"{self.run_name} - start training cv")
scores = []
va_idxes = []
preds = []
# Adversarial validation
if self.advanced and self.advanced.AdversarialValidation is not None:
X_train = self.X_train.copy()
X_test = self.X_test.copy()
X_train["target"] = 0
X_test["target"] = 1
X_train = pd.concat([X_train, X_test],
sort=False).reset_index(drop=True)
y_train = X_train["target"]
X_train.drop("target", axis=1, inplace=True)
X_test.drop("target", axis=1, inplace=True)
self.X_train = X_train
self.y_train = y_train
# 特徴量選択
if self.advanced and self.advanced.Selector is not None:
self.logger.info(f"{self.run_name} - start feature_selection")
self.logger.info(
f"{self.run_name} - #features before selection: {len(self.X_train.columns.tolist())}"
)
selector_params = dataclasses.asdict(self.advanced.Selector)
selector_name = selector_params.pop("name")
selector = None
if selector_name == "GBDTFeatureSelector":
selector = GBDTFeatureSelector(
input_cols=self.X_train.columns.tolist(),
target_col=self.column.target,
**selector_params,
)
else:
ValueError(f"{selector_name} is not implemented")
self.X_train = selector.fit_transform(
pd.concat([self.X_train, self.y_train], axis=1))
self.X_test = selector.transform(self.X_test)
self.logger.info(
f"{self.run_name} - #features after selection: {len(self.X_train.columns.tolist())}"
)
self.logger.info(f"{self.run_name} - end feature_selection")
os.makedirs(f"{ModelPath.selector}", exist_ok=True)
Jbl.save(selector,
f"{ModelPath.selector}/{self.run_name}.selector")
# 各foldで学習を行う
for i_fold in range(self.cv.n_splits):
# 学習を行う
self.logger.info(f"{self.run_name} fold {i_fold} - start training")
model, va_idx, va_pred, score = self.train_fold(i_fold)
fold_score_log = (
f"{self.run_name} fold {i_fold} - end training - score {score}"
)
if hasattr(model.model, "best_iteration"):
fold_score_log += f"\tbest_iteration: {model.model.best_iteration}"
self.logger.info(fold_score_log)
self.logger.info(
f"{self.run_name} fold {i_fold} - best threshold - {self.best_threshold}"
)
# モデルを保存する
model.save_model()
# 結果を保持する
va_idxes.append(va_idx)
scores.append(score)
preds.append(va_pred)
# 各foldの結果をまとめる
va_idxes = np.concatenate(va_idxes)
order = np.argsort(va_idxes)
preds = np.concatenate(preds, axis=0)
preds = preds[order]
cv_score = self.evaluate(self.y_train.values, preds)
preds_binarized = np.where(preds > self.best_threshold, 1, 0)
self.logger.info(
f"{self.run_name} - end training cv - score {cv_score}")
self.logger.info(
f"{self.run_name} - best threshold - {self.best_threshold}")
# 予測結果の保存
Jbl.save(preds, f"{ModelPath.prediction}/{self.run_name}-train.jbl")
Jbl.save(
preds_binarized,
f"{ModelPath.prediction}/{self.run_name}-train-binarized.jbl",
)
Jbl.save(
self.best_threshold,
f"{ModelPath.prediction}/{self.run_name}-best-threshold.jbl",
)
# mlflow
self.mlflow(cv_score, scores)