def test_empty_patterns(self, conf_paths):
"""Check the error if no config patterns were specified"""
pattern = (r"`patterns` must contain at least one glob pattern "
r"to match config filenames against")
with pytest.raises(ValueError, match=pattern):
ConfigLoader(conf_paths).get()
def test_load_base_config(self, tmp_path, conf_paths, base_config):
"""Test config loading if `local/` directory is empty"""
(tmp_path / "local").mkdir(exist_ok=True)
catalog = ConfigLoader(conf_paths).get("catalog*.yml")
assert catalog == base_config
def test_nested_subdirs(self, tmp_path):
"""Test loading the config from subdirectories"""
catalog = ConfigLoader(str(tmp_path / "base")).get("**/catalog*")
assert (catalog["cars"]["type"] == catalog["prod"]["cars"]["type"] ==
"CSVLocalDataSet")
assert catalog["cars"]["save_args"]["index"] is True
def register_config_loader(self,
conf_paths: Iterable[str]) -> ConfigLoader:
return ConfigLoader(conf_paths)
def __get_creds():
conf_paths = ["conf/base", "conf/local"]
conf_loader = ConfigLoader(conf_paths)
credentials = conf_loader.get("credentials*", "credentials*/**")
return credentials
def register_config_loader(self,
conf_paths: Iterable[str]) -> ConfigLoader:
import kedro.config.config
kedro.config.config._load_config = _load_config
return ConfigLoader(conf_paths)
def register_config_loader(self, conf_paths: Iterable[str], env: str,
extra_params: Dict[str, Any]) -> ConfigLoader:
return ConfigLoader(conf_paths)
def register_config_loader(self,
conf_paths: Iterable[str]) -> ConfigLoader:
self.logger.info("Registering config loader",
extra={"conf_paths": conf_paths})
return ConfigLoader(conf_paths)
def register_config_loader(self, conf_paths) -> ConfigLoader:
return ConfigLoader(conf_paths)
import lightgbm as lgb
import numpy as np
import pandas as pd
import shap
import datetime
import matplotlib.pyplot as plt
import seaborn as sns
import logging
from hyperopt import STATUS_OK
from digital_reputation_challenge.nodes.datatransform import get_fold_data
from kedro.config import ConfigLoader
conf_paths = ['conf/base', 'conf/local']
conf_loader = ConfigLoader(conf_paths)
conf_parameters = conf_loader.get('parameters*', 'parameters*/**')
from sklearn.model_selection import StratifiedKFold, KFold, RepeatedStratifiedKFold
class CV_score:
def __init__(self,
params,
cols_all,
col_target,
cols_cat='auto',
num_boost_round=99999,
early_stopping_rounds=50,
valid=True):
self.params = params
self.cols_all = cols_all
def register_config_loader(self, conf_paths): # pylint: disable=no-self-use
return ConfigLoader(conf_paths)
from kedro.config import ConfigLoader
from kedro.io import DataCatalog
# Find the configuration (catalog.yaml) in the current working directory and load it
conf_loader = ConfigLoader(".")
conf_catalog = conf_loader.get("catalog*")
# Create the Data Catalog from the catalog.yml file
io = DataCatalog.from_config(conf_catalog)
df = io.load("titanic_training_data")
print(df.head())
def kedro_conf(mock_config):
config = ConfigLoader(["conf/base", "conf/local"])
config_dict = config.get("catalog*", "catalog*/**")
return config_dict
def valid_rf(race_results_df_processed_valid, model_rf, parameters):
# mlflow
print('FILE_DIR: ' + FILE_DIR)
mlflow.set_tracking_uri(FILE_DIR + '/../../../logs/mlruns/')
mlflow.set_experiment('forecast_keiba_valid')
run_info = mlflow.start_run()
mlflow.set_tag('model', 'lr')
# 検証のデータ準備
race_results_df_processed_valid = race_results_df_processed_valid
# 説明変数の取得
X_valid = race_results_df_processed_valid.drop(['rank'], axis=1)
# 目的変数の取得
y_valid = race_results_df_processed_valid['rank']
# 推論実行
y_valid_pred = model_rf.predict(X_valid)
# 集計用に処理
valid_results_df = pd.DataFrame({'pred': y_valid_pred, 'actual': y_valid})
race_id_list = list(set(list(valid_results_df.index)))
valid_results_list = valid_results_df.reset_index().values.tolist()
# シャッフル
random.shuffle(valid_results_list)
# 集計（馬単）
correct_count = 0
for race_id in race_id_list:
pred_cnt_by_race = 0
cnt_by_race = 0
for rank in [1]:
for i in range(len(valid_results_list)):
# 対象レースidのうち、{rank}位と予測された馬
if valid_results_list[i][0] == race_id and valid_results_list[
i][1] == rank:
pred_cnt_by_race += 1
if pred_cnt_by_race <= 1 and (valid_results_list[i][2]
== 1):
cnt_by_race += 1
if cnt_by_race == 1:
correct_count += 1
acc_exacta_1 = correct_count / 100
print('acc_exacta_1: ' + str(acc_exacta_1))
# 集計（馬連）
correct_count = 0
for race_id in race_id_list:
pred_cnt_by_race = 0
cnt_by_race = 0
for rank in [1, 2]:
for i in range(len(valid_results_list)):
# 対象レースidのうち、{rank}位と予測された馬
if valid_results_list[i][0] == race_id and valid_results_list[
i][1] == rank:
pred_cnt_by_race += 1
if pred_cnt_by_race <= 2 and (valid_results_list[i][2] == 1
or valid_results_list[i][2]
== 2):
cnt_by_race += 1
if cnt_by_race == 2:
correct_count += 1
acc_quinella_2 = correct_count / 100
print('acc_quinella_2: ' + str(acc_quinella_2))
# 集計（三連複）
correct_count = 0
for race_id in race_id_list:
pred_cnt_by_race = 0
cnt_by_race = 0
for rank in [1, 2, 3]:
for i in range(len(valid_results_list)):
# 対象レースidのうち、{rank}位と予測された馬
if valid_results_list[i][0] == race_id and valid_results_list[
i][1] == rank:
pred_cnt_by_race += 1
if pred_cnt_by_race <= 3 and (
valid_results_list[i][2] == 1
or valid_results_list[i][2] == 2
or valid_results_list[i][2] == 3):
cnt_by_race += 1
if cnt_by_race == 3:
correct_count += 1
acc_trio_3 = correct_count / 100
print('acc_trio_3: ' + str(acc_trio_3))
mlflow.log_metric("acc_exacta_1", acc_exacta_1)
mlflow.log_metric("acc_quinella_2", acc_quinella_2)
mlflow.log_metric("acc_trio_3", acc_trio_3)
# 通知
if parameters['is_notify']:
run_result_dict = mlflow.get_run(run_info.info.run_id).to_dictionary()
run_result_str = json.dumps(run_result_dict, indent=4)
conf_paths = [
FILE_DIR + "/../../../conf/base", FILE_DIR + "/../../../conf/local"
]
conf_loader = ConfigLoader(conf_paths)
credentials = conf_loader.get("credentials*", "credentials*/**")
token = credentials['dev_line']['access_token']
url = "https://notify-api.line.me/api/notify"
headers = {"Authorization": "Bearer " + token}
payload = {"message": "model_rf" + run_result_str}
requests.post(url, headers=headers, data=payload)
mlflow.end_run()
def _get_model_params() -> DictConfig:
conf = ConfigLoader(MODEL_PARAMETERS_PATH).get('*.yaml')
return conf['best_evaluated_params']
# AUTOGENERATED! DO NOT EDIT! File to edit: catalog.ipynb (unless otherwise specified).
__all__ = ['conf_loader', 'conf_test_data_catalog', 'test_data_catalog']
# Cell
from kedro.config import ConfigLoader
from kedro.io import DataCatalog
# Cell
conf_loader = ConfigLoader("conf/base")
conf_test_data_catalog = conf_loader.get("catalog*.yaml", "catalog*/*.yaml")
test_data_catalog = DataCatalog.from_config(conf_test_data_catalog)
