def on_epoch_begin(self, epoch, logs=None):
log_metric("MO_learning_rate", eval(self.model.optimizer.lr))
enc = OneHotEncoder()
Y = enc.fit_transform(y[:, np.newaxis]).toarray()
# Scale data to have mean 0 and variance 1
# which is importance for convergence of the neural network
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
x_train, x_test, y_train, y_test = train_test_split(X_scaled,
Y,
test_size=0.2,
random_state=42)
with mlflow.start_run():
model = tf.keras.Sequential([
tf.keras.layers.Dense(10, activation=tf.nn.relu,
input_shape=(4, )),
tf.keras.layers.Dense(10, activation=tf.nn.relu),
tf.keras.layers.Dense(3, activation="softmax")
])
model.compile(optimizer='adam',
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=['accuracy'])
model.fit(x_train, y_train, epochs=1000)
test_loss, test_acc = model.evaluate(x_test, y_test)
log_metric('acc', test_acc)
log_model(keras_model=model,
registered_model_name='Keras-Iris-Model',
artifact_path='model_artifact',
conda_env=conda_env)
if __name__ == '__main__':
with open('test.file', 'wb') as f:
test = range(10)
joblib.dump(test, f)
local_registry = "sqlite:///mlruns.db"
print(f"Running local model registry={local_registry}")
mlflow.set_tracking_uri(local_registry)
clt = MlflowClient()
exp_a_uri = "file:///tmp/exp_A"
exp_a_id = get_exp_id("experiment_A", local_registry, local_registry,
exp_a_uri)
exp_b_uri = "file:///tmp/exp_B"
exp_b_id = get_exp_id("experiment_B", local_registry, local_registry,
exp_b_uri)
for i in range(3):
with mlflow.start_run(experiment_id=exp_a_id):
mlflow.log_metric("MEAN SQUARE ERROR", 0.25 * random())
mlflow.log_artifact('test.file')
print(f"artifact_uri={mlflow.get_artifact_uri()}")
print("-" * 75)
for i in range(3):
with mlflow.start_run(experiment_id=exp_b_id):
mlflow.log_metric("MEAN SQUARE ERROR", 0.25 * random())
mlflow.log_artifact('test.file')
print(f"artifact_uri={mlflow.get_artifact_uri()}")
'parameters': str(rand()),
'in': str(rand()),
'this': str(rand()),
'experiement': str(rand()),
'run': str(rand()),
'because': str(rand()),
'we': str(rand()),
'need': str(rand()),
'to': str(rand()),
'check': str(rand()),
'how': str(rand()),
'it': str(rand()),
'handles': str(rand()),
}
log_params(parameters)
mlflow.log_metric('test_metric', 1)
with mlflow.start_run(run_name='child_metrics.py', nested=True):
metrics = {
'lot': [rand()],
'of': [rand()],
'parameters': [rand()],
'in': [rand()],
'this': [rand()],
'experiement': [rand()],
'run': [rand()],
'because': [rand()],
'we': [rand()],
'need': [rand()],
'to': [rand()],
'check': [rand()],
def train(in_alpha, in_l1_ratio, masterid):
import os
import warnings
import sys
import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
from sklearn.model_selection import train_test_split
from sklearn.linear_model import ElasticNet
import mlflow
import mlflow.sklearn
import logging
""
logging.basicConfig(level=logging.WARN)
logger = logging.getLogger(__name__)
def eval_metrics(actual, pred):
rmse = np.sqrt(mean_squared_error(actual, pred))
mae = mean_absolute_error(actual, pred)
r2 = r2_score(actual, pred)
return rmse, mae, r2
warnings.filterwarnings("ignore")
np.random.seed(40)
#print("xxxxxxxxxxxxxxxxxxxxx")
# Read data from hive table prepared before
data = {}
try:
print(masterid)
data = query_hive_data(masterid)[[
"sequence", "x_basic_hour", "x_basic_horizon", "i_set", "ec_ws",
"ec_wd", "ec_tmp", "ec_press", "ec_rho", "ec_dist", "gfs_ws",
"gfs_wd", "gfs_tmp", "gfs_press", "gfs_rho", "gfs_dist", "speed",
"power"
]]
data = data.fillna('1')
#print(data)
except Exception as e:
logger.exception("Get Hive Data Error: %s", e)
print(data)
# Split the data into training and test sets. (0.75, 0.25) split.
train, test = train_test_split(data)
# The predicted column is "power"
train_x = train.drop(["power", "speed"], axis=1)
test_x = test.drop(["power", "speed"], axis=1)
train_y = train[["power"]]
test_y = test[["power"]]
# Set default values if no alpha is provided
if float(in_alpha) is None:
alpha = 0.5
else:
alpha = float(in_alpha)
# Set default values if no l1_ratio is provided
if float(in_l1_ratio) is None:
l1_ratio = 0.5
else:
l1_ratio = float(in_l1_ratio)
# Useful for multiple runs (only doing one run in this sample notebook)
with mlflow.start_run():
# Execute ElasticNet
lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42)
lr.fit(train_x, train_y)
#print(test_x)
# Evaluate Metrics
predicted_powers = lr.predict(test_x)
(rmse, mae, r2) = eval_metrics(test_y, predicted_powers)
# Print out metrics
print("Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio))
print(" RMSE: %s" % rmse)
print(" MAE: %s" % mae)
print(" R2: %s" % r2)
# Log parameter, metrics, and model to MLflow
mlflow.log_param("alpha", alpha)
mlflow.log_param("l1_ratio", l1_ratio)
mlflow.log_metric("rmse", rmse)
mlflow.log_metric("r2", r2)
mlflow.log_metric("mae", mae)
mlflow.sklearn.log_model(lr, "model")
def eval_Slim(params, cfg, train_mat, eval_mat, experiment):
# This function is what Hyperopt is going to optimize (minimize 'loss' value)
print(experiment)
with mlflow.start_run(experiment_id=experiment):
# Log the config
utils.config_helpers.log_config(dict(cfg.model))
n_users, n_items = train_mat.shape
np.random.seed(seed=cfg.model.seed)
# Log relevant parameters for this run.
mlflow.log_param("alpha", params['alpha'])
mlflow.log_param("l1_ratio", params['l1_ratio'])
mlflow.log_param("max_iter", params['max_iter'])
mlflow.log_param("tol", params['tol'])
# Log this run
log.info(
f"Testing alpha: {params['alpha']}, l1_ratio: {params['l1_ratio']}, max_iter: {params['max_iter']} and tol: {params['tol']}"
)
start = time.time()
# Create model
slim = RecModel.Slim(num_items=n_items, num_users=n_users)
# Train Model
slim.train(X=train_mat.copy(),
alpha=params['alpha'],
l1_ratio=params['l1_ratio'],
max_iter=params['max_iter'],
tolerance=params['tol'],
cores=1,
verbose=int(cfg.model.verbose))
# Log run-time
mlflow.log_metric("Runtime", int(round(time.time() - start, 0)))
# Evaluate model
perf_all = slim.eval_topn(eval_mat.copy(),
rand_sampled=int(cfg.model.rand_sampled),
topn=np.array(cfg.model.top_n_performances,
dtype=np.int32),
random_state=int(cfg.model.seed),
cores=int(cfg.model.cores))
# Log the performance of the model
for pos in range(len(cfg.model.top_n_performances)):
mlflow.log_metric(
f"recallAT{cfg.model.top_n_performances[pos]}_of_{cfg.model.rand_sampled}",
perf_all[f"Recall@{cfg.model.top_n_performances[pos]}"])
mlflow.log_metric('MAE_train', slim.eval_prec(train_mat.copy()))
mlflow.log_metric('MAE_eval', slim.eval_prec(eval_mat.copy()))
#We will always choose the first topn performance. Hopefully, that is also the smallest is most relevant for us.g
rel_topn_perf = perf_all[f"Recall@{cfg.model.top_n_performances[0]}"]
log.info(
f"Current recallAT{cfg.model.top_n_performances[0]}_of_{cfg.model.rand_sampled} performance was {rel_topn_perf}"
)
loss = -rel_topn_perf
return {'loss': loss, 'status': hp.STATUS_OK, 'eval_time': time.time()}
def log_metric(metric_name, value, step=None):
print(f"[INFO] Logging metric: {metric_name}")
mlflow.log_metric(metric_name, value, step=None)
def log_metric(self, metric: str, value: Union[int, float]):
mlflow.log_metric(metric, value)
if batch % log_step == 0:
mlflow.log_metric('training_loss', logs.get('loss'))
mlflow.log_metric('training_acc', logs.get('acc'))
batch_callback = LambdaCallback(
on_batch_end=lambda batch, logs: track_loss_acc(logs))
epoch_callback = LambdaCallback(
on_epoch_end=lambda epoch, logs: print("Epoch Complete"))
complete_callback = LambdaCallback(on_train_end=lambda logs: print(logs))
# def main():
inputs = Input(shape=(image_pixels, ))
# hidden = Dense(int(image_pixels * 0.37), activation='relu')(inputs)
outputs = Dense(num_labels, activation='softmax')(inputs)
model = Model(inputs=inputs, outputs=outputs)
sgd = SGD(lr=learning_rate)
model.compile(optimizer=sgd, loss=loss_func, metrics=metrics)
fit_resp = model.fit(x=x_train, y=y_train, callbacks=[batch_callback])
mlflow.log_metric('training_loss', fit_resp.history.get('loss')[0])
mlflow.log_metric('training_acc', fit_resp.history.get('acc')[0])
loss, acc = model.evaluate(x_test, y_test)
mlflow.log_metric('loss', loss)
mlflow.log_metric('acc', acc)
# if __name__ == '__main__':
# main()
def track_loss_acc(logs, log_step=100):
batch = logs.get('batch')
if batch % log_step == 0:
mlflow.log_metric('training_loss', logs.get('loss'))
mlflow.log_metric('training_acc', logs.get('acc'))
def valid_lr(race_results_df_processed_valid, model_lr, parameters):
# mlflow
print('FILE_DIR: ' + FILE_DIR)
mlflow.set_tracking_uri(FILE_DIR + '/../../../logs/mlruns/')
mlflow.set_experiment('forecast_keiba_valid')
run_info = mlflow.start_run()
mlflow.set_tag('model', 'lr')
# 検証のデータ準備
race_results_df_processed_valid = race_results_df_processed_valid
# 説明変数の取得
X_valid = race_results_df_processed_valid.drop(['rank'],axis=1)
# 目的変数の取得
y_valid = race_results_df_processed_valid['rank']
# 推論実行
y_valid_pred = model_lr.predict(X_valid)
# 集計用に処理
valid_results_df = pd.DataFrame({'pred':y_valid_pred,'actual':y_valid})
race_id_list = list(set(list(valid_results_df.index)))
valid_results_list = valid_results_df.reset_index().values.tolist()
# シャッフル
random.shuffle(valid_results_list)
# 集計（馬単）
correct_count = 0
for race_id in race_id_list:
pred_cnt_by_race = 0
cnt_by_race = 0
for rank in [1]:
for i in range(len(valid_results_list)):
# 対象レースidのうち、{rank}位と予測された馬
if valid_results_list[i][0] == race_id and valid_results_list[i][1] == rank:
pred_cnt_by_race += 1
if pred_cnt_by_race <= 1 and (valid_results_list[i][2] == 1):
cnt_by_race += 1
if cnt_by_race == 1:
correct_count += 1
acc_exacta_1 = correct_count/100
print('acc_exacta_1: ' + str(acc_exacta_1))
# 集計（馬連）
correct_count = 0
for race_id in race_id_list:
pred_cnt_by_race = 0
cnt_by_race = 0
for rank in [1, 2]:
for i in range(len(valid_results_list)):
# 対象レースidのうち、{rank}位と予測された馬
if valid_results_list[i][0] == race_id and valid_results_list[i][1] == rank:
pred_cnt_by_race += 1
if pred_cnt_by_race <= 2 and (valid_results_list[i][2] == 1 or valid_results_list[i][2] == 2):
cnt_by_race += 1
if cnt_by_race == 2:
correct_count += 1
acc_quinella_2 = correct_count/100
print('acc_quinella_2: ' + str(acc_quinella_2))
# 集計（三連複）
correct_count = 0
for race_id in race_id_list:
pred_cnt_by_race = 0
cnt_by_race = 0
for rank in [1, 2, 3]:
for i in range(len(valid_results_list)):
# 対象レースidのうち、{rank}位と予測された馬
if valid_results_list[i][0] == race_id and valid_results_list[i][1] == rank:
pred_cnt_by_race += 1
if pred_cnt_by_race <= 3 and (valid_results_list[i][2] == 1 or valid_results_list[i][2] == 2 or valid_results_list[i][2] == 3):
cnt_by_race += 1
if cnt_by_race == 3:
correct_count += 1
acc_trio_3 = correct_count/100
print('acc_trio_3: ' + str(acc_trio_3))
mlflow.log_metric("acc_exacta_1", acc_exacta_1)
mlflow.log_metric("acc_quinella_2", acc_quinella_2)
mlflow.log_metric("acc_trio_3", acc_trio_3)
# 通知
if parameters['is_notify']:
run_result_dict = mlflow.get_run(run_info.info.run_id).to_dictionary()
run_result_str = json.dumps(run_result_dict, indent=4)
conf_paths = [FILE_DIR + "/../../../conf/base", FILE_DIR + "/../../../conf/local"]
conf_loader = ConfigLoader(conf_paths)
credentials = conf_loader.get("credentials*", "credentials*/**")
token = credentials['dev_line']['access_token']
url = "https://notify-api.line.me/api/notify"
headers = {"Authorization": "Bearer " + token}
payload = {"message": "model_lr" + run_result_str}
requests.post(url, headers=headers, data=payload)
mlflow.end_run()
max_iterations = 150
# log parameters
mlflow.log_param("random_state", random_state)
mlflow.log_param("solver", solver)
mlflow.log_param("multi_class", multi_class)
mlflow.log_param("max_iterations", max_iterations)
mlflow.set_tag("framework", "sklearn")
# create a model
model = LogisticRegression(random_state=random_state,
solver=solver,
multi_class=multi_class,
max_iter=max_iterations)
model = model.fit(X, y)
# log metris and persiste model via mlflow.<flavour>.log_model
score = model.score(X, y)
mlflow.log_metric("score", model.score(X, y))
mlflow.sklearn.log_model(model, artifact_path)
# obtain run_id for next steps
run_id = mlflow.active_run().info.run_id
print("run_id: {}".format(run_id))
print("Score: {}".format(model.score(X, y)))
# COMMAND ----------
# MAGIC %md ### Load model
# COMMAND ----------
# MAGIC %md
# MAGIC In this step we read the model from a persistent artifact storage we created in the previous step. This is to demonstrate the retrieval of created models using `run_id` as we will need in the next step when deploying to AzureML.
X = pd.DataFrame(boston.data, columns=boston.feature_names)
y = pd.Series(boston.target)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.25)
mlflow.set_tag(EVALUATION_SET_TAG, "25 percent Split")
print("building regressor")
regressor = xgb.XGBRegressor(n_estimators=100,
reg_lambda=1,
gamma=0,
max_depth=3)
print("fitting model")
regressor.fit(X_train, y_train)
print("predicting on eval set")
y_pred = regressor.predict(X_test)
# Log the mean squared error
print("calculating MSE")
mse = mean_squared_error(y_test, y_pred)
mlflow.log_metric("mse", mse)
# Save the model artifact
print("Saving model")
regressor.save_model(f"{BASE_DIR}/outputs-xgb/model.json")
mlflow.log_artifacts(f"{BASE_DIR}/outputs-xgb")
print("done.")
def log_learning_curve(model_name: str, model: Any, fold=0):
'''
Function to log learning curve.
For GBDT models, the schema of evals_result is uniform like below:
evals_result = {
'validation_0': {'logloss': ['0.604835', '0.531479']},
'validation_1': {'logloss': ['0.41965', '0.17686']}
}
example key for output: fold0_valid0-logloss
'''
if model_name == 'XGBClassifier':
evals_result = model.evals_result()
for eval_idx in range(len(evals_result)):
validation_X_raw = f'validation_{eval_idx}' # this is the raw expression from model
metricdict = evals_result[validation_X_raw]
for metricname, scorelist in metricdict.items(
): # this loops only once
metricname = get_normalized_metricname(metricname)
for i, score in enumerate(scorelist):
mlflow.log_metric(
f'fold{fold}_valid{eval_idx}-{metricname}_vsstep',
score, i)
elif model_name == 'LGBMClassifier':
evals_result = model.evals_result_
for eval_idx in range(len(evals_result)):
validation_X_raw = 'training' if eval_idx == 0 else f'valid_{eval_idx}' # this is the raw expression from model
metricdict = evals_result[validation_X_raw]
for metricname, scorelist in metricdict.items(
): # this loops only once
metricname = get_normalized_metricname(metricname)
for i, score in enumerate(scorelist):
mlflow.log_metric(
f'fold{fold}_valid{eval_idx}-{metricname}_vsstep',
score, i)
elif model_name == 'CatBoostClassifier':
evals_result = model.get_evals_result()
for eval_idx in range(
len(evals_result) - 1
): # skip key 'learn', which contains same value as validation_0
validation_X_raw = f'validation_{eval_idx}'
metricdict = evals_result[validation_X_raw]
for metricname, scorelist in metricdict.items(
): # this loops only once
metricname = get_normalized_metricname(metricname)
for i, score in enumerate(scorelist):
mlflow.log_metric(
f'fold{fold}_valid{eval_idx}-{metricname}_vsstep',
score, i)
elif model_name == 'RandomForestClassifier2':
evals_result = model.get_evals_result()
for eval_idx in range(len(evals_result)):
validation_X_raw = f'valid{eval_idx}'
metricdict = evals_result[validation_X_raw]
for metricname, scorelist in metricdict.items(
): # this loops only once
metricname = get_normalized_metricname(metricname)
for i, score in enumerate(scorelist):
mlflow.log_metric(
f'fold{fold}_valid{eval_idx}-{metricname}_vsstep',
score, i)
else:
raise ValueError(f'Invalid model_name: {model_name}')
def main(args):
data_dir = args.data_dir
figure_path = args.figure_dir
model_path = args.model_dir
file_name = "data.hdf5"
# Set skip_training to False if the model has to be trained, to True if the model has to be loaded.
skip_training = False
# Set the torch device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("Device = {}".format(device))
# Initialize parameters
parameters = Params_cross(
subject_n=args.sub,
hand=args.hand,
batch_size=args.batch_size,
valid_batch_size=args.batch_size_valid,
test_batch_size=args.batch_size_test,
epochs=args.epochs,
lr=args.learning_rate,
wd=args.weight_decay,
patience=args.patience,
device=device,
y_measure=args.y_measure,
desc=args.desc,
)
# Import data and generate train-, valid- and test-set
# Set if generate with RPS values or not (check network architecture used later)
print("Testing: {} ".format(parameters.desc))
dataset = MEG_Cross_Dataset_no_bp(
data_dir,
file_name,
parameters.subject_n,
mode="train",
y_measure=parameters.y_measure,
)
test_dataset = MEG_Cross_Dataset_no_bp(
data_dir,
file_name,
parameters.subject_n,
mode="test",
y_measure=parameters.y_measure,
)
# split the dataset in train, test and valid sets.
train_len, valid_len = len_split_cross(len(dataset))
# train_dataset, valid_test, test_dataset = random_split(dataset, [train_len, valid_len, test_len],
# generator=torch.Generator().manual_seed(42))
train_dataset, valid_dataset = random_split(
dataset, [train_len, valid_len]
)
print(
"Train dataset len {}, valid dataset len {}, test dataset len {}".format(
len(train_dataset), len(valid_dataset), len(test_dataset)
)
)
# Initialize the dataloaders
trainloader = DataLoader(
train_dataset,
batch_size=parameters.batch_size,
shuffle=True,
num_workers=4,
)
validloader = DataLoader(
valid_dataset,
batch_size=parameters.valid_batch_size,
shuffle=True,
num_workers=4,
)
testloader = DataLoader(
test_dataset,
batch_size=parameters.test_batch_size,
shuffle=False,
num_workers=4,
)
# Initialize network
with torch.no_grad():
sample, y = iter(trainloader).next()
n_times = sample.shape[-1]
net = MNet(n_times)
print(net)
# Training loop or model loading
if not skip_training:
print("Begin training....")
# Check the optimizer before running (different from model to model)
# optimizer = Adam(net.parameters(), lr=parameters.lr, weight_decay=5e-4)
optimizer = SGD(net.parameters(), lr=parameters.lr, weight_decay=5e-4)
scheduler = ReduceLROnPlateau(optimizer, mode="min", factor=0.5,
patience=15)
print("scheduler : ", scheduler)
loss_function = torch.nn.MSELoss()
start_time = timer.time()
net, train_loss, valid_loss = train(
net,
trainloader,
validloader,
optimizer,
scheduler,
loss_function,
parameters.device,
parameters.epochs,
parameters.patience,
parameters.hand,
model_path,
)
train_time = timer.time() - start_time
print("Training done in {:.4f}".format(train_time))
# visualize the loss as the network trained
fig = plt.figure(figsize=(10, 4))
plt.plot(
range(1, len(train_loss) + 1), train_loss, label="Training Loss"
)
plt.plot(
range(1, len(valid_loss) + 1), valid_loss, label="Validation Loss"
)
# find position of lowest validation loss
minposs = valid_loss.index(min(valid_loss)) + 1
plt.axvline(
minposs,
linestyle="--",
color="r",
label="Early Stopping Checkpoint",
)
plt.xlabel("epochs")
plt.ylabel("loss")
# plt.ylim(0, 0.5) # consistent scale
# plt.xlim(0, len(train_loss)+1) # consistent scale
plt.grid(True)
plt.legend()
plt.tight_layout()
plt.show()
image1 = fig
plt.savefig(os.path.join(figure_path, "loss_plot.pdf"))
if not skip_training:
# Save the trained model
save_pytorch_model(net, model_path, "model.pth")
else:
# Load the model (properly select the model architecture)
net = MNet()
net = load_pytorch_model(
net, os.path.join(model_path, "model.pth"), parameters.device
)
# Evaluation
print("Evaluation...")
net.eval()
y_pred = []
y = []
y_pred_valid = []
y_valid = []
with torch.no_grad():
for data, labels in testloader:
data, labels = (
data.to(parameters.device),
labels.to(parameters.device),
)
y.extend(list(labels[:, parameters.hand]))
y_pred.extend((list(net(data))))
for data, labels in validloader:
data, labels = (
data.to(parameters.device),
labels.to(parameters.device),
)
y_valid.extend(list(labels[:, parameters.hand]))
y_pred_valid.extend((list(net(data))))
# Calculate Evaluation measures
print("Evaluation measures")
mse = mean_squared_error(y, y_pred)
rmse = mean_squared_error(y, y_pred, squared=False)
mae = mean_absolute_error(y, y_pred)
r2 = r2_score(y, y_pred)
rmse_valid = mean_squared_error(y_valid, y_pred_valid, squared=False)
r2_valid = r2_score(y_valid, y_pred_valid)
valid_loss_last = min(valid_loss)
print("Test set ")
print("mean squared error {}".format(mse))
print("root mean squared error {}".format(rmse))
print("mean absolute error {}".format(mae))
print("r2 score {}".format(r2))
print("Validation set")
print("root mean squared error valid {}".format(rmse_valid))
print("r2 score valid {}".format(r2_valid))
print("last value of the validation loss:".format(valid_loss_last))
# plot y_new against the true value focus on 100 timepoints
fig, ax = plt.subplots(1, 1, figsize=[10, 4])
times = np.arange(200)
ax.plot(times, y_pred[0:200], color="b", label="Predicted")
ax.plot(times, y[0:200], color="r", label="True")
ax.set_xlabel("Times")
ax.set_ylabel("{}".format(parameters.y_measure))
ax.set_title(
"Sub {}, hand {}, {} prediction".format(
str(parameters.subject_n),
"sx" if parameters.hand == 0 else "dx",
parameters.y_measure,
)
)
plt.legend()
plt.savefig(os.path.join(figure_path, "Times_prediction_focus.pdf"))
plt.show()
# plot y_new against the true value
fig, ax = plt.subplots(1, 1, figsize=[10, 4])
times = np.arange(len(y_pred))
ax.plot(times, y_pred, color="b", label="Predicted")
ax.plot(times, y, color="r", label="True")
ax.set_xlabel("Times")
ax.set_ylabel("{}".format(parameters.y_measure))
ax.set_title(
"Sub {}, hand {}, {} prediction".format(
str(parameters.subject_n),
"sx" if parameters.hand == 0 else "dx",
parameters.y_measure,
)
)
plt.legend()
plt.savefig(os.path.join(figure_path, "Times_prediction.pdf"))
plt.show()
# scatterplot y predicted against the true value
fig, ax = plt.subplots(1, 1, figsize=[10, 4])
ax.scatter(np.array(y), np.array(y_pred), color="b", label="Predicted")
ax.set_xlabel("True")
ax.set_ylabel("Predicted")
# plt.legend()
plt.savefig(os.path.join(figure_path, "Scatter.pdf"))
plt.show()
# scatterplot y predicted against the true value
fig, ax = plt.subplots(1, 1, figsize=[10, 4])
ax.scatter(
np.array(y_valid), np.array(y_pred_valid), color="b", label="Predicted"
)
ax.set_xlabel("True")
ax.set_ylabel("Predicted")
# plt.legend()
plt.savefig(os.path.join(figure_path, "Scatter_valid.pdf"))
plt.show()
# log the model and parameters using mlflow tracker
with mlflow.start_run(experiment_id=args.experiment) as run:
for key, value in vars(parameters).items():
mlflow.log_param(key, value)
mlflow.log_param("Time", train_time)
mlflow.log_metric("MSE", mse)
mlflow.log_metric("RMSE", rmse)
mlflow.log_metric("MAE", mae)
mlflow.log_metric("R2", r2)
mlflow.log_metric("RMSE_Valid", rmse_valid)
mlflow.log_metric("R2_Valid", r2_valid)
mlflow.log_metric("Valid_loss", valid_loss_last)
mlflow.log_artifact(os.path.join(figure_path, "Times_prediction.pdf"))
mlflow.log_artifact(
os.path.join(figure_path, "Times_prediction_focus.pdf")
)
mlflow.log_artifact(os.path.join(figure_path, "loss_plot.pdf"))
mlflow.log_artifact(os.path.join(figure_path, "Scatter.pdf"))
mlflow.log_artifact(os.path.join(figure_path, "Scatter_valid.pdf"))
mlflow.pytorch.log_model(net, "models")
average_method='arithmetic'))
clf.fit(X_train, y_train)
i = 0
for p in clf.cv_results_['params']:
# Log best params
run = client.create_run(experiment_id=experiment.experiment_id,
source_version=ver_name,
run_name='run_{0}'.format(str(i)))
with mlflow.start_run(run_uuid=run.info.run_uuid, nested=True):
for k, v in p.items():
mlflow.log_param(k, v)
for k in [
'rank_test_score', 'mean_test_score', 'std_test_score',
'mean_fit_time', 'std_fit_time'
]:
mlflow.log_metric(k, clf.cv_results_[k][i])
for cvix in range(clf.n_splits_):
keyname = 'split{0}_train_score'.format(cvix)
mlflow.log_metric('train_score_cv',
clf.cv_results_[keyname][i])
keyname = 'split{0}_test_score'.format(cvix)
mlflow.log_metric('test_score_cv', clf.cv_results_[keyname][i])
i = i + 1
def run(data_url, target, tracking_uri, model_name, exp_id, output_path, msg):
print("msg", msg)
class XGBWrapper(mlflow.pyfunc.PythonModel):
def load_context(self, context):
import xgboost as xgb
self.xgb_model = xgb.Booster()
self.xgb_model.load_model(context.artifacts["xgb_model"])
def predict(self, context, model_input):
input_matrix = xgb.DMatrix(model_input.values)
return {
'prediction': self.xgb_model.predict(input_matrix),
'msg': msg
}
print("xgboost version", xgb.__version__)
data = pd.read_csv(data_url, sep=";")
label_encoder = preprocessing.LabelEncoder()
label_encoder.fit(data[target].values)
data[target] = label_encoder.transform(data[target])
train, test = train_test_split(data)
train_x = train.drop([target], axis=1)
test_x = test.drop([target], axis=1)
train_y = train[target]
test_y = test[target]
learning_rate = np.random.choice([0.01, 0.1, 0.2])
colsample_bytree = 0.2 + np.random.rand() * 0.4
subsample = 0.2 + np.random.rand() * 0.4
mlflow.set_tracking_uri(tracking_uri)
mlflow.xgboost.autolog()
dtrain = xgb.DMatrix(train_x, label=train_y)
dtest = xgb.DMatrix(test_x, label=test_y)
with mlflow.start_run(run_name=model_name, experiment_id=exp_id):
params = {
'max_depth': 5,
"objective": "multi:softprob",
"num_class": len(np.unique(train_y)),
"learning_rate": learning_rate,
"eval_metric": "mlogloss",
"colsample_bytree": colsample_bytree,
"subsample": subsample,
}
model = xgb.train(params,
dtrain,
num_boost_round=int(5 / learning_rate),
evals=[(dtrain, "train")],
verbose_eval=False)
model.save_model(xgb_model_path)
mlflow.pyfunc.log_model(artifact_path='model',
conda_env=conda_env,
python_model=XGBWrapper(),
artifacts=artifacts,
registered_model_name=model_name)
# evaluate model
train_y_proba = model.predict(dtrain)
train_y_pred = train_y_proba.argmax(axis=1)
test_y_proba = model.predict(dtest)
test_y_pred = test_y_proba.argmax(axis=1)
mlflow.log_metric("train_accuracy",
accuracy_score(train_y_pred, train_y))
mlflow.log_metric("test_accuracy", accuracy_score(test_y_pred, test_y))
# mlflow.xgboost.log_model(model, 'model', registered_model_name=model_name)
client = MlflowClient(tracking_uri)
model_uri = client.search_model_versions(f"name='{model_name}'")[-1].source
info = {"model_uri": model_uri}
print(f"Artifacts saved at {model_uri}.")
pathlib.Path(os.path.dirname(output_path)).mkdir(parents=True,
exist_ok=True)
with open(output_path, 'w') as f:
json.dump(info, f)
def main(params: dict, output_dir: str):
import mlflow
print("start params={}".format(params))
df = pd.read_feather(
"../../riiid_takoi/notebook/data/train_sort.feather").head(len_train)
# df = pd.read_pickle("../input/riiid-test-answer-prediction/split10/train_0.pickle").sort_values(["user_id", "timestamp"]).reset_index(drop=True)
if is_debug:
df = df.head(30000)
for d in load_feature_dir:
df_ = pd.read_feather(d).head(len_train)
if is_debug:
df_ = df_.head(30000)
df = pd.concat([df, df_], axis=1)
print(df.isnull().sum())
# ====================
# preprocess
# ====================
df["content_id"] = df["content_id"] + 2
df["prior_question_had_explanation"] = df[
"prior_question_had_explanation"].fillna(-1) + 2
df["content_id_with_lecture"] = df["content_id"]
df.loc[df["content_type_id"] == 1,
"content_id_with_lecture"] = df["content_id"] + 14000
df["answered_correctly"] += 3
df["task_container_id"] += 1
df["part"] += 1
df["prior_question_elapsed_time"] = df[
"prior_question_elapsed_time"].fillna(0)
df["timestamp_delta"] = df["timestamp_delta"].fillna(0)
df["uid_win_rate"] = df["uid_win_rate"].fillna(
0.65) # target_encoding(user_id)
# 以下は特徴作成時に処理済
# df["content_id_delta"] = df["content_id_delta"].fillna(-1) + 2
# df["last_content_id_acc"] = df["last_content_id_acc"].fillna(-1) + 2
# ====================
# data prepare
# ====================
agg_dict = {
"content_id_with_lecture": list,
"prior_question_had_explanation": list,
"prior_question_elapsed_time": list,
"answered_correctly": list,
"task_container_id": list,
"part": list,
"content_id_delta": list,
"last_content_id_acc": list,
"uid_win_rate": list,
"is_val": list,
"timestamp_delta": list,
}
df_val_row = pd.read_feather(
"../../riiid_takoi/notebook/fe/validation_row_id.feather").head(
len_train // 10)
if is_debug:
df_val_row = df_val_row.head(3000)
df_val_row["is_val"] = 1
df = pd.merge(df, df_val_row, how="left", on="row_id")
df["is_val"] = df["is_val"].fillna(0)
print(df["is_val"].value_counts())
if not load_pickle or is_debug:
# 100件ずつgroupを作る. 例えば950件あったら、 1~50, 51~150, 151~250 のように、先頭が端数になるように
w_df = df[df["is_val"] == 0]
w_df["group"] = (
w_df.groupby("user_id")["user_id"].transform("count") -
w_df.groupby("user_id").cumcount()) // params["max_seq"]
group = w_df.groupby(["user_id", "group"]).agg(agg_dict).T.to_dict()
dataset_train = SAKTDataset(group,
n_skill=60000,
max_seq=params["max_seq"])
del w_df
gc.collect()
group = df[df["content_type_id"] == 0].groupby("user_id").agg(
agg_dict).T.to_dict()
dataset_val = SAKTDataset(group,
is_test=True,
n_skill=60000,
max_seq=params["max_seq"])
os.makedirs("../input/feature_engineering/model200", exist_ok=True)
if not is_debug and not load_pickle:
with open(f"../input/feature_engineering/model200/train.pickle",
"wb") as f:
pickle.dump(dataset_train, f)
with open(f"../input/feature_engineering/model200/val.pickle",
"wb") as f:
pickle.dump(dataset_val, f)
if not is_debug and load_pickle:
with open(f"../input/feature_engineering/model200/train.pickle",
"rb") as f:
dataset_train = pickle.load(f)
with open(f"../input/feature_engineering/model200/val.pickle",
"rb") as f:
dataset_val = pickle.load(f)
print("loaded!")
dataloader_train = DataLoader(dataset_train,
batch_size=params["batch_size"],
shuffle=True,
num_workers=1)
dataloader_val = DataLoader(dataset_val,
batch_size=params["batch_size"],
shuffle=False,
num_workers=1)
model = SAKTModel(n_skill=60000,
embed_dim=params["embed_dim"],
max_seq=params["max_seq"],
dropout=dropout,
cont_emb=params["cont_emb"])
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = AdaBelief(
optimizer_grouped_parameters,
lr=params["lr"],
)
num_train_optimization_steps = int(len(dataloader_train) * 20)
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=params["num_warmup_steps"],
num_training_steps=num_train_optimization_steps)
criterion = nn.BCEWithLogitsLoss()
model.to(device)
criterion.to(device)
for epoch in range(epochs):
loss, acc, auc, auc_val = train_epoch(model, dataloader_train,
dataloader_val, optimizer,
criterion, scheduler, epoch,
device)
print("epoch - {} train_loss - {:.3f} auc - {:.4f} auc-val: {:.4f}".
format(epoch, loss, auc, auc_val))
torch.save(
model.state_dict(),
f"{output_dir}/transformers_epoch{epoch}_auc{round(auc_val, 4)}.pth"
)
preds = []
labels = []
with torch.no_grad():
for item in tqdm(dataloader_val):
label = item["label"].to(device).float()
output = model(item, device)
preds.extend(torch.nn.Sigmoid()(
output[:, -1]).view(-1).data.cpu().numpy().tolist())
labels.extend(label[:, -1].view(-1).data.cpu().numpy().tolist())
auc_transformer = roc_auc_score(labels, preds)
print("single transformer: {:.4f}".format(auc_transformer))
df_oof = pd.DataFrame()
# df_oof["row_id"] = df.loc[val_idx].index
print(len(dataloader_val))
print(len(preds))
df_oof["predict"] = preds
df_oof["target"] = labels
df_oof.to_csv(f"{output_dir}/transformers1.csv", index=False)
"""
df_oof2 = pd.read_csv("../output/ex_237/20201213110353/oof_train_0_lgbm.csv")
df_oof2.columns = ["row_id", "predict_lgbm", "target"]
df_oof2 = pd.merge(df_oof, df_oof2, how="inner")
auc_lgbm = roc_auc_score(df_oof2["target"].values, df_oof2["predict_lgbm"].values)
print("lgbm: {:.4f}".format(auc_lgbm))
print("ensemble")
max_auc = 0
max_nn_ratio = 0
for r in np.arange(0, 1.05, 0.05):
auc = roc_auc_score(df_oof2["target"].values, df_oof2["predict_lgbm"].values*(1-r) + df_oof2["predict"].values*r)
print("[nn_ratio: {:.2f}] AUC: {:.4f}".format(r, auc))
if max_auc < auc:
max_auc = auc
max_nn_ratio = r
print(len(df_oof2))
"""
if not is_debug:
mlflow.start_run(experiment_id=10, run_name=os.path.basename(__file__))
for key, value in params.items():
mlflow.log_param(key, value)
mlflow.log_metric("auc_val", auc_transformer)
mlflow.end_run()
torch.save(model.state_dict(), f"{output_dir}/transformers.pth")
del model
torch.cuda.empty_cache()
with open(f"{output_dir}/transformer_param.json", "w") as f:
json.dump(params, f)
weight_decay=args.weight_decay)
logging.info("Training classifier")
for epoch in trange(args.epochs):
model.train()
running_loss = []
for idx, batch in enumerate(tqdm(train_loader)):
optimizer.zero_grad()
data = batch["data"].to(device)
target = batch["target"].to(device)
output = model(data)
loss_value = loss(output, target)
loss_value.backward()
optimizer.step()
running_loss.append(loss_value.item())
mlflow.log_metric("train_loss",
sum(running_loss) / len(running_loss), epoch)
if validation_dataset:
logging.info("Evaluating model on validation")
model.eval()
running_loss = []
targets = []
predictions = []
with torch.no_grad():
for batch in tqdm(validation_loader):
data = batch["data"].to(device)
target = batch["target"].to(device)
output = model(data)
running_loss.append(loss(output, target).item())
targets.extend(batch["target"].numpy())
predictions.extend(
def train(ngram, nb_hash, word2vec, maxiter, oversampling):
maxIter = maxiter
warnings.filterwarnings("ignore")
np.random.seed(40)
spark = SparkSession.builder \
.master("local[*]") \
.appName("ML") \
.getOrCreate()
#wine_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),'data/cdiscount_train.csv')
RowDF = spark.read.format('com.databricks.spark.csv').options(
header='true', inferschema='true',
sep=",").load('data/operation bancaire.csv')
train = RowDF.dropna(subset='lib4')
print("nbr de classe", train.select(train.columns[4]).distinct().count())
"""
df = train.fillna(0,subset='credit')
def trasf(ligne):
if ligne > 0:
return 1
else :
return 0
udftrasform = udf(lambda x: trasf(x),)
dataClean = df.withColumn("credit_oui_non", udftrasform(df['credit']))
dataClean.show(n=20,truncate=True)
"""
df = train.select('lib1',
train.credit.isNull().cast('float').alias('credit_o_n'),
'lib4')
#train.show(22,truncate=True)
# Taux de sous-échantillonnage des données pour tester le programme de préparation
# sur un petit jeu de données
taux_donnees = [0.7, 0.3]
dataTrain, DataTest = df.randomSplit(taux_donnees, seed=42)
n_train = dataTrain.count()
n_test = DataTest.count()
print("DataTrain : size = %d, DataTest : size = %d" % (n_train, n_test))
if oversampling:
def sur_echant(df, p=0.7):
counts = df.groupBy('lib4').count().collect()
categories = [i[0] for i in counts]
values = [i[1] for i in counts]
max_v = max(values)
indx = values.index(max_v)
dic = {j: 1 for i, j in enumerate(categories) if i != indx}
dic[categories[indx]] = p
df = df.sampleBy("lib4", fractions=dic)
p = int(max_v * p)
for i, cat in enumerate(categories):
if i != indx:
a = df
data = a.sampleBy("lib4", fractions={
cat: 1
}).toPandas().sample(p - values[i], replace=True)
spark_df = spark.createDataFrame(data)
df = df.union(spark_df)
return df
dataTrain = sur_echant(dataTrain, p=0.8)
print("DataTrain apres le sur_echantillonage: size = %d" %
(dataTrain.count()))
opm = "mp"
with mlflow.start_run():
pipeline = Pipeline_model(ngram,
nb_hash,
data=df,
opm=opm,
vec=word2vec,
maxIter=maxIter)
time_start = time.time()
# On applique toutes les étapes sur la DataFrame d'apprentissage.
model = pipeline.fit(dataTrain)
time_end = time.time()
time_lrm = (time_end - time_start)
print(
"LR prend %d s pour un echantillon d'apprentissage de taille : n = %d"
% (time_lrm, n_train))
predictionsDF = model.transform(DataTest)
#labelsAndPredictions = predictionsDF.select("categoryIndex","prediction").collect()
#nb_good_prediction = sum([r[0]==r[1] for r in labelsAndPredictions])
#testScore =nb_good_prediction/n_test
#print('Test score = , pour un echantillon test de taille n = %d' + str(testScore))
# Select (prediction, true label) and compute test error
#evaluator1 = MulticlassClassificationEvaluator(labelCol="categoryIndex", predictionCol="prediction", metricName="accuracy")
#accuracy = evaluator1.evaluate(predictionsDF)
evaluator = MulticlassClassificationEvaluator(
labelCol="categoryIndex", predictionCol="prediction")
f1 = evaluator.evaluate(predictionsDF, {evaluator.metricName: "f1"})
accuracy = evaluator.evaluate(predictionsDF,
{evaluator.metricName: "accuracy"})
print("Test scor accuracy= %g" % (accuracy))
print("Test scor f1score = %g" % (f1))
mlflow.log_param("word2vec", word2vec)
mlflow.log_param("ngram", ngram)
mlflow.log_param("nb_hash", nb_hash)
mlflow.log_param("maxIter", maxIter)
mlflow.log_param("oversampling", oversampling)
mlflow.log_metric("accuracy", accuracy)
mlflow.log_metric("f1Score", f1)
mlflow.spark.log_model(model, "spark-model")
def main(params: dict):
print("start params={}".format(params))
df = pd.read_pickle(
"../input/riiid-test-answer-prediction/split10/train_0.pickle"
).sort_values(["user_id", "timestamp"]).reset_index(drop=True)
if is_debug:
df = df.head(10000)
df = df[df.content_type_id == False]
train_idx = []
val_idx = []
np.random.seed(0)
for _, w_df in df.groupby("user_id"):
if np.random.random() < 0.1:
# all val
val_idx.extend(w_df.index.tolist())
else:
train_num = int(len(w_df) * 0.9)
train_idx.extend(w_df[:train_num].index.tolist())
val_idx.extend(w_df[train_num:].index.tolist())
df["is_val"] = 0
df["is_val"].loc[val_idx] = 1
df["group"] = (df.groupby("user_id")["user_id"].transform("count") -
df.groupby("user_id").cumcount()) // max_len
print(df["group"].value_counts())
print(len(train_idx))
print(len(val_idx))
group = df[[
'user_id', 'content_id', 'answered_correctly', "group", "part",
'is_val'
]].groupby([
'user_id', "group"
]).apply(lambda r: (r['content_id'].values, r['part'].values, r[
'answered_correctly'].values, r["is_val"].values))
dataset_train = SAKTDataset(group, 13523, max_seq=params["max_seq"])
group = df[[
'user_id', 'content_id', 'answered_correctly', "part", 'is_val'
]].groupby(['user_id']).apply(lambda r: (r['content_id'].values, r[
'part'].values, r['answered_correctly'].values, r["is_val"].values))
dataset_val = SAKTDataset(group,
13523,
is_test=True,
max_seq=params["max_seq"])
dataloader_train = DataLoader(dataset_train,
batch_size=1024,
shuffle=True,
num_workers=1)
dataloader_val = DataLoader(dataset_val,
batch_size=1024,
shuffle=False,
num_workers=1)
device = torch.device("cuda")
model = SAKTModel(13523,
embed_dim=params["embed_dim"],
max_seq=params["max_seq"])
# optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.99, weight_decay=0.005)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
criterion = nn.BCEWithLogitsLoss()
model.to(device)
criterion.to(device)
epochs = 20
for epoch in range(epochs):
loss, acc, auc, auc_val = train_epoch(model, dataloader_train,
dataloader_val, optimizer,
criterion, device)
print("epoch - {} train_loss - {:.3f} auc - {:.4f} auc-val: {:.4f}".
format(epoch, loss, auc, auc_val))
preds = []
labels = []
for d in tqdm(dataloader_val):
x = d[0].to(device).long()
target_id = d[1].to(device).long()
part = d[2].to(device).long()
label = d[3].to(device).long()
output, atten_weight = model(x, target_id, part)
preds.extend(torch.nn.Sigmoid()(
output[:, -1]).view(-1).data.cpu().numpy().tolist())
labels.extend(label[:, -1].view(-1).data.cpu().numpy())
df_oof = pd.DataFrame()
df_oof["row_id"] = df.loc[val_idx].index
df_oof["predict"] = preds
df_oof["target"] = df.loc[val_idx]["answered_correctly"].values
df_oof.to_csv(f"{output_dir}/transformers1.csv", index=False)
df_oof2 = pd.read_csv(
"../output/ex_172/20201202080625/oof_train_0_lgbm.csv")
df_oof2.columns = ["row_id", "predict_lgbm", "target"]
df_oof2 = pd.merge(df_oof, df_oof2, how="inner")
auc_transformer = roc_auc_score(df_oof2["target"].values,
df_oof2["predict"].values)
auc_lgbm = roc_auc_score(df_oof2["target"].values,
df_oof2["predict_lgbm"].values)
print("single transformer: {:.4f}".format(auc_transformer))
print("lgbm: {:.4f}".format(auc_lgbm))
print("ensemble")
max_auc = 0
max_nn_ratio = 0
for r in np.arange(0, 1.05, 0.05):
auc = roc_auc_score(
df_oof2["target"].values, df_oof2["predict_lgbm"].values *
(1 - r) + df_oof2["predict"].values * r)
print("[nn_ratio: {:.2f}] AUC: {:.4f}".format(r, auc))
if max_auc < auc:
max_auc = auc
max_nn_ratio = r
print(len(df_oof2))
if not is_debug:
mlflow.start_run(experiment_id=10, run_name=os.path.basename(__file__))
mlflow.log_param("count_row", len(df))
for key, value in params.items():
mlflow.log_param(key, value)
mlflow.log_metric("auc_val", auc_transformer)
mlflow.log_metric("auc_lgbm", auc_lgbm)
mlflow.log_metric("auc_ensemble", max_auc)
mlflow.log_metric("ensemble_nn_ratio", max_nn_ratio)
mlflow.end_run()
def embeddings_quality(embeddings_file, configuration):
"""Function used to test the quality of the embeddings provided in file 'embeddings_file'. The tests must already
be provided in the directory 'test_dir' to be executed. Please refer to the readme for info about the test format.
:param embeddings_file: path to the embeddings file to be tested
:param test_dir: path to the directory that contains all the tests.
"""
print('# Executing EQ tests.')
test_dir = configuration['test_dir']
test_dir = test_dir.strip('/') + '/'
if configuration['training_algorithm'] == 'fasttext':
wv = models.KeyedVectors.load(embeddings_file, mmap='r')
else:
wv = models.KeyedVectors.load_word2vec_format(embeddings_file, unicode_errors='ignore')
sum_total = 0
count_tests = 0
result_col = {}
result_con = {}
result_row = {}
nmr_tests = []
nmcon_tests = []
nmc_tests = []
for fin in os.listdir(test_dir):
if fin.startswith('nmr'):
nmr_tests.append(test_dir + fin)
elif fin.startswith('nmcon'):
nmcon_tests.append(test_dir + fin)
elif fin.startswith('nmc'):
nmc_tests.append(test_dir + fin)
if len(nmc_tests) == len(nmcon_tests) == len(nmr_tests) == 0:
raise ValueError('No valid test files found. Exiting. ')
if len(nmc_tests) > 0:
print('# Testing columns.')
result_col = _test_no_match_columns(wv, nmc_tests)
sum_total += result_col['nmc_avg']
count_tests += 1
print('# nmc_avg: {:.2f}'.format(result_col['nmc_avg']))
else:
warnings.warn('No valid nmc tests found. ')
if len(nmr_tests) > 0:
print('# Testing rows.')
result_row = _test_no_match_rows(wv, nmr_tests)
sum_total += result_row['nmr_avg']
count_tests += 1
print('# nmr_avg: {:.2f}'.format(result_row['nmr_avg']))
else:
warnings.warn('No valid nmr tests found. ')
if len(nmcon_tests) > 0:
print('# Testing concepts.')
result_con = _test_no_match_concept(wv, nmcon_tests)
sum_total += result_con['nmcon_avg']
count_tests += 1
print('# nmcon_avg: {:.2f}'.format(result_con['nmcon_avg']))
else:
warnings.warn('No valid nmcon tests found. ')
try:
avg_results = sum_total / count_tests
except ZeroDivisionError:
print('No tests were executed.')
avg_results = 0
print('# EQ average: {:.2f}'.format(avg_results))
if configuration['mlflow']:
with mlflow.active_run():
mlflow.log_metric('eq_avg', avg_results)
mlflow.log_metric('nmc_avg', result_col['nmc_avg'])
mlflow.log_metric('nmcon_avg', result_con['nmcon_avg'])
mlflow.log_metric('nmr_avg', result_row['nmr_avg'])
for k in result_col:
mlflow.log_metric(k, result_col[k])
for k in result_row:
mlflow.log_metric(k, result_row[k])
for k in result_con:
mlflow.log_metric(k, result_con[k])
result_dict = dict(chain.from_iterable(d.items() for d in (result_row, result_col, result_con)))
_r = ['nmc_avg', 'nmr_avg', 'nmcon_avg', 'eq_avg']
result_dict['eq_avg'] = avg_results
print('\t'.join(_r))
for k in _r:
print(result_dict[k], end='\t')
print()
return result_dict
def run(epochs, lr):
"""
Train and test CNN with given parameters
"""
# ==============
# Load Data
# ==============
mixdata = h5py.File(
"../train/scsn_p_2000_2017_6sec_0.5r_pick_train_mix.hdf5", "r")
testdata = h5py.File(
"../test/scsn_p_2000_2017_6sec_0.5r_pick_test_mix.hdf5", "r")
batch_size = 500
train_size = 1 * 10**5
train_ratio = 0.5
test_size = 1 * 10**5
train_val_data = mixdata["X"][:train_size]
train_val_labels = mixdata["pwave"][:train_size]
(trainset, trainlabels), (valset, val_labels) = split_trainset(
train_val_data, train_val_labels, train_ratio)
trainset = list(zip(trainset, trainlabels))
valset = list(zip(valset, val_labels))
trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(valset, batch_size=batch_size, shuffle=True)
# ======================
# Use GPU, if available
# ======================
if torch.cuda.is_available():
device = "cuda"
else:
device = "cpu"
model = CNN()
model = parallelize(model)
model.to(device)
#lr = float(args.lr)
# ===================================
# Define Optimizer and Loss Function
# ===================================
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr=lr)
mlflow.set_tracking_uri("file:.\mlruns")
mlflow.start_run()
#epochs = int(args.epochs)
train_losses = []
val_losses = []
min_val_loss = float("inf")
for epoch in range(epochs):
model.train()
train_loss = 0
for batch, labels in trainloader:
# ============================================
# TRAINING
# ============================================
batch, labels = batch.to(device), labels.to(device)
# Clear gradients in optimizer
optimizer.zero_grad()
# Forward pass
output = model.forward(batch)
# Calculate loss
loss = criterion(
output,
labels.type(torch.cuda.LongTensor).view(labels.shape, 1))
train_loss += loss.item()
# Backpropagation
loss.backward()
# Update weights
optimizer.step()
else:
with torch.no_grad():
model.eval()
val_loss = 0
for batch, labels in val_loader:
# ============================================
# VALIDATION
# ============================================
batch, labels = batch.to(device), labels.to(device)
# Forward pass
ouput = model.forward(batch)
# Calculate loss
loss = criterion(
output,
labels.type(torch.cuda.LongTensor).view(
labels.shape, 1))
val_loss += loss.item()
# Print epoch summary
t_loss_avg = train_loss / len(trainloader)
v_loss_avg = val_loss / len(val_loader)
if v_loss_avg < min_val_loss:
torch.save(model.state_dict(), "./artifacts/model.pth")
mlflow.log_artifact("./artifacts/model.pth")
mlflow.log_metric("train_loss", t_loss_avg)
mlflow.log_metric("val_loss", v_loss_avg)
train_losses.append(t_loss_avg)
val_losses.append(v_loss_avg)
print(
'Epoch [{:5d}/{:5d}] | train loss: {:6.4f} | validation loss: {:6.4f}'
.format(epoch + 1, epochs, t_loss_avg, v_loss_avg))
# ==============
# Test model
# ==============
model.load_state_dict(torch.load("./artifacts/model.pth"))
test_path = "../test/scsn_p_2000_2017_6sec_0.5r_pick_test_mix.hdf5"
y_true, y_pred, y_probs = test_model(model,
test_path,
test_size,
device="cuda")
report = classification_report(y_true,
y_pred,
target_names=["pwave", "noise"])
report_dict = classification_report(y_true,
y_pred,
target_names=["pwave", "noise"],
output_dict=True)
accuracy = accuracy_score(y_true, y_pred)
roc_score = roc_auc_score(y_true, y_probs)
print(report)
print("Accuracy: {:.4}%".format(accuracy * 100))
print("ROC Score: ", roc_score)
mlflow.log_param("epochs", epochs)
mlflow.log_param("learning_rate", lr)
mlflow.log_param("device", device)
mlflow.log_metric("accuracy", accuracy)
mlflow.log_metric("auc_roc_score", roc_score)
for category in report_dict:
for metric, value in report_dict[category].items():
metric_name = category + "_" + metric
mlflow.log_metric(metric_name, value)
mlflow.end_run()
graph.write_png(local_image_path)
# Load Data
iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(iris.data,
iris.target,
test_size=0.2,
random_state=RANDOM_STATE)
# Set MLflow tags
mlflow.set_tags({'platform': 'local-mlrun'})
# Log params
max_depth = int(sys.argv[1]) if len(sys.argv) > 1 else 1
# Model training
model = DecisionTreeClassifier(max_depth=max_depth)
model.fit(X_train, y_train)
# Log model
mlflow.sklearn.log_model(model, "model")
# Log metrics
accuracy = model.score(X_test, y_test)
mlflow.log_metric("accuracy", accuracy)
# Log artifact
export_tree(model, TREE_IMAGE_PATH)
mlflow.log_artifact(TREE_IMAGE_PATH)
import mlflow
import os
from mlflow.sklearn import save_model, log_model
mlflow.set_tracking_uri(os.environ["MLFLOW_HOST"])
mlflow.set_experiment("iris-exp")
# Injest data
iris = load_iris()
# Prepare training data
X, y = iris.data, iris.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
with mlflow.start_run():
# Train model
n_estimators = 150
mlflow.log_param("n_estimators", n_estimators)
clf = RandomForestClassifier(n_estimators=n_estimators)
clf.fit(X_train, y_train)
# Evaluate
y_pred = clf.predict(X_test)
score = metrics.accuracy_score(y_test, y_pred)
print(f"Accuracy: {score}")
mlflow.log_metric(key="accuracy", value=score)
# Save model
# save_model(clf, 'iris_model') uncomment if you want to save localy
log_model(clf, 'iris_model')
def log_metrics(metrics):
for k, values in metrics.items():
for v in values:
mlflow.log_metric(k, v)
def main():
os.environ['MLFLOW_TRACKING_URI'] = "http://localhost:5000"
os.environ['MLFLOW_TRACKING_USERNAME'] = "******"
os.environ['MLFLOW_TRACKING_PASSWORD'] = "******"
mlflow.create_experiment("diabetes2", artifact_location="../tmp/artifacts")
mlflow.set_experiment("diabetes2")
mlflow.start_run(run_name=None)
# Load Diabetes datasets
diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target
# Create pandas DataFrame for sklearn ElasticNet linear_model
Y = np.array([y]).transpose()
d = np.concatenate((X, Y), axis=1)
cols = diabetes.feature_names + ["progression"]
data = pd.DataFrame(d, columns=cols)
np.random.seed(40)
# Split the data into training and test sets. (0.75, 0.25) split.
train, test = train_test_split(data)
# The predicted column is "progression" which is a quantitative
# measure of disease progression one year after baseline
train_x = train.drop(["progression"], axis=1)
test_x = test.drop(["progression"], axis=1)
train_y = train[["progression"]]
test_y = test[["progression"]]
alpha = float(sys.argv[1]) if len(sys.argv) > 1 else 0.05
l1_ratio = float(sys.argv[2]) if len(sys.argv) > 2 else 0.05
# Run ElasticNet
lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42)
lr.fit(train_x, train_y)
predicted_qualities = lr.predict(test_x)
(rmse, mae, r2) = eval_metrics(test_y, predicted_qualities)
# Print out ElasticNet model metrics
print("Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio))
print(" RMSE: %s" % rmse)
print(" MAE: %s" % mae)
print(" R2: %s" % r2)
# Log mlflow attributes for mlflow UI
mlflow.log_param("alpha", alpha)
mlflow.log_param("l1_ratio", l1_ratio)
mlflow.log_metric("rmse", rmse)
mlflow.log_metric("r2", r2)
mlflow.log_metric("mae", mae)
mlflow.sklearn.log_model(lr, "model")
# Compute paths
eps = 5e-3 # the smaller it is the longer is the path
print("Computing regularization path using the elastic net.")
alphas_enet, coefs_enet, _ = enet_path(X,
y,
eps=eps,
l1_ratio=l1_ratio,
fit_intercept=False)
# Display results
fig = plt.figure(1)
plt.gca()
colors = cycle(["b", "r", "g", "c", "k"])
neg_log_alphas_enet = -np.log10(alphas_enet)
for coef_e, c in zip(coefs_enet, colors):
l2 = plt.plot(neg_log_alphas_enet, coef_e, linestyle="--", c=c)
plt.xlabel("-Log(alpha)")
plt.ylabel("coefficients")
title = "ElasticNet Path by alpha for l1_ratio = " + str(l1_ratio)
plt.title(title)
plt.axis("tight")
# Save figures
fig.savefig("ElasticNet-paths.png")
# Close plot
plt.close(fig)
# Log artifacts (output files)
mlflow.log_artifact("ElasticNet-paths.png")
import os
from random import random, randint
import mlflow
if __name__ == "__main__":
print("Running the test script ...")
mlflow.set_tracking_uri("http://127.0.0.1:5000")
if not os.path.exists("artifact_folder"):
os.makedirs("artifact_folder")
mlflow.log_param("param1", randint(0, 100))
mlflow.log_metric("foo", random())
mlflow.log_metric("foo", random() + 1)
mlflow.log_metric("foo", random() + 2)
with open("artifact_folder/test.txt", "w") as f:
f.write("hello world!")
mlflow.log_artifacts("artifact_folder")
def main(params: dict, output_dir: str):
import mlflow
print("start params={}".format(params))
model_id = "train_0"
logger = get_logger()
# df = pd.read_pickle("../input/riiid-test-answer-prediction/train_merged.pickle")
df = pd.read_pickle(
"../input/riiid-test-answer-prediction/split10/train_0.pickle"
).sort_values(["user_id", "timestamp"]).reset_index(drop=True)
if is_debug:
df = df.head(30000)
df["prior_question_had_explanation"] = df[
"prior_question_had_explanation"].fillna(-1)
column_config = {
("content_id", "content_type_id"): {
"type": "category"
},
"user_answer": {
"type": "leakage_feature"
},
"answered_correctly": {
"type": "leakage_feature"
},
"part": {
"type": "category"
},
"prior_question_elapsed_time_bin300": {
"type": "category"
},
"duration_previous_content_bin300": {
"type": "category"
},
"prior_question_had_explanation": {
"type": "category"
},
"rating_diff_content_user_id": {
"type": "numeric"
},
"task_container_id_bin300": {
"type": "category"
}
}
if not load_pickle or is_debug:
feature_factory_dict = {"user_id": {}}
feature_factory_dict["user_id"][
"DurationPreviousContent"] = DurationPreviousContent()
feature_factory_dict["user_id"][
"ElapsedTimeBinningEncoder"] = ElapsedTimeBinningEncoder()
feature_factory_dict["user_id"][
"UserContentRateEncoder"] = UserContentRateEncoder(
rate_func="elo", column="user_id")
feature_factory_manager = FeatureFactoryManager(
feature_factory_dict=feature_factory_dict,
logger=logger,
split_num=1,
model_id="train_0",
load_feature=not is_debug,
save_feature=not is_debug)
print("all_predict")
df = feature_factory_manager.all_predict(df)
df["task_container_id_bin300"] = [
x if x < 300 else 300 for x in df["task_container_id"]
]
df = df[[
"user_id", "content_id", "content_type_id", "part", "user_answer",
"answered_correctly", "prior_question_elapsed_time_bin300",
"duration_previous_content_bin300",
"prior_question_had_explanation", "rating_diff_content_user_id",
"task_container_id_bin300"
]]
print(df.head(10))
print("data preprocess")
train_idx = []
val_idx = []
np.random.seed(0)
for _, w_df in df[df["content_type_id"] == 0].groupby("user_id"):
if np.random.random() < 0.01:
# all val
val_idx.extend(w_df.index.tolist())
else:
train_num = int(len(w_df) * 0.95)
train_idx.extend(w_df[:train_num].index.tolist())
val_idx.extend(w_df[train_num:].index.tolist())
ff_for_transformer = FeatureFactoryForTransformer(
column_config=column_config,
dict_path="../feature_engineering/",
sequence_length=params["max_seq"],
logger=logger)
ff_for_transformer.make_dict(df=df)
n_skill = len(ff_for_transformer.embbed_dict[("content_id",
"content_type_id")])
if not load_pickle or is_debug:
df["is_val"] = 0
df["is_val"].loc[val_idx] = 1
w_df = df[df["is_val"] == 0]
w_df["group"] = (
w_df.groupby("user_id")["user_id"].transform("count") -
w_df.groupby("user_id").cumcount()) // params["max_seq"]
w_df["user_id"] = w_df["user_id"].astype(
str) + "_" + w_df["group"].astype(str)
group = ff_for_transformer.all_predict(w_df)
dataset_train = SAKTDataset(group,
n_skill=n_skill,
max_seq=params["max_seq"])
del w_df
gc.collect()
ff_for_transformer = FeatureFactoryForTransformer(
column_config=column_config,
dict_path="../feature_engineering/",
sequence_length=params["max_seq"],
logger=logger)
if not load_pickle or is_debug:
group = ff_for_transformer.all_predict(df[df["content_type_id"] == 0])
dataset_val = SAKTDataset(group,
is_test=True,
n_skill=n_skill,
max_seq=params["max_seq"])
os.makedirs("../input/feature_engineering/model142", exist_ok=True)
if not is_debug and not load_pickle:
with open(f"../input/feature_engineering/model142/train.pickle",
"wb") as f:
pickle.dump(dataset_train, f)
with open(f"../input/feature_engineering/model142/val.pickle",
"wb") as f:
pickle.dump(dataset_val, f)
if not is_debug and load_pickle:
with open(f"../input/feature_engineering/model142/train.pickle",
"rb") as f:
dataset_train = pickle.load(f)
with open(f"../input/feature_engineering/model142/val.pickle",
"rb") as f:
dataset_val = pickle.load(f)
print("loaded!")
dataloader_train = DataLoader(dataset_train,
batch_size=params["batch_size"],
shuffle=True,
num_workers=1)
dataloader_val = DataLoader(dataset_val,
batch_size=params["batch_size"],
shuffle=False,
num_workers=1)
model = SAKTModel(n_skill,
embed_dim=params["embed_dim"],
max_seq=params["max_seq"],
dropout=dropout)
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = AdamW(
optimizer_grouped_parameters,
lr=params["lr"],
weight_decay=0.01,
)
num_train_optimization_steps = int(len(dataloader_train) * epochs)
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=params["num_warmup_steps"],
num_training_steps=num_train_optimization_steps)
criterion = nn.BCEWithLogitsLoss()
model.to(device)
criterion.to(device)
for epoch in range(epochs):
loss, acc, auc, auc_val = train_epoch(model, dataloader_train,
dataloader_val, optimizer,
criterion, scheduler, device)
print("epoch - {} train_loss - {:.3f} auc - {:.4f} auc-val: {:.4f}".
format(epoch, loss, auc, auc_val))
preds = []
labels = []
with torch.no_grad():
for item in tqdm(dataloader_val):
label = item["label"].to(device).float()
output = model(item, device)
preds.extend(torch.nn.Sigmoid()(
output[:, -1]).view(-1).data.cpu().numpy().tolist())
labels.extend(label[:, -1].view(-1).data.cpu().numpy().tolist())
auc_transformer = roc_auc_score(labels, preds)
print("single transformer: {:.4f}".format(auc_transformer))
df_oof = pd.DataFrame()
# df_oof["row_id"] = df.loc[val_idx].index
print(len(dataloader_val))
print(len(preds))
df_oof["predict"] = preds
df_oof["target"] = labels
df_oof.to_csv(f"{output_dir}/transformers1.csv", index=False)
"""
df_oof2 = pd.read_csv("../output/ex_237/20201213110353/oof_train_0_lgbm.csv")
df_oof2.columns = ["row_id", "predict_lgbm", "target"]
df_oof2 = pd.merge(df_oof, df_oof2, how="inner")
auc_lgbm = roc_auc_score(df_oof2["target"].values, df_oof2["predict_lgbm"].values)
print("lgbm: {:.4f}".format(auc_lgbm))
print("ensemble")
max_auc = 0
max_nn_ratio = 0
for r in np.arange(0, 1.05, 0.05):
auc = roc_auc_score(df_oof2["target"].values, df_oof2["predict_lgbm"].values*(1-r) + df_oof2["predict"].values*r)
print("[nn_ratio: {:.2f}] AUC: {:.4f}".format(r, auc))
if max_auc < auc:
max_auc = auc
max_nn_ratio = r
print(len(df_oof2))
"""
if not is_debug:
mlflow.start_run(experiment_id=10, run_name=os.path.basename(__file__))
for key, value in params.items():
mlflow.log_param(key, value)
mlflow.log_metric("auc_val", auc_transformer)
mlflow.end_run()
torch.save(model.state_dict(), f"{output_dir}/transformers.pth")
del model
torch.cuda.empty_cache()
with open(f"{output_dir}/transformer_param.json", "w") as f:
json.dump(params, f)
if is_make_feature_factory:
# feature factory
feature_factory_dict = {"user_id": {}}
feature_factory_dict["user_id"][
"DurationPreviousContent"] = DurationPreviousContent(
is_partial_fit=True)
feature_factory_dict["user_id"][
"ElapsedTimeBinningEncoder"] = ElapsedTimeBinningEncoder()
feature_factory_manager = FeatureFactoryManager(
feature_factory_dict=feature_factory_dict,
logger=logger,
split_num=1,
model_id="all",
load_feature=not is_debug,
save_feature=not is_debug)
ff_for_transformer = FeatureFactoryForTransformer(
column_config=column_config,
dict_path="../feature_engineering/",
sequence_length=params["max_seq"],
logger=logger)
df = pd.read_pickle(
"../input/riiid-test-answer-prediction/train_merged.pickle")
if is_debug:
df = df.head(10000)
df = df.sort_values(["user_id", "timestamp"]).reset_index(drop=True)
feature_factory_manager.fit(df)
df = feature_factory_manager.all_predict(df)
for dicts in feature_factory_manager.feature_factory_dict.values():
for factory in dicts.values():
factory.logger = None
feature_factory_manager.logger = None
with open(f"{output_dir}/feature_factory_manager.pickle", "wb") as f:
pickle.dump(feature_factory_manager, f)
ff_for_transformer.fit(df)
ff_for_transformer.logger = None
with open(
f"{output_dir}/feature_factory_manager_for_transformer.pickle",
"wb") as f:
pickle.dump(ff_for_transformer, f)
import mlflow
import os
import time
from mlflow import log_metric, log_param, log_artifact
if __name__ == "__main__":
mlflow.set_experiment("First")
with mlflow.start_run():
# Log a parameter (key-value pair)
log_param("param1", 5)
# Log a metric; metrics can be updated throughout the run
for i in range(200):
time.sleep(0.1)
log_metric("foo1", 1 * i)
log_metric("foo2", 2 * i)
log_metric("foo3", 3 * i)
log_metric("foo4", 3 * i)
log_metric("foo5", 3 * i)
log_metric("foo6", 3 * i)
log_metric("foo7", 3 * i)
log_metric("foo8", 3 * i)
log_metric("foo9", 3 * i)
log_metric("foo10", 3 * i)
log_metric("foo11", 3 * i)
log_metric("foo12", 3 * i)
log_metric("foo13", 3 * i)
log_metric("foo14", 3 * i)
log_metric("foo15", 3 * i)
log_metric("foo16", 3 * i)
def train_torch_KFold(train_df: pd.DataFrame, feat_cols: List[str],
target_cols: List[str], model_name: str,
model_param: DictConfig, train_param: DictConfig,
cv_param: DictConfig, optimizer: DictConfig,
scheduler: DictConfig, loss_function: DictConfig,
OUT_DIR: str) -> None:
'''
1. Create model
2. Split training data into folds
3. Train model
4. Calculate validation metrics
5. Calculate average validation metrics
'''
# store scores in schema: {'train-acc': {'fold': [0.1, 0.8, ...], 'avg': 0.005}, ...}
metrics = ['train-acc', 'valid-acc', 'train-auc', 'valid-auc']
scores: dict = {}
for metric in metrics:
scores[metric] = {'vsfold': [], 'avg': None}
device = get_device()
target = target_cols[0]
kf = KFold(**cv_param)
for fold, (tr, te) in enumerate(
kf.split(train_df[target].values, train_df[target].values)):
print(
f'Starting fold: {fold}, train size: {len(tr)}, validation size: {len(te)}'
)
# split data
train = train_df.loc[tr, :]
valid = train_df.loc[te, :]
y_tr = train_df.loc[tr, target]
y_val = train_df.loc[te, target]
# create dataset
train_set = TitanicDataset(train, feat_cols, target_cols)
train_loader = DataLoader(train_set,
batch_size=train_param.batch_size,
shuffle=True,
num_workers=4)
valid_set = TitanicDataset(valid, feat_cols, target_cols)
valid_loader = DataLoader(valid_set,
batch_size=train_param.batch_size,
shuffle=False,
num_workers=4)
torch.cuda.empty_cache()
model = get_model(model_name,
model_param,
optimizer,
scheduler,
loss_function,
feat_cols=feat_cols,
target_cols=target_cols,
fold=fold,
device=device)
early_stop_callback = EarlyStopping(
'val-loss_vsepoch',
patience=train_param.early_stopping_rounds,
mode='max')
trainer = pl.Trainer(
max_epochs=train_param.epochs,
fast_dev_run=False, # TODO: pass from param!
callbacks=[early_stop_callback],
)
trainer.fit(model, train_loader, valid_loader)
# log model
torch.save(model.state_dict(), f'{OUT_DIR}/model_{fold}.pth')
# log metrics per fold
pred_tr = inference_fn(model, train_loader, device, target_cols)
pred_val = inference_fn(model, valid_loader, device, target_cols)
pred_tr = np.where(pred_tr >= 0.5, 1, 0).astype(int)
pred_val = np.where(pred_val >= 0.5, 1, 0).astype(int)
score = {
metrics[0]: accuracy_score(y_tr, pred_tr),
metrics[1]: accuracy_score(y_val, pred_val),
metrics[2]: roc_auc_score(y_tr, pred_tr),
metrics[3]: roc_auc_score(y_val, pred_val)
}
for metric in metrics:
scores[metric]['vsfold'].append(score[metric])
mlflow.log_metric(f'{metric}_vsfold', score[metric], step=fold)
# log metrics averaged over folds
for metric in metrics:
scores[metric]['avg'] = np.array(scores[metric]['vsfold']).mean()
mlflow.log_metric(f'{metric}_foldavg', scores[metric]['avg'])
print('End training')
return None
