def save_regression(additional_metrics, model_desc, model_path, fold_cnt,
repeat_cnt):
max_metrics = additional_metrics["max_metrics"]
with open(os.path.join(model_path, "README.md"), "w",
encoding="utf-8") as fout:
fout.write(model_desc)
fout.write("\n### Metric details:\n{}\n\n".format(
tabulate(max_metrics.values,
max_metrics.columns,
tablefmt="pipe")))
AdditionalMetrics.add_learning_curves(fout)
AdditionalMetrics.add_tree_viz(fout, model_path, fold_cnt,
repeat_cnt)
AdditionalMetrics.add_linear_coefs(fout, model_path, fold_cnt,
repeat_cnt)
AdditionalMetrics.add_permutation_importance(
fout, model_path, fold_cnt, repeat_cnt)
plots = additional_metrics.get("additional_plots")
if plots is not None:
AdditionalPlots.append(fout, model_path, plots)
AdditionalMetrics.add_shap_importance(fout, model_path, fold_cnt,
repeat_cnt)
AdditionalMetrics.add_shap_regression(fout, model_path, fold_cnt,
repeat_cnt)
fout.write("\n\n[<< Go back](../README.md)\n")
def save_multiclass_classification(additional_metrics, model_desc,
model_path, fold_cnt, repeat_cnt):
max_metrics = additional_metrics["max_metrics"].transpose()
confusion_matrix = additional_metrics["confusion_matrix"]
with open(os.path.join(model_path, "README.md"), "w",
encoding="utf-8") as fout:
fout.write(model_desc)
fout.write("\n### Metric details\n{}\n\n".format(
max_metrics.to_markdown()))
fout.write("\n## Confusion matrix\n{}".format(
confusion_matrix.to_markdown()))
AdditionalMetrics.add_learning_curves(fout)
AdditionalMetrics.add_tree_viz(fout, model_path, fold_cnt,
repeat_cnt)
AdditionalMetrics.add_linear_coefs(fout, model_path, fold_cnt,
repeat_cnt)
AdditionalMetrics.add_permutation_importance(
fout, model_path, fold_cnt, repeat_cnt)
plots = additional_metrics.get("additional_plots")
if plots is not None:
AdditionalPlots.append(fout, model_path, plots)
AdditionalMetrics.add_shap_importance(fout, model_path, fold_cnt,
repeat_cnt)
AdditionalMetrics.add_shap_multiclass(fout, model_path, fold_cnt,
repeat_cnt)
fout.write("\n\n[<< Go back](../README.md)\n")
def regression(target, predictions, sample_weight=None):
regression_metrics = {
"MAE":
mean_absolute_error,
"MSE":
mean_squared_error,
"RMSE":
lambda t, p, sample_weight: np.sqrt(
mean_squared_error(t, p, sample_weight=sample_weight)),
"R2":
r2_score,
"MAPE":
mean_absolute_percentage_error,
}
max_metrics = {}
for k, v in regression_metrics.items():
max_metrics[k] = v(target,
predictions,
sample_weight=sample_weight)
return {
"max_metrics":
pd.DataFrame({
"Metric": list(max_metrics.keys()),
"Score": list(max_metrics.values()),
}),
"additional_plots":
AdditionalPlots.plots_regression(target, predictions),
}
def multiclass_classification(target, predictions, sample_weight=None):
all_labels = [i[11:] for i in predictions.columns.tolist()[:-1]]
predicted_probas = predictions[predictions.columns[:-1]]
ll = logloss(target,
predictions[predictions.columns[:-1]],
sample_weight=sample_weight)
if "target" in target.columns.tolist():
# multiclass coding with integer
labels = {i: l for i, l in enumerate(all_labels)}
target = target["target"].map(labels)
else:
# multiclass coding with one-hot encoding
old_columns = target.columns
t = target[old_columns[0]]
for l in all_labels:
t[target[f"target_{l}"] == 1] = l
target = pd.DataFrame({"target": t})
# Print the confusion matrix
predicted_labels = predictions["label"]
predictions = predictions["label"]
if not pd.api.types.is_string_dtype(predictions):
predictions = predictions.astype(str)
if not pd.api.types.is_string_dtype(target):
target = target.astype(str)
conf_matrix = confusion_matrix(target,
predictions,
labels=all_labels,
sample_weight=sample_weight)
rows = [f"Predicted as {a}" for a in all_labels]
cols = [f"Labeled as {a}" for a in all_labels]
conf_matrix = pd.DataFrame(conf_matrix, columns=rows, index=cols)
max_metrics = classification_report(
target,
predictions,
digits=6,
labels=all_labels,
output_dict=True,
sample_weight=sample_weight,
)
max_metrics["logloss"] = ll
return {
"max_metrics":
pd.DataFrame(max_metrics).transpose(),
"confusion_matrix":
conf_matrix,
"additional_plots":
AdditionalPlots.plots_multiclass(target, predicted_labels,
predicted_probas),
}
def binary_classification(target, predictions, sample_weight=None):
negative_label, positive_label = "0", "1"
mapping = None
try:
pred_col = predictions.columns[0]
if "_0_for_" in pred_col and "_1_for_" in pred_col:
t = pred_col.split("_0_for_")[1]
t = t.split("_1_for_")
negative_label, positive_label = t[0], t[1]
mapping = {0: negative_label, 1: positive_label}
except Exception as e:
pass
predictions = np.array(predictions)
sorted_predictions = np.sort(predictions)
STEPS = 100 # can go lower for speed increase ???
details = {
"threshold": [],
"f1": [],
"accuracy": [],
"precision": [],
"recall": [],
"mcc": [],
}
samples_per_step = max(1, np.floor(predictions.shape[0] / STEPS))
for i in range(STEPS):
idx = int(i * samples_per_step)
if idx + 1 >= predictions.shape[0]:
break
if i == 0:
th = 0.9 * np.min(sorted_predictions)
else:
th = float(
0.5 *
(sorted_predictions[idx] + sorted_predictions[idx + 1]))
if np.sum(predictions > th) < 1:
break
response = (predictions > th).astype(int)
details["threshold"] += [th]
details["f1"] += [
f1_score(target, response, sample_weight=sample_weight)
]
details["accuracy"] += [
accuracy_score(target, response, sample_weight=sample_weight)
]
details["precision"] += [
precision_score(target, response, sample_weight=sample_weight)
]
details["recall"] += [
recall_score(target, response, sample_weight=sample_weight)
]
if i == 0:
details["mcc"] += [0.0]
else:
details["mcc"] += [
matthews_corrcoef(target,
response,
sample_weight=sample_weight)
]
# max metrics
max_metrics = {
"logloss": {
"score": logloss(target,
predictions,
sample_weight=sample_weight),
"threshold": None,
}, # there is no threshold for LogLoss
"auc": {
"score":
roc_auc_score(target, predictions,
sample_weight=sample_weight),
"threshold":
None,
}, # there is no threshold for AUC
"f1": {
"score": np.max(details["f1"]),
"threshold": details["threshold"][np.argmax(details["f1"])],
},
"accuracy": {
"score": np.max(details["accuracy"]),
"threshold":
details["threshold"][np.argmax(details["accuracy"])],
},
"precision": {
"score": np.max(details["precision"]),
"threshold":
details["threshold"][np.argmax(details["precision"])],
},
"recall": {
"score": np.max(details["recall"]),
"threshold":
details["threshold"][np.argmax(details["recall"])],
},
"mcc": {
"score": np.max(details["mcc"]),
"threshold": details["threshold"][np.argmax(details["mcc"])],
},
}
threshold = float(max_metrics["accuracy"]["threshold"])
# if sample_weight is not None:
# new_max_metrics = {}
# for k, v in max_metrics.items():
# new_max_metrics["weighted_" + k] = v
# max_metrics = new_max_metrics
# confusion matrix
conf_matrix = confusion_matrix(target,
predictions > threshold,
sample_weight=sample_weight)
conf_matrix = pd.DataFrame(
conf_matrix,
columns=[
f"Predicted as {negative_label}",
f"Predicted as {positive_label}",
],
index=[
f"Labeled as {negative_label}", f"Labeled as {positive_label}"
],
)
predicted_labels = pd.Series(
(predictions.ravel() > threshold).astype(int))
predicted_probas = pd.DataFrame({
"proba_0": 1 - predictions.ravel(),
"proba_1": predictions.ravel(),
})
if mapping is not None:
labeled_target = target["target"].map(mapping)
predicted_labels = predicted_labels.map(mapping)
else:
labeled_target = target
return {
"metric_details":
pd.DataFrame(details),
"max_metrics":
pd.DataFrame(max_metrics),
"confusion_matrix":
conf_matrix,
"threshold":
threshold,
"additional_plots":
AdditionalPlots.plots_binary(labeled_target, predicted_labels,
predicted_probas),
}