def eval_metrics(
model: BaseEstimator, *data: Sequence[np.ndarray]
) -> Tuple[np.ndarray, np.ndarray]:
X_train, X_test, y_train, y_test = data
acc_train = model.score(X_train, y_train)
decision_func_train = model.decision_function(X_train)
auc_train = roc_auc_score(y_train, decision_func_train)
fpr_train, tpr_train, _ = roc_curve(y_train, decision_func_train)
print(f"Training set: AUC={auc_train}, acc={acc_train}")
acc_test = model.score(X_test, y_test)
decision_func_test = model.decision_function(X_test)
auc_test = roc_auc_score(y_test, decision_func_test)
fpr_test, tpr_test, _ = roc_curve(y_test, decision_func_test)
print(f"Testing set: AUC={auc_test}, acc={acc_test}")
return decision_func_train, decision_func_test
def _predict(model: BaseEstimator, x: pd.DataFrame, _type_of_target: str):
if _type_of_target in ('binary', 'multiclass'):
if hasattr(model, "predict_proba"):
proba = model.predict_proba(x)
elif hasattr(model, "decision_function"):
warnings.warn(
'Since {} does not have predict_proba method, '
'decision_function is used for the prediction instead.'.
format(type(model)))
proba = model.decision_function(x)
else:
raise RuntimeError(
'Estimator in classification problem should have '
'either predict_proba or decision_function')
if proba.ndim == 1:
return proba
else:
return proba[:, 1] if proba.shape[1] == 2 else proba
else:
return model.predict(x)
