def test_averaging_with_metrics():
X, y = make_classification_df()
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
oof, test = _make_1st_stage_preds(X_train, y_train, X_test)
result = averaging(test, oof, y_train, eval_func=roc_auc_score)
assert result.score == roc_auc_score(y_train, result.oof_prediction)
def test_weight_averaging():
X, y = make_classification_df()
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
oof, test = _make_1st_stage_preds(X_train, y_train, X_test)
result = averaging(test, oof, y_train, weights=[0.2, 0.4, 0.3])
assert_array_almost_equal(0.2 * test[0] + 0.4 * test[1] + 0.3 * test[2], result.test_prediction)
assert_array_almost_equal(0.2 * oof[0] + 0.4 * oof[1] + 0.3 * oof[2], result.oof_prediction)
assert result.score is None
def test_averaging_multiclass():
X, y = make_classification_df(n_classes=5)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
oof, test = _make_1st_stage_preds(X_train, y_train, X_test)
result = averaging(test, oof, y_train)
assert_array_almost_equal((test[0] + test[1] + test[2]) / 3, result.test_prediction)
assert_array_almost_equal((oof[0] + oof[1] + oof[2]) / 3, result.oof_prediction)
assert result.score is None
def test_rank_averaging():
X, y = make_classification_df(n_samples=1024)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
oof, test = _make_1st_stage_preds(X_train, y_train, X_test)
result = averaging(test, rank_averaging=True)
test_rank = [stats.rankdata(t) / len(X_test) for t in test]
assert_array_almost_equal((test_rank[0] + test_rank[1] + test_rank[2]) / 3, result.test_prediction)
assert result.score is None
def test_rank_averaging_opt_maximize():
X, y = make_classification_df(n_samples=1024)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
oof, test = _make_1st_stage_preds(X_train, y_train, X_test)
best_single_model = max(roc_auc_score(y_train, oof[0]),
roc_auc_score(y_train, oof[1]),
roc_auc_score(y_train, oof[2]))
result = averaging_opt(test, oof, y_train, roc_auc_score, higher_is_better=True, rank_averaging=True)
assert result.score >= best_single_model
result_simple_avg = averaging(test, oof, y_train, eval_func=roc_auc_score, rank_averaging=True)
assert result.score >= result_simple_avg.score
def test_averaging_opt_minimize():
X, y = make_regression_df(n_samples=1024)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
oof, test = _make_1st_stage_preds(X_train, y_train, X_test)
best_single_model = min(mean_squared_error(y_train, oof[0]),
mean_squared_error(y_train, oof[1]),
mean_squared_error(y_train, oof[2]))
result = averaging_opt(test, oof, y_train, mean_squared_error, higher_is_better=False)
assert result.score <= best_single_model
result_simple_avg = averaging(test, oof, y_train, eval_func=mean_squared_error)
assert result.score <= result_simple_avg.score
