def test_no_agg_fx_specified_with_subsample() -> None:
"""Test that a warning is raised if at least one residual is nan."""
with pytest.raises(ValueError,
match=r"You need to specify an aggregation*"):
mapie_reg = MapieRegressor(cv=Subsample(n_resamplings=1),
agg_function=None)
mapie_reg.fit(X, y)
def test_prediction_between_low_up(strategy: str) -> None:
"""Test that prediction lies between low and up prediction intervals."""
mapie = MapieRegressor(**STRATEGIES[strategy])
mapie.fit(X, y)
y_pred, y_pis = mapie.predict(X, alpha=0.1)
assert (y_pred >= y_pis[:, 0, 0]).all()
assert (y_pred <= y_pis[:, 1, 0]).all()
def test_too_large_cv(cv: Any) -> None:
"""Test that too large cv raise sklearn errors."""
mapie_reg = MapieRegressor(cv=cv)
with pytest.raises(
ValueError,
match=rf".*Cannot have number of splits n_splits={cv} greater.*",
):
mapie_reg.fit(X_toy, y_toy)
def test_valid_estimator(strategy: str) -> None:
"""Test that valid estimators are not corrupted, for all strategies."""
mapie_reg = MapieRegressor(estimator=DummyRegressor(),
**STRATEGIES[strategy])
mapie_reg.fit(X_toy, y_toy)
assert isinstance(mapie_reg.single_estimator_, DummyRegressor)
for estimator in mapie_reg.estimators_:
assert isinstance(estimator, DummyRegressor)
def test_results_for_ordered_alpha(strategy: str) -> None:
"""
Test that prediction intervals lower (upper) bounds give
consistent results for ordered alphas.
"""
mapie = MapieRegressor(**STRATEGIES[strategy])
mapie.fit(X, y)
y_pred, y_pis = mapie.predict(X, alpha=[0.05, 0.1])
assert (y_pis[:, 0, 0] <= y_pis[:, 0, 1]).all()
assert (y_pis[:, 1, 0] >= y_pis[:, 1, 1]).all()
def test_linear_data_confidence_interval(strategy: str) -> None:
"""
Test that MapieRegressor applied on a linear regression model
fitted on a linear curve results in null uncertainty.
"""
mapie = MapieRegressor(**STRATEGIES[strategy])
mapie.fit(X_toy, y_toy)
y_pred, y_pis = mapie.predict(X_toy, alpha=0.2)
np.testing.assert_allclose(y_pis[:, 0, 0], y_pis[:, 1, 0])
np.testing.assert_allclose(y_pred, y_pis[:, 0, 0])
def test_results_for_same_alpha(strategy: str) -> None:
"""
Test that predictions and intervals
are similar with two equal values of alpha.
"""
mapie_reg = MapieRegressor(**STRATEGIES[strategy])
mapie_reg.fit(X, y)
_, y_pis = mapie_reg.predict(X, alpha=[0.1, 0.1])
np.testing.assert_allclose(y_pis[:, 0, 0], y_pis[:, 0, 1])
np.testing.assert_allclose(y_pis[:, 1, 0], y_pis[:, 1, 1])
def test_predict_output_shape(strategy: str, alpha: Any,
dataset: Tuple[NDArray, NDArray]) -> None:
"""Test predict output shape."""
mapie_reg = MapieRegressor(**STRATEGIES[strategy])
(X, y) = dataset
mapie_reg.fit(X, y)
y_pred, y_pis = mapie_reg.predict(X, alpha=alpha)
n_alpha = len(alpha) if hasattr(alpha, "__len__") else 1
assert y_pred.shape == (X.shape[0], )
assert y_pis.shape == (X.shape[0], 2, n_alpha)
def test_invalid_agg_function(agg_function: Any) -> None:
"""Test that invalid agg_functions raise errors."""
mapie_reg = MapieRegressor(agg_function=agg_function)
with pytest.raises(ValueError, match=r".*Invalid aggregation function.*"):
mapie_reg.fit(X_toy, y_toy)
mapie_reg = MapieRegressor(agg_function=None)
with pytest.raises(ValueError, match=r".*If ensemble is True*"):
mapie_reg.fit(X_toy, y_toy)
mapie_reg.predict(X_toy, ensemble=True)
def test_pred_loof_isnan() -> None:
"""Test that if validation set is empty then prediction is empty."""
mapie_reg = MapieRegressor()
y_pred: ArrayLike
_, y_pred, _ = mapie_reg._fit_and_predict_oof_model(
estimator=LinearRegression(),
X=X_toy,
y=y_toy,
train_index=[0, 1, 2, 3, 4],
val_index=[],
)
assert len(y_pred) == 0
def compute_PIs(
estimator: BaseEstimator,
X_train: NDArray,
y_train: NDArray,
X_test: NDArray,
method: str,
cv: Any,
alpha: float,
agg_function: Optional[str] = None,
) -> pd.DataFrame:
"""
Train and test a model with a MAPIE method,
and return a DataFrame of upper and lower bounds of the predictions
on the test set.
Parameters
----------
estimator : BaseEstimator
Base model to fit.
X_train : NDArray
Features of training set.
y_train : NDArray
Target of training set.
X_test : NDArray
Features of testing set.
method : str
Method for estimating prediction intervals.
cv : Any
Strategy for computing residuals.
alpha : float
1 - (target coverage level).
agg_function: str
'mean' or 'median'.
Function to aggregate the predictions of the B estimators.
Returns
-------
pd.DataFrame
DataFrame of upper and lower predictions.
"""
mapie_estimator = MapieRegressor(
estimator=estimator,
method=method,
cv=cv,
n_jobs=-1,
agg_function=agg_function,
)
mapie_estimator = mapie_estimator.fit(X=X_train, y=y_train)
_, y_pis = mapie_estimator.predict(X=X_test, alpha=alpha)
PI = np.c_[y_pis[:, 0, 0], y_pis[:, 1, 0]]
return pd.DataFrame(PI, columns=["lower", "upper"])
def test_results_single_and_multi_jobs(strategy: str) -> None:
"""
Test that MapieRegressor gives equal predictions
regardless of number of parallel jobs.
"""
mapie_single = MapieRegressor(n_jobs=1, **STRATEGIES[strategy])
mapie_multi = MapieRegressor(n_jobs=-1, **STRATEGIES[strategy])
mapie_single.fit(X_toy, y_toy)
mapie_multi.fit(X_toy, y_toy)
y_pred_single, y_pis_single = mapie_single.predict(X_toy, alpha=0.2)
y_pred_multi, y_pis_multi = mapie_multi.predict(X_toy, alpha=0.2)
np.testing.assert_allclose(y_pred_single, y_pred_multi)
np.testing.assert_allclose(y_pis_single, y_pis_multi)
def test_results_prefit_ignore_method() -> None:
"""Test that method is ignored when ``cv="prefit"``."""
estimator = LinearRegression().fit(X, y)
all_y_pis: List[NDArray] = []
for method in METHODS:
mapie_reg = MapieRegressor(estimator=estimator,
cv="prefit",
method=method)
mapie_reg.fit(X, y)
_, y_pis = mapie_reg.predict(X, alpha=0.1)
all_y_pis.append(y_pis)
for y_pis1, y_pis2 in combinations(all_y_pis, 2):
np.testing.assert_allclose(y_pis1, y_pis2)
def test_results_prefit_naive() -> None:
"""
Test that prefit, fit and predict on the same dataset
is equivalent to the "naive" method.
"""
estimator = LinearRegression().fit(X, y)
mapie_reg = MapieRegressor(estimator=estimator, cv="prefit")
mapie_reg.fit(X, y)
_, y_pis = mapie_reg.predict(X, alpha=0.05)
width_mean = (y_pis[:, 1, 0] - y_pis[:, 0, 0]).mean()
coverage = regression_coverage_score(y, y_pis[:, 0, 0], y_pis[:, 1, 0])
np.testing.assert_allclose(width_mean, WIDTHS["naive"], rtol=1e-2)
np.testing.assert_allclose(coverage, COVERAGES["naive"], rtol=1e-2)
def test_linear_regression_results(strategy: str) -> None:
"""
Test expected prediction intervals for
a multivariate linear regression problem
with fixed random state.
"""
mapie = MapieRegressor(**STRATEGIES[strategy])
mapie.fit(X, y)
_, y_pis = mapie.predict(X, alpha=0.05)
y_pred_low, y_pred_up = y_pis[:, 0, 0], y_pis[:, 1, 0]
width_mean = (y_pred_up - y_pred_low).mean()
coverage = regression_coverage_score(y, y_pred_low, y_pred_up)
np.testing.assert_allclose(width_mean, WIDTHS[strategy], rtol=1e-2)
np.testing.assert_allclose(coverage, COVERAGES[strategy], rtol=1e-2)
def test_results_prefit() -> None:
"""Test prefit results on a standard train/validation/test split."""
X_train_val, X_test, y_train_val, y_test = train_test_split(X,
y,
test_size=1 /
10,
random_state=1)
X_train, X_val, y_train, y_val = train_test_split(X_train_val,
y_train_val,
test_size=1 / 9,
random_state=1)
estimator = LinearRegression().fit(X_train, y_train)
mapie_reg = MapieRegressor(estimator=estimator, cv="prefit")
mapie_reg.fit(X_val, y_val)
_, y_pis = mapie_reg.predict(X_test, alpha=0.05)
width_mean = (y_pis[:, 1, 0] - y_pis[:, 0, 0]).mean()
coverage = regression_coverage_score(y_test, y_pis[:, 0, 0], y_pis[:, 1,
0])
np.testing.assert_allclose(width_mean, WIDTHS["prefit"], rtol=1e-2)
np.testing.assert_allclose(coverage, COVERAGES["prefit"], rtol=1e-2)
def test_aggregate_with_mask_with_prefit() -> None:
"""
Test ``aggregate_with_mask`` in case ``cv`` is ``"prefit"``.
"""
mapie_reg = MapieRegressor(cv="prefit")
with pytest.raises(
ValueError,
match=r".*There should not be aggregation of predictions if cv is*",
):
mapie_reg.aggregate_with_mask(k, k)
mapie_reg = MapieRegressor(agg_function="nonsense")
with pytest.raises(
ValueError,
match=r".*The value of self.agg_function is not correct*",
):
mapie_reg.aggregate_with_mask(k, k)
def test_results_for_alpha_as_float_and_arraylike(strategy: str,
alpha: Any) -> None:
"""Test that output values do not depend on type of alpha."""
mapie_reg = MapieRegressor(**STRATEGIES[strategy])
mapie_reg.fit(X, y)
y_pred_float1, y_pis_float1 = mapie_reg.predict(X, alpha=alpha[0])
y_pred_float2, y_pis_float2 = mapie_reg.predict(X, alpha=alpha[1])
y_pred_array, y_pis_array = mapie_reg.predict(X, alpha=alpha)
np.testing.assert_allclose(y_pred_float1, y_pred_array)
np.testing.assert_allclose(y_pred_float2, y_pred_array)
np.testing.assert_allclose(y_pis_float1[:, :, 0], y_pis_array[:, :, 0])
np.testing.assert_allclose(y_pis_float2[:, :, 0], y_pis_array[:, :, 1])
def test_prediction_agg_function(method: str, agg_function: str) -> None:
"""
Test that predictions differ when ensemble is True/False,
but not prediction intervals.
"""
mapie = MapieRegressor(method=method, cv=2, agg_function=agg_function)
mapie.fit(X, y)
y_pred_1, y_pis_1 = mapie.predict(X, ensemble=True, alpha=0.1)
y_pred_2, y_pis_2 = mapie.predict(X, ensemble=False, alpha=0.1)
np.testing.assert_allclose(y_pis_1[:, 0, 0], y_pis_2[:, 0, 0])
np.testing.assert_allclose(y_pis_1[:, 1, 0], y_pis_2[:, 1, 0])
with pytest.raises(AssertionError):
np.testing.assert_allclose(y_pred_1, y_pred_2)
def test_pipeline_compatibility() -> None:
"""Check that MAPIE works on pipeline based on pandas dataframes"""
X = pd.DataFrame({
"x_cat": ["A", "A", "B", "A", "A", "B"],
"x_num": [0, 1, 1, 4, np.nan, 5],
"y": [5, 7, 3, 9, 10, 8]
})
y = pd.Series([5, 7, 3, 9, 10, 8])
numeric_preprocessor = Pipeline([
("imputer", SimpleImputer(strategy="mean")),
])
categorical_preprocessor = Pipeline(
steps=[("encoding", OneHotEncoder(handle_unknown="ignore"))])
preprocessor = ColumnTransformer([
("cat", categorical_preprocessor, ["x_cat"]),
("num", numeric_preprocessor, ["x_num"])
])
pipe = make_pipeline(preprocessor, LinearRegression())
mapie = MapieRegressor(pipe)
mapie.fit(X, y)
mapie.predict(X)
Params = TypedDict("Params", {"method": str, "cv": int})
STRATEGIES = {
"jackknife": Params(method="base", cv=-1),
"jackknife_plus": Params(method="plus", cv=-1),
"jackknife_minmax": Params(method="minmax", cv=-1),
"cv": Params(method="base", cv=10),
"cv_plus": Params(method="plus", cv=10),
"cv_minmax": Params(method="minmax", cv=10),
}
fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(2,
3,
figsize=(3 * 6, 12))
axs = [ax1, ax2, ax3, ax4, ax5, ax6]
for i, (strategy, params) in enumerate(STRATEGIES.items()):
mapie = MapieRegressor(polyn_model,
agg_function="median",
n_jobs=-1,
**params)
mapie.fit(X_train.reshape(-1, 1), y_train)
y_pred, y_pis = mapie.predict(
X_test.reshape(-1, 1),
alpha=0.05,
)
plot_1d_data(
X_train,
y_train,
X_test,
y_test,
y_test_sigma,
y_pred,
y_pis[:, 0, 0],
y_pis[:, 1, 0],
cv_obj = RandomizedSearchCV(
rf_model,
param_distributions=rf_params,
n_iter=n_iter,
cv=cv,
scoring="neg_root_mean_squared_error",
return_train_score=True,
verbose=0,
random_state=random_state,
n_jobs=-1,
)
cv_obj.fit(X_train, y_train)
best_est = cv_obj.best_estimator_
mapie_non_nested = MapieRegressor(best_est,
method="plus",
cv=cv,
agg_function="median",
n_jobs=-1)
mapie_non_nested.fit(X_train, y_train)
y_pred_non_nested, y_pis_non_nested = mapie_non_nested.predict(X_test,
alpha=alpha)
widths_non_nested = y_pis_non_nested[:, 1, 0] - y_pis_non_nested[:, 0, 0]
coverage_non_nested = regression_coverage_score(y_test, y_pis_non_nested[:, 0,
0],
y_pis_non_nested[:, 1, 0])
score_non_nested = mean_squared_error(y_test, y_pred_non_nested, squared=False)
# Nested approach with the CV+ strategy using the Random Forest model.
cv_obj = RandomizedSearchCV(
rf_model,
param_distributions=rf_params,
def test_valid_agg_function(agg_function: str) -> None:
"""Test that valid agg_functions raise no errors."""
mapie_reg = MapieRegressor(agg_function=agg_function)
mapie_reg.fit(X_toy, y_toy)
polyn_model = Pipeline([
("poly", PolynomialFeatures(degree=degree_polyn)),
("linear", LinearRegression()),
])
# Estimating prediction intervals
Params = TypedDict("Params", {"method": str, "cv": int})
STRATEGIES = {
"jackknife_plus": Params(method="plus", cv=-1),
"jackknife_minmax": Params(method="minmax", cv=-1),
"cv_plus": Params(method="plus", cv=10),
"cv_minmax": Params(method="minmax", cv=10),
}
y_pred, y_pis = {}, {}
for strategy, params in STRATEGIES.items():
mapie = MapieRegressor(polyn_model, **params)
mapie.fit(X_train, y_train)
y_pred[strategy], y_pis[strategy] = mapie.predict(X_test, alpha=0.05)
# Visualization
def plot_1d_data(
X_train: NDArray,
y_train: NDArray,
X_test: NDArray,
y_test: NDArray,
y_sigma: float,
y_pred: NDArray,
y_pred_low: NDArray,
y_pred_up: NDArray,
ax: plt.Axes,
def test_valid_method(method: str) -> None:
"""Test that valid methods raise no errors."""
mapie_reg = MapieRegressor(method=method)
mapie_reg.fit(X_toy, y_toy)
check_is_fitted(mapie_reg, mapie_reg.fit_attributes)
y = f(X) + np.random.normal(0, sigma, n_samples)
# Train/validation/test split
X_train_val, X_test, y_train_val, y_test = train_test_split(X,
y,
test_size=1 / 10)
X_train, X_val, y_train, y_val = train_test_split(X_train_val,
y_train_val,
test_size=1 / 9)
# Train model on training set
model = MLPRegressor(activation="relu", random_state=1)
model.fit(X_train.reshape(-1, 1), y_train)
# Calibrate uncertainties on validation set
mapie = MapieRegressor(model, cv="prefit")
mapie.fit(X_val.reshape(-1, 1), y_val)
# Evaluate prediction and coverage level on testing set
alpha = 0.1
y_pred, y_pis = mapie.predict(X_test.reshape(-1, 1), alpha=alpha)
y_pred_low, y_pred_up = y_pis[:, 0, 0], y_pis[:, 1, 0]
coverage = regression_coverage_score(y_test, y_pred_low, y_pred_up)
# Plot obtained prediction intervals on testing set
theoretical_semi_width = scipy.stats.norm.ppf(1 - alpha) * sigma
y_test_theoretical = f(X_test)
order = np.argsort(X_test)
plt.scatter(X_test, y_test, color="red", alpha=0.3, label="testing", s=2)
plt.plot(
An example plot of :class:`mapie.regression.MapieRegressor` used
in the Quickstart.
"""
import numpy as np
from matplotlib import pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.datasets import make_regression
from mapie.regression import MapieRegressor
from mapie.metrics import regression_coverage_score
regressor = LinearRegression()
X, y = make_regression(n_samples=500, n_features=1, noise=20, random_state=59)
alpha = [0.05, 0.32]
mapie = MapieRegressor(regressor, method="plus")
mapie.fit(X, y)
y_pred, y_pis = mapie.predict(X, alpha=alpha)
coverage_scores = [
regression_coverage_score(y, y_pis[:, 0, i], y_pis[:, 1, i])
for i, _ in enumerate(alpha)
]
plt.xlabel("x")
plt.ylabel("y")
plt.scatter(X, y, alpha=0.3)
plt.plot(X, y_pred, color="C1")
order = np.argsort(X[:, 0])
plt.plot(X[order], y_pis[order][:, 0, 1], color="C1", ls="--")
plt.plot(X[order], y_pis[order][:, 1, 1], color="C1", ls="--")
def test_not_enough_resamplings() -> None:
"""Test that a warning is raised if at least one residual is nan."""
with pytest.warns(UserWarning, match=r"WARNING: at least one point of*"):
mapie_reg = MapieRegressor(cv=Subsample(n_resamplings=1),
agg_function="mean")
mapie_reg.fit(X, y)
def test_valid_cv(cv: Any) -> None:
"""Test that valid cv raise no errors."""
mapie = MapieRegressor(cv=cv)
mapie.fit(X_toy, y_toy)
def test_results_with_constant_sample_weights(strategy: str) -> None:
"""
Test predictions when sample weights are None
or constant with different values.
"""
n_samples = len(X)
mapie0 = MapieRegressor(**STRATEGIES[strategy])
mapie1 = MapieRegressor(**STRATEGIES[strategy])
mapie2 = MapieRegressor(**STRATEGIES[strategy])
mapie0.fit(X, y, sample_weight=None)
mapie1.fit(X, y, sample_weight=np.ones(shape=n_samples))
mapie2.fit(X, y, sample_weight=np.ones(shape=n_samples) * 5)
y_pred0, y_pis0 = mapie0.predict(X, alpha=0.05)
y_pred1, y_pis1 = mapie1.predict(X, alpha=0.05)
y_pred2, y_pis2 = mapie2.predict(X, alpha=0.05)
np.testing.assert_allclose(y_pred0, y_pred1)
np.testing.assert_allclose(y_pred1, y_pred2)
np.testing.assert_allclose(y_pis0, y_pis1)
np.testing.assert_allclose(y_pis1, y_pis2)