def test_invalid_agg_function(agg_function: Any) -> None:
"""Test that invalid agg_functions raise errors."""
mapie_reg = MapieRegressor(agg_function=agg_function)
with pytest.raises(ValueError, match=r".*Invalid aggregation function.*"):
mapie_reg.fit(X_toy, y_toy)
mapie_reg = MapieRegressor(agg_function=None)
with pytest.raises(ValueError, match=r".*If ensemble is True*"):
mapie_reg.fit(X_toy, y_toy)
mapie_reg.predict(X_toy, ensemble=True)
def test_results_for_alpha_as_float_and_arraylike(strategy: str,
alpha: Any) -> None:
"""Test that output values do not depend on type of alpha."""
mapie_reg = MapieRegressor(**STRATEGIES[strategy])
mapie_reg.fit(X, y)
y_pred_float1, y_pis_float1 = mapie_reg.predict(X, alpha=alpha[0])
y_pred_float2, y_pis_float2 = mapie_reg.predict(X, alpha=alpha[1])
y_pred_array, y_pis_array = mapie_reg.predict(X, alpha=alpha)
np.testing.assert_allclose(y_pred_float1, y_pred_array)
np.testing.assert_allclose(y_pred_float2, y_pred_array)
np.testing.assert_allclose(y_pis_float1[:, :, 0], y_pis_array[:, :, 0])
np.testing.assert_allclose(y_pis_float2[:, :, 0], y_pis_array[:, :, 1])
def test_prediction_agg_function(method: str, agg_function: str) -> None:
"""
Test that predictions differ when ensemble is True/False,
but not prediction intervals.
"""
mapie = MapieRegressor(method=method, cv=2, agg_function=agg_function)
mapie.fit(X, y)
y_pred_1, y_pis_1 = mapie.predict(X, ensemble=True, alpha=0.1)
y_pred_2, y_pis_2 = mapie.predict(X, ensemble=False, alpha=0.1)
np.testing.assert_allclose(y_pis_1[:, 0, 0], y_pis_2[:, 0, 0])
np.testing.assert_allclose(y_pis_1[:, 1, 0], y_pis_2[:, 1, 0])
with pytest.raises(AssertionError):
np.testing.assert_allclose(y_pred_1, y_pred_2)
def test_results_single_and_multi_jobs(strategy: str) -> None:
"""
Test that MapieRegressor gives equal predictions
regardless of number of parallel jobs.
"""
mapie_single = MapieRegressor(n_jobs=1, **STRATEGIES[strategy])
mapie_multi = MapieRegressor(n_jobs=-1, **STRATEGIES[strategy])
mapie_single.fit(X_toy, y_toy)
mapie_multi.fit(X_toy, y_toy)
y_pred_single, y_pis_single = mapie_single.predict(X_toy, alpha=0.2)
y_pred_multi, y_pis_multi = mapie_multi.predict(X_toy, alpha=0.2)
np.testing.assert_allclose(y_pred_single, y_pred_multi)
np.testing.assert_allclose(y_pis_single, y_pis_multi)
def test_prediction_between_low_up(strategy: str) -> None:
"""Test that prediction lies between low and up prediction intervals."""
mapie = MapieRegressor(**STRATEGIES[strategy])
mapie.fit(X, y)
y_pred, y_pis = mapie.predict(X, alpha=0.1)
assert (y_pred >= y_pis[:, 0, 0]).all()
assert (y_pred <= y_pis[:, 1, 0]).all()
def test_results_with_constant_sample_weights(strategy: str) -> None:
"""
Test predictions when sample weights are None
or constant with different values.
"""
n_samples = len(X)
mapie0 = MapieRegressor(**STRATEGIES[strategy])
mapie1 = MapieRegressor(**STRATEGIES[strategy])
mapie2 = MapieRegressor(**STRATEGIES[strategy])
mapie0.fit(X, y, sample_weight=None)
mapie1.fit(X, y, sample_weight=np.ones(shape=n_samples))
mapie2.fit(X, y, sample_weight=np.ones(shape=n_samples) * 5)
y_pred0, y_pis0 = mapie0.predict(X, alpha=0.05)
y_pred1, y_pis1 = mapie1.predict(X, alpha=0.05)
y_pred2, y_pis2 = mapie2.predict(X, alpha=0.05)
np.testing.assert_allclose(y_pred0, y_pred1)
np.testing.assert_allclose(y_pred1, y_pred2)
np.testing.assert_allclose(y_pis0, y_pis1)
np.testing.assert_allclose(y_pis1, y_pis2)
def test_results_for_ordered_alpha(strategy: str) -> None:
"""
Test that prediction intervals lower (upper) bounds give
consistent results for ordered alphas.
"""
mapie = MapieRegressor(**STRATEGIES[strategy])
mapie.fit(X, y)
y_pred, y_pis = mapie.predict(X, alpha=[0.05, 0.1])
assert (y_pis[:, 0, 0] <= y_pis[:, 0, 1]).all()
assert (y_pis[:, 1, 0] >= y_pis[:, 1, 1]).all()
def test_results_for_same_alpha(strategy: str) -> None:
"""
Test that predictions and intervals
are similar with two equal values of alpha.
"""
mapie_reg = MapieRegressor(**STRATEGIES[strategy])
mapie_reg.fit(X, y)
_, y_pis = mapie_reg.predict(X, alpha=[0.1, 0.1])
np.testing.assert_allclose(y_pis[:, 0, 0], y_pis[:, 0, 1])
np.testing.assert_allclose(y_pis[:, 1, 0], y_pis[:, 1, 1])
def test_predict_output_shape(strategy: str, alpha: Any,
dataset: Tuple[NDArray, NDArray]) -> None:
"""Test predict output shape."""
mapie_reg = MapieRegressor(**STRATEGIES[strategy])
(X, y) = dataset
mapie_reg.fit(X, y)
y_pred, y_pis = mapie_reg.predict(X, alpha=alpha)
n_alpha = len(alpha) if hasattr(alpha, "__len__") else 1
assert y_pred.shape == (X.shape[0], )
assert y_pis.shape == (X.shape[0], 2, n_alpha)
def test_linear_data_confidence_interval(strategy: str) -> None:
"""
Test that MapieRegressor applied on a linear regression model
fitted on a linear curve results in null uncertainty.
"""
mapie = MapieRegressor(**STRATEGIES[strategy])
mapie.fit(X_toy, y_toy)
y_pred, y_pis = mapie.predict(X_toy, alpha=0.2)
np.testing.assert_allclose(y_pis[:, 0, 0], y_pis[:, 1, 0])
np.testing.assert_allclose(y_pred, y_pis[:, 0, 0])
def test_pipeline_compatibility() -> None:
"""Check that MAPIE works on pipeline based on pandas dataframes"""
X = pd.DataFrame({
"x_cat": ["A", "A", "B", "A", "A", "B"],
"x_num": [0, 1, 1, 4, np.nan, 5],
"y": [5, 7, 3, 9, 10, 8]
})
y = pd.Series([5, 7, 3, 9, 10, 8])
numeric_preprocessor = Pipeline([
("imputer", SimpleImputer(strategy="mean")),
])
categorical_preprocessor = Pipeline(
steps=[("encoding", OneHotEncoder(handle_unknown="ignore"))])
preprocessor = ColumnTransformer([
("cat", categorical_preprocessor, ["x_cat"]),
("num", numeric_preprocessor, ["x_num"])
])
pipe = make_pipeline(preprocessor, LinearRegression())
mapie = MapieRegressor(pipe)
mapie.fit(X, y)
mapie.predict(X)
def compute_PIs(
estimator: BaseEstimator,
X_train: NDArray,
y_train: NDArray,
X_test: NDArray,
method: str,
cv: Any,
alpha: float,
agg_function: Optional[str] = None,
) -> pd.DataFrame:
"""
Train and test a model with a MAPIE method,
and return a DataFrame of upper and lower bounds of the predictions
on the test set.
Parameters
----------
estimator : BaseEstimator
Base model to fit.
X_train : NDArray
Features of training set.
y_train : NDArray
Target of training set.
X_test : NDArray
Features of testing set.
method : str
Method for estimating prediction intervals.
cv : Any
Strategy for computing residuals.
alpha : float
1 - (target coverage level).
agg_function: str
'mean' or 'median'.
Function to aggregate the predictions of the B estimators.
Returns
-------
pd.DataFrame
DataFrame of upper and lower predictions.
"""
mapie_estimator = MapieRegressor(
estimator=estimator,
method=method,
cv=cv,
n_jobs=-1,
agg_function=agg_function,
)
mapie_estimator = mapie_estimator.fit(X=X_train, y=y_train)
_, y_pis = mapie_estimator.predict(X=X_test, alpha=alpha)
PI = np.c_[y_pis[:, 0, 0], y_pis[:, 1, 0]]
return pd.DataFrame(PI, columns=["lower", "upper"])
def test_results_prefit_naive() -> None:
"""
Test that prefit, fit and predict on the same dataset
is equivalent to the "naive" method.
"""
estimator = LinearRegression().fit(X, y)
mapie_reg = MapieRegressor(estimator=estimator, cv="prefit")
mapie_reg.fit(X, y)
_, y_pis = mapie_reg.predict(X, alpha=0.05)
width_mean = (y_pis[:, 1, 0] - y_pis[:, 0, 0]).mean()
coverage = regression_coverage_score(y, y_pis[:, 0, 0], y_pis[:, 1, 0])
np.testing.assert_allclose(width_mean, WIDTHS["naive"], rtol=1e-2)
np.testing.assert_allclose(coverage, COVERAGES["naive"], rtol=1e-2)
def test_results_prefit_ignore_method() -> None:
"""Test that method is ignored when ``cv="prefit"``."""
estimator = LinearRegression().fit(X, y)
all_y_pis: List[NDArray] = []
for method in METHODS:
mapie_reg = MapieRegressor(estimator=estimator,
cv="prefit",
method=method)
mapie_reg.fit(X, y)
_, y_pis = mapie_reg.predict(X, alpha=0.1)
all_y_pis.append(y_pis)
for y_pis1, y_pis2 in combinations(all_y_pis, 2):
np.testing.assert_allclose(y_pis1, y_pis2)
def test_linear_regression_results(strategy: str) -> None:
"""
Test expected prediction intervals for
a multivariate linear regression problem
with fixed random state.
"""
mapie = MapieRegressor(**STRATEGIES[strategy])
mapie.fit(X, y)
_, y_pis = mapie.predict(X, alpha=0.05)
y_pred_low, y_pred_up = y_pis[:, 0, 0], y_pis[:, 1, 0]
width_mean = (y_pred_up - y_pred_low).mean()
coverage = regression_coverage_score(y, y_pred_low, y_pred_up)
np.testing.assert_allclose(width_mean, WIDTHS[strategy], rtol=1e-2)
np.testing.assert_allclose(coverage, COVERAGES[strategy], rtol=1e-2)
def test_results_prefit() -> None:
"""Test prefit results on a standard train/validation/test split."""
X_train_val, X_test, y_train_val, y_test = train_test_split(X,
y,
test_size=1 /
10,
random_state=1)
X_train, X_val, y_train, y_val = train_test_split(X_train_val,
y_train_val,
test_size=1 / 9,
random_state=1)
estimator = LinearRegression().fit(X_train, y_train)
mapie_reg = MapieRegressor(estimator=estimator, cv="prefit")
mapie_reg.fit(X_val, y_val)
_, y_pis = mapie_reg.predict(X_test, alpha=0.05)
width_mean = (y_pis[:, 1, 0] - y_pis[:, 0, 0]).mean()
coverage = regression_coverage_score(y_test, y_pis[:, 0, 0], y_pis[:, 1,
0])
np.testing.assert_allclose(width_mean, WIDTHS["prefit"], rtol=1e-2)
np.testing.assert_allclose(coverage, COVERAGES["prefit"], rtol=1e-2)
def PIs_vs_dimensions(
strategies: Dict[str, Any],
alpha: float,
n_trial: int,
dimensions: NDArray,
) -> Dict[str, Dict[int, Dict[str, NDArray]]]:
"""
Compute the prediction intervals for a linear regression problem.
Function adapted from Foygel-Barber et al. (2020).
It generates several times linear data with random noise whose
signal-to-noise is equal to 10 and for several given dimensions,
given by the dimensions list.
Here we use MAPIE, with a LinearRegression base model, to estimate
the width means and the coverage levels of the prediction intervals
estimated by all the available strategies as a function of
the dataset dimension.
This simulation is carried out to emphasize the instability
of the prediction intervals estimated by the Jackknife strategy
when the dataset dimension is equal to the number
of training samples (here 100).
Parameters
----------
strategies : Dict[str, Dict[str, Any]]
List of strategies for estimating prediction intervals,
with corresponding parameters.
alpha : float
1 - (target coverage level).
n_trial : int
Number of trials for each dimension for estimating
prediction intervals.
For each trial, a new random noise is generated.
dimensions : List[int]
List of dimension values of input data.
Returns
-------
Dict[str, Dict[int, Dict[str, NDArray]]]
Prediction interval widths and coverages for each strategy, trial,
and dimension value.
"""
n_train = 100
n_test = 100
SNR = 10
results: Dict[str, Dict[int, Dict[str, NDArray]]] = {
strategy: {
dimension: {
"coverage": np.empty(n_trial),
"width_mean": np.empty(n_trial),
}
for dimension in dimensions
}
for strategy in strategies
}
for dimension in dimensions:
for trial in range(n_trial):
beta = np.random.normal(size=dimension)
beta_norm = np.sqrt(np.square(beta).sum())
beta = beta / beta_norm * np.sqrt(SNR)
X_train = np.random.normal(size=(n_train, dimension))
noise_train = np.random.normal(size=n_train)
noise_test = np.random.normal(size=n_test)
y_train = X_train.dot(beta) + noise_train
X_test = np.random.normal(size=(n_test, dimension))
y_test = X_test.dot(beta) + noise_test
for strategy, params in strategies.items():
mapie = MapieRegressor(LinearRegression(),
agg_function="median",
n_jobs=-1,
**params)
mapie.fit(X_train, y_train)
_, y_pis = mapie.predict(X_test, alpha=alpha)
coverage = regression_coverage_score(y_test, y_pis[:, 0, 0],
y_pis[:, 1, 0])
results[strategy][dimension]["coverage"][trial] = coverage
width_mean = regression_mean_width_score(
y_pis[:, 0, 0], y_pis[:, 1, 0])
results[strategy][dimension]["width_mean"][trial] = width_mean
return results
("linear", LinearRegression()),
])
# Estimating prediction intervals
Params = TypedDict("Params", {"method": str, "cv": int})
STRATEGIES = {
"jackknife_plus": Params(method="plus", cv=-1),
"jackknife_minmax": Params(method="minmax", cv=-1),
"cv_plus": Params(method="plus", cv=10),
"cv_minmax": Params(method="minmax", cv=10),
}
y_pred, y_pis = {}, {}
for strategy, params in STRATEGIES.items():
mapie = MapieRegressor(polyn_model, **params)
mapie.fit(X_train, y_train)
y_pred[strategy], y_pis[strategy] = mapie.predict(X_test, alpha=0.05)
# Visualization
def plot_1d_data(
X_train: NDArray,
y_train: NDArray,
X_test: NDArray,
y_test: NDArray,
y_sigma: float,
y_pred: NDArray,
y_pred_low: NDArray,
y_pred_up: NDArray,
ax: plt.Axes,
title: str,
) -> None:
"""
import numpy as np
from matplotlib import pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.datasets import make_regression
from mapie.regression import MapieRegressor
from mapie.metrics import regression_coverage_score
regressor = LinearRegression()
X, y = make_regression(n_samples=500, n_features=1, noise=20, random_state=59)
alpha = [0.05, 0.32]
mapie = MapieRegressor(regressor, method="plus")
mapie.fit(X, y)
y_pred, y_pis = mapie.predict(X, alpha=alpha)
coverage_scores = [
regression_coverage_score(y, y_pis[:, 0, i], y_pis[:, 1, i])
for i, _ in enumerate(alpha)
]
plt.xlabel("x")
plt.ylabel("y")
plt.scatter(X, y, alpha=0.3)
plt.plot(X, y_pred, color="C1")
order = np.argsort(X[:, 0])
plt.plot(X[order], y_pis[order][:, 0, 1], color="C1", ls="--")
plt.plot(X[order], y_pis[order][:, 1, 1], color="C1", ls="--")
plt.fill_between(
X[order].ravel(),
test_size=1 / 10)
X_train, X_val, y_train, y_val = train_test_split(X_train_val,
y_train_val,
test_size=1 / 9)
# Train model on training set
model = MLPRegressor(activation="relu", random_state=1)
model.fit(X_train.reshape(-1, 1), y_train)
# Calibrate uncertainties on validation set
mapie = MapieRegressor(model, cv="prefit")
mapie.fit(X_val.reshape(-1, 1), y_val)
# Evaluate prediction and coverage level on testing set
alpha = 0.1
y_pred, y_pis = mapie.predict(X_test.reshape(-1, 1), alpha=alpha)
y_pred_low, y_pred_up = y_pis[:, 0, 0], y_pis[:, 1, 0]
coverage = regression_coverage_score(y_test, y_pred_low, y_pred_up)
# Plot obtained prediction intervals on testing set
theoretical_semi_width = scipy.stats.norm.ppf(1 - alpha) * sigma
y_test_theoretical = f(X_test)
order = np.argsort(X_test)
plt.scatter(X_test, y_test, color="red", alpha=0.3, label="testing", s=2)
plt.plot(
X_test[order],
y_test_theoretical[order],
color="gray",
label="True confidence intervals",
)
cv=cv,
scoring="neg_root_mean_squared_error",
return_train_score=True,
verbose=0,
random_state=random_state,
n_jobs=-1,
)
cv_obj.fit(X_train, y_train)
best_est = cv_obj.best_estimator_
mapie_non_nested = MapieRegressor(best_est,
method="plus",
cv=cv,
agg_function="median",
n_jobs=-1)
mapie_non_nested.fit(X_train, y_train)
y_pred_non_nested, y_pis_non_nested = mapie_non_nested.predict(X_test,
alpha=alpha)
widths_non_nested = y_pis_non_nested[:, 1, 0] - y_pis_non_nested[:, 0, 0]
coverage_non_nested = regression_coverage_score(y_test, y_pis_non_nested[:, 0,
0],
y_pis_non_nested[:, 1, 0])
score_non_nested = mean_squared_error(y_test, y_pred_non_nested, squared=False)
# Nested approach with the CV+ strategy using the Random Forest model.
cv_obj = RandomizedSearchCV(
rf_model,
param_distributions=rf_params,
n_iter=n_iter,
cv=cv,
scoring="neg_root_mean_squared_error",
return_train_score=True,
verbose=0,