def test_no_agg_fx_specified_with_subsample() -> None: """Test that a warning is raised if at least one residual is nan.""" with pytest.raises(ValueError, match=r"You need to specify an aggregation*"): mapie_reg = MapieRegressor(cv=Subsample(n_resamplings=1), agg_function=None) mapie_reg.fit(X, y)
def test_prediction_between_low_up(strategy: str) -> None: """Test that prediction lies between low and up prediction intervals.""" mapie = MapieRegressor(**STRATEGIES[strategy]) mapie.fit(X, y) y_pred, y_pis = mapie.predict(X, alpha=0.1) assert (y_pred >= y_pis[:, 0, 0]).all() assert (y_pred <= y_pis[:, 1, 0]).all()
def test_too_large_cv(cv: Any) -> None: """Test that too large cv raise sklearn errors.""" mapie_reg = MapieRegressor(cv=cv) with pytest.raises( ValueError, match=rf".*Cannot have number of splits n_splits={cv} greater.*", ): mapie_reg.fit(X_toy, y_toy)
def test_valid_estimator(strategy: str) -> None: """Test that valid estimators are not corrupted, for all strategies.""" mapie_reg = MapieRegressor(estimator=DummyRegressor(), **STRATEGIES[strategy]) mapie_reg.fit(X_toy, y_toy) assert isinstance(mapie_reg.single_estimator_, DummyRegressor) for estimator in mapie_reg.estimators_: assert isinstance(estimator, DummyRegressor)
def test_results_for_ordered_alpha(strategy: str) -> None: """ Test that prediction intervals lower (upper) bounds give consistent results for ordered alphas. """ mapie = MapieRegressor(**STRATEGIES[strategy]) mapie.fit(X, y) y_pred, y_pis = mapie.predict(X, alpha=[0.05, 0.1]) assert (y_pis[:, 0, 0] <= y_pis[:, 0, 1]).all() assert (y_pis[:, 1, 0] >= y_pis[:, 1, 1]).all()
def test_predict_output_shape(strategy: str, alpha: Any, dataset: Tuple[NDArray, NDArray]) -> None: """Test predict output shape.""" mapie_reg = MapieRegressor(**STRATEGIES[strategy]) (X, y) = dataset mapie_reg.fit(X, y) y_pred, y_pis = mapie_reg.predict(X, alpha=alpha) n_alpha = len(alpha) if hasattr(alpha, "__len__") else 1 assert y_pred.shape == (X.shape[0], ) assert y_pis.shape == (X.shape[0], 2, n_alpha)
def test_invalid_agg_function(agg_function: Any) -> None: """Test that invalid agg_functions raise errors.""" mapie_reg = MapieRegressor(agg_function=agg_function) with pytest.raises(ValueError, match=r".*Invalid aggregation function.*"): mapie_reg.fit(X_toy, y_toy) mapie_reg = MapieRegressor(agg_function=None) with pytest.raises(ValueError, match=r".*If ensemble is True*"): mapie_reg.fit(X_toy, y_toy) mapie_reg.predict(X_toy, ensemble=True)
def test_linear_data_confidence_interval(strategy: str) -> None: """ Test that MapieRegressor applied on a linear regression model fitted on a linear curve results in null uncertainty. """ mapie = MapieRegressor(**STRATEGIES[strategy]) mapie.fit(X_toy, y_toy) y_pred, y_pis = mapie.predict(X_toy, alpha=0.2) np.testing.assert_allclose(y_pis[:, 0, 0], y_pis[:, 1, 0]) np.testing.assert_allclose(y_pred, y_pis[:, 0, 0])
def test_results_for_same_alpha(strategy: str) -> None: """ Test that predictions and intervals are similar with two equal values of alpha. """ mapie_reg = MapieRegressor(**STRATEGIES[strategy]) mapie_reg.fit(X, y) _, y_pis = mapie_reg.predict(X, alpha=[0.1, 0.1]) np.testing.assert_allclose(y_pis[:, 0, 0], y_pis[:, 0, 1]) np.testing.assert_allclose(y_pis[:, 1, 0], y_pis[:, 1, 1])
def test_results_for_alpha_as_float_and_arraylike(strategy: str, alpha: Any) -> None: """Test that output values do not depend on type of alpha.""" mapie_reg = MapieRegressor(**STRATEGIES[strategy]) mapie_reg.fit(X, y) y_pred_float1, y_pis_float1 = mapie_reg.predict(X, alpha=alpha[0]) y_pred_float2, y_pis_float2 = mapie_reg.predict(X, alpha=alpha[1]) y_pred_array, y_pis_array = mapie_reg.predict(X, alpha=alpha) np.testing.assert_allclose(y_pred_float1, y_pred_array) np.testing.assert_allclose(y_pred_float2, y_pred_array) np.testing.assert_allclose(y_pis_float1[:, :, 0], y_pis_array[:, :, 0]) np.testing.assert_allclose(y_pis_float2[:, :, 0], y_pis_array[:, :, 1])
def test_results_prefit_naive() -> None: """ Test that prefit, fit and predict on the same dataset is equivalent to the "naive" method. """ estimator = LinearRegression().fit(X, y) mapie_reg = MapieRegressor(estimator=estimator, cv="prefit") mapie_reg.fit(X, y) _, y_pis = mapie_reg.predict(X, alpha=0.05) width_mean = (y_pis[:, 1, 0] - y_pis[:, 0, 0]).mean() coverage = regression_coverage_score(y, y_pis[:, 0, 0], y_pis[:, 1, 0]) np.testing.assert_allclose(width_mean, WIDTHS["naive"], rtol=1e-2) np.testing.assert_allclose(coverage, COVERAGES["naive"], rtol=1e-2)
def test_results_single_and_multi_jobs(strategy: str) -> None: """ Test that MapieRegressor gives equal predictions regardless of number of parallel jobs. """ mapie_single = MapieRegressor(n_jobs=1, **STRATEGIES[strategy]) mapie_multi = MapieRegressor(n_jobs=-1, **STRATEGIES[strategy]) mapie_single.fit(X_toy, y_toy) mapie_multi.fit(X_toy, y_toy) y_pred_single, y_pis_single = mapie_single.predict(X_toy, alpha=0.2) y_pred_multi, y_pis_multi = mapie_multi.predict(X_toy, alpha=0.2) np.testing.assert_allclose(y_pred_single, y_pred_multi) np.testing.assert_allclose(y_pis_single, y_pis_multi)
def test_results_prefit_ignore_method() -> None: """Test that method is ignored when ``cv="prefit"``.""" estimator = LinearRegression().fit(X, y) all_y_pis: List[NDArray] = [] for method in METHODS: mapie_reg = MapieRegressor(estimator=estimator, cv="prefit", method=method) mapie_reg.fit(X, y) _, y_pis = mapie_reg.predict(X, alpha=0.1) all_y_pis.append(y_pis) for y_pis1, y_pis2 in combinations(all_y_pis, 2): np.testing.assert_allclose(y_pis1, y_pis2)
def test_prediction_agg_function(method: str, agg_function: str) -> None: """ Test that predictions differ when ensemble is True/False, but not prediction intervals. """ mapie = MapieRegressor(method=method, cv=2, agg_function=agg_function) mapie.fit(X, y) y_pred_1, y_pis_1 = mapie.predict(X, ensemble=True, alpha=0.1) y_pred_2, y_pis_2 = mapie.predict(X, ensemble=False, alpha=0.1) np.testing.assert_allclose(y_pis_1[:, 0, 0], y_pis_2[:, 0, 0]) np.testing.assert_allclose(y_pis_1[:, 1, 0], y_pis_2[:, 1, 0]) with pytest.raises(AssertionError): np.testing.assert_allclose(y_pred_1, y_pred_2)
def test_linear_regression_results(strategy: str) -> None: """ Test expected prediction intervals for a multivariate linear regression problem with fixed random state. """ mapie = MapieRegressor(**STRATEGIES[strategy]) mapie.fit(X, y) _, y_pis = mapie.predict(X, alpha=0.05) y_pred_low, y_pred_up = y_pis[:, 0, 0], y_pis[:, 1, 0] width_mean = (y_pred_up - y_pred_low).mean() coverage = regression_coverage_score(y, y_pred_low, y_pred_up) np.testing.assert_allclose(width_mean, WIDTHS[strategy], rtol=1e-2) np.testing.assert_allclose(coverage, COVERAGES[strategy], rtol=1e-2)
def test_results_prefit() -> None: """Test prefit results on a standard train/validation/test split.""" X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=1 / 10, random_state=1) X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, test_size=1 / 9, random_state=1) estimator = LinearRegression().fit(X_train, y_train) mapie_reg = MapieRegressor(estimator=estimator, cv="prefit") mapie_reg.fit(X_val, y_val) _, y_pis = mapie_reg.predict(X_test, alpha=0.05) width_mean = (y_pis[:, 1, 0] - y_pis[:, 0, 0]).mean() coverage = regression_coverage_score(y_test, y_pis[:, 0, 0], y_pis[:, 1, 0]) np.testing.assert_allclose(width_mean, WIDTHS["prefit"], rtol=1e-2) np.testing.assert_allclose(coverage, COVERAGES["prefit"], rtol=1e-2)
def test_pipeline_compatibility() -> None: """Check that MAPIE works on pipeline based on pandas dataframes""" X = pd.DataFrame({ "x_cat": ["A", "A", "B", "A", "A", "B"], "x_num": [0, 1, 1, 4, np.nan, 5], "y": [5, 7, 3, 9, 10, 8] }) y = pd.Series([5, 7, 3, 9, 10, 8]) numeric_preprocessor = Pipeline([ ("imputer", SimpleImputer(strategy="mean")), ]) categorical_preprocessor = Pipeline( steps=[("encoding", OneHotEncoder(handle_unknown="ignore"))]) preprocessor = ColumnTransformer([ ("cat", categorical_preprocessor, ["x_cat"]), ("num", numeric_preprocessor, ["x_num"]) ]) pipe = make_pipeline(preprocessor, LinearRegression()) mapie = MapieRegressor(pipe) mapie.fit(X, y) mapie.predict(X)
def compute_PIs( estimator: BaseEstimator, X_train: NDArray, y_train: NDArray, X_test: NDArray, method: str, cv: Any, alpha: float, agg_function: Optional[str] = None, ) -> pd.DataFrame: """ Train and test a model with a MAPIE method, and return a DataFrame of upper and lower bounds of the predictions on the test set. Parameters ---------- estimator : BaseEstimator Base model to fit. X_train : NDArray Features of training set. y_train : NDArray Target of training set. X_test : NDArray Features of testing set. method : str Method for estimating prediction intervals. cv : Any Strategy for computing residuals. alpha : float 1 - (target coverage level). agg_function: str 'mean' or 'median'. Function to aggregate the predictions of the B estimators. Returns ------- pd.DataFrame DataFrame of upper and lower predictions. """ mapie_estimator = MapieRegressor( estimator=estimator, method=method, cv=cv, n_jobs=-1, agg_function=agg_function, ) mapie_estimator = mapie_estimator.fit(X=X_train, y=y_train) _, y_pis = mapie_estimator.predict(X=X_test, alpha=alpha) PI = np.c_[y_pis[:, 0, 0], y_pis[:, 1, 0]] return pd.DataFrame(PI, columns=["lower", "upper"])
def test_results_with_constant_sample_weights(strategy: str) -> None: """ Test predictions when sample weights are None or constant with different values. """ n_samples = len(X) mapie0 = MapieRegressor(**STRATEGIES[strategy]) mapie1 = MapieRegressor(**STRATEGIES[strategy]) mapie2 = MapieRegressor(**STRATEGIES[strategy]) mapie0.fit(X, y, sample_weight=None) mapie1.fit(X, y, sample_weight=np.ones(shape=n_samples)) mapie2.fit(X, y, sample_weight=np.ones(shape=n_samples) * 5) y_pred0, y_pis0 = mapie0.predict(X, alpha=0.05) y_pred1, y_pis1 = mapie1.predict(X, alpha=0.05) y_pred2, y_pis2 = mapie2.predict(X, alpha=0.05) np.testing.assert_allclose(y_pred0, y_pred1) np.testing.assert_allclose(y_pred1, y_pred2) np.testing.assert_allclose(y_pis0, y_pis1) np.testing.assert_allclose(y_pis1, y_pis2)
def PIs_vs_dimensions( strategies: Dict[str, Any], alpha: float, n_trial: int, dimensions: NDArray, ) -> Dict[str, Dict[int, Dict[str, NDArray]]]: """ Compute the prediction intervals for a linear regression problem. Function adapted from Foygel-Barber et al. (2020). It generates several times linear data with random noise whose signal-to-noise is equal to 10 and for several given dimensions, given by the dimensions list. Here we use MAPIE, with a LinearRegression base model, to estimate the width means and the coverage levels of the prediction intervals estimated by all the available strategies as a function of the dataset dimension. This simulation is carried out to emphasize the instability of the prediction intervals estimated by the Jackknife strategy when the dataset dimension is equal to the number of training samples (here 100). Parameters ---------- strategies : Dict[str, Dict[str, Any]] List of strategies for estimating prediction intervals, with corresponding parameters. alpha : float 1 - (target coverage level). n_trial : int Number of trials for each dimension for estimating prediction intervals. For each trial, a new random noise is generated. dimensions : List[int] List of dimension values of input data. Returns ------- Dict[str, Dict[int, Dict[str, NDArray]]] Prediction interval widths and coverages for each strategy, trial, and dimension value. """ n_train = 100 n_test = 100 SNR = 10 results: Dict[str, Dict[int, Dict[str, NDArray]]] = { strategy: { dimension: { "coverage": np.empty(n_trial), "width_mean": np.empty(n_trial), } for dimension in dimensions } for strategy in strategies } for dimension in dimensions: for trial in range(n_trial): beta = np.random.normal(size=dimension) beta_norm = np.sqrt(np.square(beta).sum()) beta = beta / beta_norm * np.sqrt(SNR) X_train = np.random.normal(size=(n_train, dimension)) noise_train = np.random.normal(size=n_train) noise_test = np.random.normal(size=n_test) y_train = X_train.dot(beta) + noise_train X_test = np.random.normal(size=(n_test, dimension)) y_test = X_test.dot(beta) + noise_test for strategy, params in strategies.items(): mapie = MapieRegressor(LinearRegression(), agg_function="median", n_jobs=-1, **params) mapie.fit(X_train, y_train) _, y_pis = mapie.predict(X_test, alpha=alpha) coverage = regression_coverage_score(y_test, y_pis[:, 0, 0], y_pis[:, 1, 0]) results[strategy][dimension]["coverage"][trial] = coverage width_mean = regression_mean_width_score( y_pis[:, 0, 0], y_pis[:, 1, 0]) results[strategy][dimension]["width_mean"][trial] = width_mean return results
("poly", PolynomialFeatures(degree=degree_polyn)), ("linear", LinearRegression()), ]) # Estimating prediction intervals Params = TypedDict("Params", {"method": str, "cv": int}) STRATEGIES = { "jackknife_plus": Params(method="plus", cv=-1), "jackknife_minmax": Params(method="minmax", cv=-1), "cv_plus": Params(method="plus", cv=10), "cv_minmax": Params(method="minmax", cv=10), } y_pred, y_pis = {}, {} for strategy, params in STRATEGIES.items(): mapie = MapieRegressor(polyn_model, **params) mapie.fit(X_train, y_train) y_pred[strategy], y_pis[strategy] = mapie.predict(X_test, alpha=0.05) # Visualization def plot_1d_data( X_train: NDArray, y_train: NDArray, X_test: NDArray, y_test: NDArray, y_sigma: float, y_pred: NDArray, y_pred_low: NDArray, y_pred_up: NDArray, ax: plt.Axes, title: str,
in the Quickstart. """ import numpy as np from matplotlib import pyplot as plt from sklearn.linear_model import LinearRegression from sklearn.datasets import make_regression from mapie.regression import MapieRegressor from mapie.metrics import regression_coverage_score regressor = LinearRegression() X, y = make_regression(n_samples=500, n_features=1, noise=20, random_state=59) alpha = [0.05, 0.32] mapie = MapieRegressor(regressor, method="plus") mapie.fit(X, y) y_pred, y_pis = mapie.predict(X, alpha=alpha) coverage_scores = [ regression_coverage_score(y, y_pis[:, 0, i], y_pis[:, 1, i]) for i, _ in enumerate(alpha) ] plt.xlabel("x") plt.ylabel("y") plt.scatter(X, y, alpha=0.3) plt.plot(X, y_pred, color="C1") order = np.argsort(X[:, 0]) plt.plot(X[order], y_pis[order][:, 0, 1], color="C1", ls="--") plt.plot(X[order], y_pis[order][:, 1, 1], color="C1", ls="--") plt.fill_between(
# Train/validation/test split X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=1 / 10) X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, test_size=1 / 9) # Train model on training set model = MLPRegressor(activation="relu", random_state=1) model.fit(X_train.reshape(-1, 1), y_train) # Calibrate uncertainties on validation set mapie = MapieRegressor(model, cv="prefit") mapie.fit(X_val.reshape(-1, 1), y_val) # Evaluate prediction and coverage level on testing set alpha = 0.1 y_pred, y_pis = mapie.predict(X_test.reshape(-1, 1), alpha=alpha) y_pred_low, y_pred_up = y_pis[:, 0, 0], y_pis[:, 1, 0] coverage = regression_coverage_score(y_test, y_pred_low, y_pred_up) # Plot obtained prediction intervals on testing set theoretical_semi_width = scipy.stats.norm.ppf(1 - alpha) * sigma y_test_theoretical = f(X_test) order = np.argsort(X_test) plt.scatter(X_test, y_test, color="red", alpha=0.3, label="testing", s=2) plt.plot( X_test[order],
"jackknife_plus": Params(method="plus", cv=-1), "jackknife_minmax": Params(method="minmax", cv=-1), "cv": Params(method="base", cv=10), "cv_plus": Params(method="plus", cv=10), "cv_minmax": Params(method="minmax", cv=10), } fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(2, 3, figsize=(3 * 6, 12)) axs = [ax1, ax2, ax3, ax4, ax5, ax6] for i, (strategy, params) in enumerate(STRATEGIES.items()): mapie = MapieRegressor(polyn_model, agg_function="median", n_jobs=-1, **params) mapie.fit(X_train.reshape(-1, 1), y_train) y_pred, y_pis = mapie.predict( X_test.reshape(-1, 1), alpha=0.05, ) plot_1d_data( X_train, y_train, X_test, y_test, y_test_sigma, y_pred, y_pis[:, 0, 0], y_pis[:, 1, 0], axs[i], strategy,
n_iter=n_iter, cv=cv, scoring="neg_root_mean_squared_error", return_train_score=True, verbose=0, random_state=random_state, n_jobs=-1, ) cv_obj.fit(X_train, y_train) best_est = cv_obj.best_estimator_ mapie_non_nested = MapieRegressor(best_est, method="plus", cv=cv, agg_function="median", n_jobs=-1) mapie_non_nested.fit(X_train, y_train) y_pred_non_nested, y_pis_non_nested = mapie_non_nested.predict(X_test, alpha=alpha) widths_non_nested = y_pis_non_nested[:, 1, 0] - y_pis_non_nested[:, 0, 0] coverage_non_nested = regression_coverage_score(y_test, y_pis_non_nested[:, 0, 0], y_pis_non_nested[:, 1, 0]) score_non_nested = mean_squared_error(y_test, y_pred_non_nested, squared=False) # Nested approach with the CV+ strategy using the Random Forest model. cv_obj = RandomizedSearchCV( rf_model, param_distributions=rf_params, n_iter=n_iter, cv=cv, scoring="neg_root_mean_squared_error",
def test_valid_method(method: str) -> None: """Test that valid methods raise no errors.""" mapie_reg = MapieRegressor(method=method) mapie_reg.fit(X_toy, y_toy) check_is_fitted(mapie_reg, mapie_reg.fit_attributes)
def test_valid_agg_function(agg_function: str) -> None: """Test that valid agg_functions raise no errors.""" mapie_reg = MapieRegressor(agg_function=agg_function) mapie_reg.fit(X_toy, y_toy)
def test_not_enough_resamplings() -> None: """Test that a warning is raised if at least one residual is nan.""" with pytest.warns(UserWarning, match=r"WARNING: at least one point of*"): mapie_reg = MapieRegressor(cv=Subsample(n_resamplings=1), agg_function="mean") mapie_reg.fit(X, y)
def test_valid_cv(cv: Any) -> None: """Test that valid cv raise no errors.""" mapie = MapieRegressor(cv=cv) mapie.fit(X_toy, y_toy)