def get_coverage_width(PIs: pd.DataFrame, y: NDArray) -> Tuple[float, float]:
"""
Computes the mean coverage and width of the predictions intervals of a
DataFrame given by the ``compute_PIs`` function
Parameters
----------
PIs : pd.DataFrame
DataFrame returned by `compute_PIs``, with lower and upper bounds of
the PIs.
y : NDArray
Targets supposedly covered by the PIs.
Returns
-------
(coverage, width) : Tuple[float, float]
The mean coverage and width of the PIs.
"""
coverage = regression_coverage_score(y_true=y,
y_pred_low=PIs["lower"],
y_pred_up=PIs["upper"])
width = regression_mean_width_score(y_pred_low=PIs["lower"],
y_pred_up=PIs["upper"])
return (coverage, width)
def PIs_vs_dimensions(
strategies: Dict[str, Any],
alpha: float,
n_trial: int,
dimensions: NDArray,
) -> Dict[str, Dict[int, Dict[str, NDArray]]]:
"""
Compute the prediction intervals for a linear regression problem.
Function adapted from Foygel-Barber et al. (2020).
It generates several times linear data with random noise whose
signal-to-noise is equal to 10 and for several given dimensions,
given by the dimensions list.
Here we use MAPIE, with a LinearRegression base model, to estimate
the width means and the coverage levels of the prediction intervals
estimated by all the available strategies as a function of
the dataset dimension.
This simulation is carried out to emphasize the instability
of the prediction intervals estimated by the Jackknife strategy
when the dataset dimension is equal to the number
of training samples (here 100).
Parameters
----------
strategies : Dict[str, Dict[str, Any]]
List of strategies for estimating prediction intervals,
with corresponding parameters.
alpha : float
1 - (target coverage level).
n_trial : int
Number of trials for each dimension for estimating
prediction intervals.
For each trial, a new random noise is generated.
dimensions : List[int]
List of dimension values of input data.
Returns
-------
Dict[str, Dict[int, Dict[str, NDArray]]]
Prediction interval widths and coverages for each strategy, trial,
and dimension value.
"""
n_train = 100
n_test = 100
SNR = 10
results: Dict[str, Dict[int, Dict[str, NDArray]]] = {
strategy: {
dimension: {
"coverage": np.empty(n_trial),
"width_mean": np.empty(n_trial),
}
for dimension in dimensions
}
for strategy in strategies
}
for dimension in dimensions:
for trial in range(n_trial):
beta = np.random.normal(size=dimension)
beta_norm = np.sqrt(np.square(beta).sum())
beta = beta / beta_norm * np.sqrt(SNR)
X_train = np.random.normal(size=(n_train, dimension))
noise_train = np.random.normal(size=n_train)
noise_test = np.random.normal(size=n_test)
y_train = X_train.dot(beta) + noise_train
X_test = np.random.normal(size=(n_test, dimension))
y_test = X_test.dot(beta) + noise_test
for strategy, params in strategies.items():
mapie = MapieRegressor(LinearRegression(),
agg_function="median",
n_jobs=-1,
**params)
mapie.fit(X_train, y_train)
_, y_pis = mapie.predict(X_test, alpha=alpha)
coverage = regression_coverage_score(y_test, y_pis[:, 0, 0],
y_pis[:, 1, 0])
results[strategy][dimension]["coverage"][trial] = coverage
width_mean = regression_mean_width_score(
y_pis[:, 0, 0], y_pis[:, 1, 0])
results[strategy][dimension]["width_mean"][trial] = width_mean
return results
def test_regression_same_length() -> None:
"Test when y_true and y_preds have different lengths."
with pytest.raises(ValueError, match=r".*could not be broadcast*"):
regression_coverage_score(y_toy, y_preds[:-1, 1], y_preds[:-1, 2])
with pytest.raises(ValueError, match=r".*y should be a 1d array*"):
regression_mean_width_score(y_preds[:, :2], y_preds[:, 2])
def test_regression_ypredup_shape() -> None:
"Test shape of y_pred_up."
with pytest.raises(ValueError, match=r".*y should be a 1d array*"):
regression_coverage_score(y_toy, y_preds[:, 1], y_preds[:, 1:])
with pytest.raises(ValueError, match=r".*y should be a 1d array*"):
regression_mean_width_score(y_preds[:, :2], y_preds[:, 2])
def test_regression_ypredup_type_mean_width_score() -> None:
"Test that list(y_pred_up) gives right coverage."
scr = regression_mean_width_score(y_preds[:, 1], list(y_preds[:, 2]))
assert scr == 2.3
def test_regression_ypredlow_type_mean_width_score() -> None:
"Test that list(y_pred_low) gives right coverage."
scr = regression_mean_width_score(list(y_preds[:, 1]), y_preds[:, 2])
assert scr == 2.3
def test_regression_toydata_mean_width_score() -> None:
"Test mean_width_score for toy data."
scr = regression_mean_width_score(y_preds[:, 1], y_preds[:, 2])
assert scr == 2.3
