def test_regression_bootstrap_sampled_hyperpar_tune(self):
"""Compares the single and multi hyperparameter tuning."""
# Single hyperparameter tune prior to bootstrapping.
kwargs = {'data': self.data,
'target': self.target,
'bootstraps': 5}
elastic_net = linear_model.ElasticNet(random_state=1)
elastic_net_cv = linear_model.ElasticNetCV(random_state=10, cv=3)
outer_tune = bootstrap.regression_bootstrap(
regressor=elastic_net, regressor_cv=elastic_net_cv, **kwargs)
outer_coef_std = outer_tune.std(axis=0).mean()
# Hyperparameters re-tuned on every bootstrap sample.
elastic_net = linear_model.ElasticNetCV(random_state=10, cv=3)
elastic_net_cv = None
outer_inner_tune = bootstrap.regression_bootstrap(
regressor=elastic_net, regressor_cv=elastic_net_cv, **kwargs)
outer_inner_coef_std = outer_inner_tune.std(axis=0).mean()
# Confirm that running separate instances gives same results for single
# tune. This is identical setup to outer_tune.
elastic_net = linear_model.ElasticNet(random_state=1)
elastic_net_cv = linear_model.ElasticNetCV(random_state=10, cv=3)
outer_tune2 = bootstrap.regression_bootstrap(
regressor=elastic_net, regressor_cv=elastic_net_cv, **kwargs)
outer2_coef_std = outer_tune2.std(axis=0).mean()
self.assertNotEqual(outer_coef_std, outer_inner_coef_std)
self.assertEqual(outer_coef_std, outer2_coef_std)
def test_regression_bootstrap_sample_frac_valueerror(self, sample_frac):
linear = linear_model.LinearRegression()
with self.assertRaises(ValueError):
bootstrap.regression_bootstrap(
self.data, self.target, linear,
regressor_cv=None,
bootstraps=5,
sample_frac=sample_frac)
def test_regression_bootstrap_runtime_error(
self, regressor, regressor_cv):
with self.assertRaises(RuntimeError):
bootstrap.regression_bootstrap(
self.data,
self.target,
regressor=regressor,
regressor_cv=regressor_cv)
def test_regression_bootstrap_unsupported_regressor_cv(
self, regressor, regressor_cv):
with self.assertRaises(NotImplementedError):
bootstrap.regression_bootstrap(
self.data,
self.target,
regressor=regressor,
regressor_cv=regressor_cv)
def test_regression_bootstrap_mismatch_regressor_cv(
self, regressor, regressor_cv, bootstraps, n_jobs):
with self.assertRaises(ValueError):
bootstrap.regression_bootstrap(
data=self.data,
target=self.target,
regressor=regressor,
regressor_cv=regressor_cv,
bootstraps=bootstraps,
n_jobs=n_jobs,
verbose=False)
def test_regression_bootstrap_without_replacement(self):
"""Compares results with and without replacement."""
kwargs = {'data': self.data,
'target': self.target,
'regressor': linear_model.Ridge(),
'regressor_cv': None,
'sample_frac': 0.8,
'bootstraps': 5}
replacement_result = bootstrap.regression_bootstrap(
replacement=True, **kwargs)
without_replacement_result = bootstrap.regression_bootstrap(
replacement=False, **kwargs)
# Results should be different as data has been resampled differently.
self.assertFalse(replacement_result.equals(without_replacement_result))
def test_regression_bootstrap_sample_frac(self, sample_frac):
linear = linear_model.LinearRegression()
result = bootstrap.regression_bootstrap(
self.data, self.target, linear,
regressor_cv=None,
bootstraps=5)
coef_std = result.std(axis=0).mean()
result_sampled = bootstrap.regression_bootstrap(
self.data, self.target, linear,
regressor_cv=None,
bootstraps=5,
sample_frac=sample_frac)
coef_std_sampled = result_sampled.std(axis=0).mean()
self.assertNotEqual(coef_std, coef_std_sampled)
def test_classification_bootstrap(self):
ridge_class = linear_model.RidgeClassifier()
ridge_class_cv = linear_model.RidgeClassifierCV()
result = bootstrap.regression_bootstrap(
data=self.data,
target=self.class_target,
regressor=ridge_class,
regressor_cv=ridge_class_cv,
verbose=False,
bootstraps=5)
self.assertIsInstance(result, pd.DataFrame)
self.assertEqual(result.shape[1], self.data.shape[1]+1)
def test_regression_bootstrap(
self, regressor, regressor_cv, bootstraps, n_jobs):
result = bootstrap.regression_bootstrap(
data=self.data,
target=self.target,
regressor=regressor,
regressor_cv=regressor_cv,
bootstraps=bootstraps,
n_jobs=n_jobs,
verbose=False)
self.assertIsInstance(result, pd.DataFrame)
self.assertLen(result, bootstraps) # Same rows as many bootstraps
self.assertEqual(result.shape[1], self.data.shape[1]+1)
def fit_bootstrap(
self,
bootstraps: int = 1000,
n_jobs: int = 1,
verbose: bool = True,
) -> None:
"""Runs a bootstrap iteration to estimate confidence intervals.
Args:
bootstraps: Number of bootstraps to perform.
n_jobs: If use multiple CPUs during the bootstrap. Default value is `1`
for no multiprocessing. You can specify the number of CPUs or specify
negative values will all CPUs available minus the value selected.
verbose: If True, will print to screen CV results and estimated completion
time for bootstrap.
Raises:
RuntimeError: If the object has not being fit before calling this method.
"""
if not self._is_fit:
raise RuntimeError(
'InferenceModel must be fit before running bootstrap.')
modelling_data, target = self._data.get_data_and_target()
# Backup coefficients and intercept.
coefficients = self.model.coef_.copy()
if self.model.fit_intercept:
intercept = self.model.intercept_.copy()
try:
self._bootstrap_results = bootstrap.regression_bootstrap(
data=modelling_data,
target=target,
regressor=self.model,
regressor_cv=None,
bootstraps=bootstraps,
n_jobs=n_jobs,
verbose=verbose)
finally:
# Restore coefficients and intercept.
self.model.coef_ = coefficients
if self.model.fit_intercept:
self.model.intercept_ = intercept