def test_bootstrap_cv_init() -> None:
# filter out warnings triggered by sk-learn/numpy
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
# check erroneous inputs
# - n_splits = 0
with pytest.raises(expected_exception=ValueError):
BootstrapCV(n_splits=0)
# - n_splits < 0
with pytest.raises(expected_exception=ValueError):
BootstrapCV(n_splits=-1)
def test_get_train_test_splits_as_indices() -> None:
n_test_splits = 200
test_x = np.arange(0, 1000, 1)
my_cv = BootstrapCV(n_splits=n_test_splits, random_state=42)
def _generate_splits() -> List[np.ndarray]:
return [test_split for _, test_split in my_cv.split(X=test_x)]
list_of_test_splits = _generate_splits()
# assert we get right amount of splits
assert len(list_of_test_splits) == n_test_splits
# check average ratio of test/train
average_test_size = (
sum(len(test_set) for test_set in list_of_test_splits) / n_test_splits
)
assert 0.35 < average_test_size / len(test_x) < 0.37
list_of_test_splits_2 = _generate_splits()
assert len(list_of_test_splits) == len(
list_of_test_splits_2
), "the number of splits should be stable"
for f1, f2 in zip(list_of_test_splits, list_of_test_splits_2):
assert np.array_equal(f1, f2), "split indices should be stable"
def test_model_inspection_classifier_binary_single_shap_output() -> None:
# simulate some data
x, y = make_classification(
n_samples=200, n_features=5, n_informative=5, n_redundant=0, random_state=42
)
sim_df = pd.DataFrame(
np.hstack((x, y[:, np.newaxis])),
columns=[*(f"f{i}" for i in range(5)), "target"],
)
# create sample object
sample_df = Sample(observations=sim_df, target_name="target")
# fit the crossfit
crossfit = LearnerCrossfit(
pipeline=ClassifierPipelineDF(
classifier=GradientBoostingClassifierDF(random_state=42)
),
cv=BootstrapCV(n_splits=5, random_state=42),
random_state=42,
n_jobs=-3,
).fit(sample_df)
# fit the inspector
LearnerInspector(n_jobs=-3).fit(crossfit=crossfit)
def test_model_ranker(regressor_grids: List[LearnerGrid[RegressorPipelineDF]],
sample: Sample, n_jobs: int) -> None:
expected_scores = [
0.745, 0.742, 0.7, 0.689, 0.675, 0.675, 0.61, 0.61, 0.61, 0.61
]
expected_learners = [
RandomForestRegressorDF,
RandomForestRegressorDF,
AdaBoostRegressorDF,
AdaBoostRegressorDF,
LinearRegressionDF,
LinearRegressionDF,
LGBMRegressorDF,
LGBMRegressorDF,
LGBMRegressorDF,
LGBMRegressorDF,
]
expected_parameters = {
0: dict(regressor__n_estimators=80, regressor__random_state=42),
1: dict(regressor__n_estimators=50, regressor__random_state=42),
2: dict(regressor__n_estimators=50, regressor__random_state=42),
3: dict(regressor__n_estimators=80, regressor__random_state=42),
}
# define the circular cross validator with just 5 splits (to speed up testing)
cv = BootstrapCV(n_splits=5, random_state=42)
ranker: LearnerRanker[RegressorPipelineDF] = LearnerRanker(
grids=regressor_grids, cv=cv, scoring="r2",
n_jobs=n_jobs).fit(sample=sample)
log.debug(f"\n{ranker.summary_report()}")
assert isinstance(ranker.best_model_crossfit_, LearnerCrossfit)
ranking = ranker.ranking_
assert len(ranking) > 0
assert isinstance(ranking[0], LearnerEvaluation)
assert all(ranking_hi.ranking_score >= ranking_lo.ranking_score
for ranking_hi, ranking_lo in zip(ranking, ranking[1:]))
# check if parameters set for estimators actually match expected:
for evaluation in ranker.ranking_:
pipeline_parameters = evaluation.pipeline.get_params()
for name, value in evaluation.parameters.items():
assert (
name in pipeline_parameters
), f"parameter {name} is a parameter in evaluation.pipeline"
assert (pipeline_parameters[name] == value
), f"evaluation.pipeline.{name} is set to {value}"
check_ranking(
ranking=ranker.ranking_,
expected_scores=expected_scores,
expected_learners=expected_learners,
expected_parameters=expected_parameters,
)
def test_model_ranker_no_preprocessing(n_jobs) -> None:
expected_learner_scores = [0.943, 0.913, 0.913, 0.884]
# define a yield-engine circular CV:
cv = BootstrapCV(n_splits=5, random_state=42)
# define parameters and pipeline
models = [
LearnerGrid(
pipeline=ClassifierPipelineDF(classifier=SVCDF(gamma="scale"),
preprocessing=None),
learner_parameters={
"kernel": ["linear", "rbf"],
"C": [1, 10]
},
)
]
# load scikit-learn test-data and convert to pd
iris = datasets.load_iris()
test_data = pd.DataFrame(
data=np.c_[iris["data"], iris["target"]],
columns=[*iris["feature_names"], "target"],
)
test_sample: Sample = Sample(observations=test_data, target_name="target")
model_ranker: LearnerRanker[ClassifierPipelineDF[SVCDF]] = LearnerRanker(
grids=models, cv=cv, n_jobs=n_jobs).fit(sample=test_sample)
log.debug(f"\n{model_ranker.summary_report()}")
check_ranking(
ranking=model_ranker.ranking_,
expected_scores=expected_learner_scores,
expected_learners=[SVCDF] * 4,
expected_parameters={
0: dict(classifier__C=10, classifier__kernel="linear"),
3: dict(classifier__C=1, classifier__kernel="rbf"),
},
)
assert (model_ranker.ranking_[0].ranking_score >=
0.8), "expected a best performance of at least 0.8"
def test_bootstrap_cv_with_sk_learn() -> None:
# filter out warnings triggered by sk-learn/numpy
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
# load example data
iris = datasets.load_iris()
# define a yield-engine circular CV:
my_cv = BootstrapCV(n_splits=50)
# define parameters and pipeline
parameters = {"kernel": ("linear", "rbf"), "C": [1, 10]}
svc = svm.SVC(gamma="scale")
# use the defined my_cv bootstrap CV within GridSearchCV:
if __sklearn_version__ < __sklearn_0_22__:
clf = GridSearchCV(svc, parameters, cv=my_cv, iid=False)
else:
clf = GridSearchCV(svc, parameters, cv=my_cv)
clf.fit(iris.data, iris.target)
# test if the number of received splits is correct:
assert (
clf.n_splits_ == 50
), "50 splits should have been generated by the bootstrap CV"
assert clf.best_score_ > 0.85, "Expected a minimum score of 0.85"
# define new parameters and a different pipeline
# use the defined my_cv circular CV again within GridSeachCV:
parameters = {
"criterion": ("gini", "entropy"),
"max_features": ["sqrt", "auto", "log2"],
}
cl2 = GridSearchCV(tree.DecisionTreeClassifier(), parameters, cv=my_cv)
cl2.fit(iris.data, iris.target)
assert cl2.best_score_ > 0.85, "Expected a minimum score of 0.85"
def cv_bootstrap() -> BaseCrossValidator:
# define a CV
return BootstrapCV(n_splits=N_BOOTSTRAPS, random_state=42)