def env_3():
def printer_callback():
def printer_helper(_rep, _fold, _run, last_evaluation_results):
print(f"{_rep}.{_fold}.{_run} {last_evaluation_results}")
return lambda_callback(
on_experiment_start=printer_helper,
on_experiment_end=printer_helper,
on_repetition_start=printer_helper,
on_repetition_end=printer_helper,
on_fold_start=printer_helper,
on_fold_end=printer_helper,
on_run_start=printer_helper,
on_run_end=printer_helper,
)
return Environment(
train_dataset=get_toy_classification_data(),
results_path=assets_dir,
metrics=["roc_auc_score"],
holdout_dataset=get_toy_classification_data(),
cv_type=RepeatedStratifiedKFold,
cv_params=dict(n_splits=3, n_repeats=2, random_state=32),
runs=2,
experiment_callbacks=[
printer_callback(),
confusion_matrix_oof(),
confusion_matrix_holdout(),
],
)
def execute():
env = Environment(
train_dataset=get_toy_classification_data(),
results_path="HyperparameterHunterAssets",
metrics=["roc_auc_score"],
cv_type=RepeatedStratifiedKFold,
cv_params=dict(n_splits=5, n_repeats=2, random_state=32),
runs=2,
# Just instantiate `Environment` with your list of callbacks, and go about business as usual
experiment_callbacks=[printer_callback(),
confusion_matrix_oof()],
# In addition to `printer_callback` made above, we're also adding the `confusion_matrix_oof` callback
# This, and other callbacks, can be found in `hyperparameter_hunter.callbacks.recipes`
)
experiment = CVExperiment(
model_initializer=XGBClassifier,
model_init_params={},
model_extra_params=dict(fit=dict(verbose=False)),
)
#################### `cv_type` ####################
@pytest.mark.parametrize("new_value",
["RepeatedStratifiedKFold", RepeatedStratifiedKFold])
def test_cv_type_setter_valid(env_fixture_0, new_value):
env_fixture_0.cv_type = new_value
assert env_fixture_0.cv_type == RepeatedStratifiedKFold
def test_cv_type_setter_attribute_error(env_fixture_0):
with pytest.raises(AttributeError,
match="'foo' not in `sklearn.model_selection._split`"):
env_fixture_0.cv_type = "foo"
#################### `experiment_callbacks` ####################
cm_oof, cm_holdout = confusion_matrix_oof(), confusion_matrix_holdout()
@pytest.mark.parametrize(
["new_value", "expected"],
[
([], []),
([cm_oof], [cm_oof]),
(cm_oof, [cm_oof]),
([cm_oof, cm_holdout], [cm_oof, cm_holdout]),
],
)
def test_experiment_callbacks_setter_valid(env_fixture_0, new_value, expected):
env_fixture_0.experiment_callbacks = new_value
assert env_fixture_0.experiment_callbacks == expected
print("on_run_start", self._rep, self._fold, self._run)
def on_run_end(self):
print("on_run_end", self._rep, self._fold, self._run)
##################################################
# Test `lambda_callback` Provision
##################################################
# noinspection PyUnusedLocal
@pytest.mark.parametrize(
"lambda_cbs",
[
dummy_lambda_cb_func(),
[dummy_lambda_cb_func()],
[dummy_lambda_cb_func(), confusion_matrix_oof()],
[dummy_lambda_cb_func(), DummyLambdaCallbackClass, confusion_matrix_oof()],
],
)
def test_provide_lambda_callbacks(lambda_cbs):
"""Test that each of the officially-supported methods of providing LambdaCallbacks to an
Experiment yields the same MRO. Specifically concerned with using the `experiment_callbacks`
kwarg of :class:`~hyperparameter_hunter.environment.Environment` and using the `callbacks`
kwarg of :class:`~hyperparameter_hunter.experiments.CVExperiment`. Also sanity check that MROs
of Experiments with LambdaCallbacks actually differ from the MRO of a basic Experiment
Parameters
----------
lambda_cbs: `LambdaCallback`, or list of `LambdaCallback`
LambdaCallback values passed to the different methods of `lambda_callback` provision"""
#################### Via `Environment`'s `experiment_callbacks` ####################