def sentinel_checker():
"""Build :func:`callbacks.bases.lambda_callback` to compare the current `CVExperiment` dataset
values with the expected values of the dataset (train, validation, and holdout) sentinels
Returns
-------
LambdaCallback
Result of :func:`callbacks.bases.lambda_callback` to check `DatasetSentinel` values"""
def on_run_start(
fold_train_input,
fold_train_target,
fold_validation_input,
fold_validation_target,
fold_holdout_input,
fold_holdout_target,
):
#################### Check Train Sentinels ####################
assert fold_train_input.equals(
G.Env.train_input.retrieve_by_sentinel())
assert fold_train_target.equals(
G.Env.train_target.retrieve_by_sentinel())
#################### Check Validation Sentinels ####################
assert fold_validation_input.equals(
G.Env.validation_input.retrieve_by_sentinel())
assert fold_validation_target.equals(
G.Env.validation_target.retrieve_by_sentinel())
#################### Check Holdout Sentinels ####################
assert fold_holdout_input.equals(
G.Env.holdout_input.retrieve_by_sentinel())
assert fold_holdout_target.equals(
G.Env.holdout_target.retrieve_by_sentinel())
return lambda_callback(on_run_start=on_run_start)
def sentinel_checker(cv_scheme):
"""Build :func:`callbacks.bases.lambda_callback` to compare the current `CVExperiment` dataset
values with the expected values of the dataset (train, validation, and holdout) sentinels
Parameters
----------
cv_scheme: Descendant instance of `sklearn.model_selection._split._BaseKFold`
Cross-validation class instance provided to :func:`expected_sentinels`
Returns
-------
LambdaCallback
Result of :func:`callbacks.bases.lambda_callback` to check DatasetSentinel values"""
train_sentinels, validation_sentinels, holdout_sentinels = expected_sentinels(
cv_scheme)
def check_sentinels(
_rep,
_fold,
_run,
#################### Actual Dataset Values ####################
fold_train_input,
fold_train_target,
fold_validation_input,
fold_validation_target,
holdout_input_data,
holdout_target_data,
#################### Current Dataset Sentinels ####################
# These are properties of :class:`environment.Environment`, accessed through `CVExperiment`
train_input,
train_target,
validation_input,
validation_target,
holdout_input,
holdout_target,
):
#################### Check Train Sentinels ####################
assert fold_train_input.equals(train_sentinels[_fold][0])
assert fold_train_target.equals(train_sentinels[_fold][1])
assert fold_train_input.equals(train_input.retrieve_by_sentinel())
assert fold_train_target.equals(train_target.retrieve_by_sentinel())
#################### Check Validation Sentinels ####################
assert fold_validation_input.equals(validation_sentinels[_fold][0])
assert fold_validation_target.equals(validation_sentinels[_fold][1])
assert fold_validation_input.equals(
validation_input.retrieve_by_sentinel())
assert fold_validation_target.equals(
validation_target.retrieve_by_sentinel())
#################### Check Holdout Sentinels ####################
assert holdout_input_data.equals(holdout_sentinels[_fold][0])
assert holdout_target_data.equals(holdout_sentinels[_fold][1])
assert holdout_input_data.equals(holdout_input.retrieve_by_sentinel())
assert holdout_target_data.equals(
holdout_target.retrieve_by_sentinel())
return lambda_callback(on_fold_end=check_sentinels)
def printer_callback():
def printer_helper(_rep, _fold, _run, last_evaluation_results):
print(f"{_rep}.{_fold}.{_run} {last_evaluation_results}")
return lambda_callback(
on_experiment_start=printer_helper,
on_experiment_end=printer_helper,
on_repetition_start=printer_helper,
on_repetition_end=printer_helper,
on_fold_start=printer_helper,
on_fold_end=printer_helper,
on_run_start=printer_helper,
on_run_end=printer_helper,
)
# `model`, which will pass the entire `Model` instance held by our Experiment
def save_model(_rep, _fold, _run, model):
model.model.save_model(f"_saved_models/model_{_rep}_{_fold}_{_run}")
print(f"Model saved for R{_rep} / f{_fold} / r{_run}")
# IMPORTANT: We're using `model.model.save_model` above because the `model` attribute of Experiments
# is a HyperparameterHunter `Model` instance that has another `model` attribute. This nested
# `model` is the actual model we're fitting (`XGBClassifier`).
# XGBoost classes define the `save_model` method to... well, save models...
# For other libraries, we would still use `model.model`, but we would replace `save_model` with
# the method/function that library defines for model saving, or whatever tool you may want to
# use to save your model (Pickle, Joblib, etc.)
cb_save_all = lambda_callback(on_run_end=save_model)
# Now we have our very own callback ready to go!
# There are two places we can tell HyperparameterHunter about our callback:
# 1. `Environment`, with the `experiment_callbacks` kwarg, or
# 2. `CVExperiment`, with the `callbacks` kwarg
# Using `Environment` will automatically tell all our Experiments about the callbacks, whereas
# providing `callbacks` directly to `CVExperiment` restricts callbacks to just that Experiment
# We'll give our `callbacks` to `CVExperiment` because we want to try different callbacks later
##################################################
# Set Up Environment
##################################################
env = Environment(
train_dataset=get_breast_cancer_data("target"),
def dummy_lambda_cb_func():
def _on_fold_start(_rep, _fold, _run):
print(_rep, _fold, _run)
return lambda_callback(on_fold_start=_on_fold_start)