def execute():
#################### Environment ####################
env = Environment(
train_dataset=get_boston_data(),
results_path="HyperparameterHunterAssets",
holdout_dataset=get_holdout_data,
target_column="DIS",
metrics=["r2_score", "median_absolute_error"],
cv_type="KFold",
cv_params=dict(n_splits=10, random_state=1),
)
#################### CVExperiment ####################
exp_0 = CVExperiment(
model_initializer=Ridge,
model_init_params=dict(),
feature_engineer=FeatureEngineer([quantile_transform]),
)
#################### Optimization ####################
# `opt_0` recognizes `exp_0`'s `feature_engineer` and its results as valid learning material
# This is because `opt_0` marks the engineer step functions omitted by `exp_0` as `optional=True`
opt_0 = DummyOptPro(iterations=10)
opt_0.forge_experiment(
model_initializer=Ridge,
model_init_params=dict(),
feature_engineer=FeatureEngineer([
Categorical([quantile_transform, log_transform], optional=True),
Categorical([standard_scale, standard_scale_BAD], optional=True),
Categorical([square_sum_feature], optional=True),
]),
)
opt_0.go()
def run_initialization_matching_optimization_0(build_fn):
optimizer = DummyOptPro(iterations=1)
optimizer.forge_experiment(
model_initializer=KerasClassifier,
model_init_params=dict(build_fn=build_fn),
model_extra_params=dict(epochs=1, batch_size=128, verbose=0),
)
optimizer.go()
return optimizer
def opt_regressor():
optimizer = DummyOptPro(iterations=1)
optimizer.forge_experiment(
model_initializer=KerasRegressor,
model_init_params=_build_fn_regressor,
model_extra_params=dict(
callbacks=[ReduceLROnPlateau(patience=Integer(5, 10))],
batch_size=Categorical([32, 64], transform="onehot"),
epochs=10,
verbose=0,
),
)
optimizer.go()
def test_optional_step_matching(env_boston, feature_engineer):
"""Tests that a Space containing `optional` `Categorical` Feature Engineering steps matches with
the expected saved Experiments. This regression test is focused on issues that arise when
`EngineerStep`s other than the last one in the `FeatureEngineer` are `optional`. The simplified
version of this test below, :func:`test_limited_optional_step_matching`, demonstrates that
result matching works properly when only the final `EngineerStep` is `optional`"""
opt_0 = DummyOptPro(iterations=20, random_state=32)
opt_0.forge_experiment(XGBRegressor, feature_engineer=feature_engineer)
opt_0.go()
opt_1 = ExtraTreesOptPro(iterations=20, random_state=32)
opt_1.forge_experiment(XGBRegressor, feature_engineer=feature_engineer)
opt_1.get_ready()
# Assert `opt_1` matched with all Experiments executed by `opt_0`
assert len(opt_1.similar_experiments) == opt_0.successful_iterations
def test_limited_optional_step_matching(env_boston, feature_engineer):
"""Simplified counterpart to above :func:`test_optional_step_matching`. Tests that a Space
containing `Categorical` Feature Engineering steps -- of which only the last ones may be
`optional` -- matches with the expected saved Experiments. These test cases do not demonstrate
the same bug being regression-tested by `test_optional_step_matching`. Instead, this test
function exists to ensure that the areas close to the above bug are behaving properly and to
help define the bug being tested by `test_optional_step_matching`. This function demonstrates
that `optional` is not problematic when used only in the final `EngineerStep`"""
opt_0 = DummyOptPro(iterations=20, random_state=32)
opt_0.forge_experiment(XGBRegressor, feature_engineer=feature_engineer)
opt_0.go()
opt_1 = ExtraTreesOptPro(iterations=20, random_state=32)
opt_1.forge_experiment(XGBRegressor, feature_engineer=feature_engineer)
opt_1.get_ready()
# Assert `opt_1` matched with all Experiments executed by `opt_0`
assert len(opt_1.similar_experiments) == opt_0.successful_iterations
allow_writing_files=False,
),
model_extra_params=dict(
fit=dict(verbose=50, eval_set=[(env.validation_input, env.validation_target)])
),
)
# Notice above that CatBoost printed scores for our `eval_set` every 50 iterations just like we said
# ... in `model_extra_params["fit"]`; although, it made our results rather difficult to read, so
# ... we'll switch back to `verbose=False` during optimization.
# And/or...
#################### 2. Hyperparameter Optimization ####################
# Notice below that `optimizer` still recognizes the results of `experiment` as valid learning material even
# ... though their `verbose` values differ. This is because it knows that `verbose` has no effect on actual results.
optimizer = DummyOptPro(iterations=10, random_state=777)
optimizer.forge_experiment(
model_initializer=CatBoostRegressor,
model_init_params=dict(
iterations=100,
learning_rate=Real(0.001, 0.2),
depth=Integer(3, 7),
bootstrap_type=Categorical(["Bayesian", "Bernoulli"]),
save_snapshot=False,
allow_writing_files=False,
),
model_extra_params=dict(
fit=dict(verbose=False, eval_set=[(env.validation_input, env.validation_target)])
),
)
optimizer.go()