def test_feature_engineer_list_optimization_equality(env_boston, steps_0,
steps_1):
"""Test that the `feature_engineer` attribute constructed by an OptPro is the same whether given
a list as input, or a :class:`~hyperparameter_hunter.feature_engineering.FeatureEngineer`"""
opt_0, opt_1, opt_2, opt_3 = GBRT(), GBRT(), GBRT(), GBRT()
opt_0.forge_experiment(Ridge, feature_engineer=steps_0)
opt_1.forge_experiment(Ridge, feature_engineer=FeatureEngineer(steps_1))
assert opt_0.feature_engineer == opt_1.feature_engineer
# Repeat above, but switch which steps are wrapped in `FeatureEngineer`
opt_2.forge_experiment(Ridge, feature_engineer=steps_1)
opt_3.forge_experiment(Ridge, feature_engineer=FeatureEngineer(steps_0))
assert opt_2.feature_engineer == opt_3.feature_engineer
def test_feature_engineer_list_optimization_inequality(env_boston, steps_0,
steps_1):
"""Test that the `feature_engineer` attribute constructed by an OptPro is NOT the same when
given a list as input vs. a :class:`~hyperparameter_hunter.feature_engineering.FeatureEngineer`
when the two are actually different. This is an insanity test to make sure that the related
test in this module, :func:`test_feature_engineer_list_optimization_equality`, is not simply
equating everything"""
opt_0, opt_1, opt_2, opt_3 = GBRT(), GBRT(), GBRT(), GBRT()
opt_0.forge_experiment(Ridge, feature_engineer=steps_0)
opt_1.forge_experiment(Ridge, feature_engineer=FeatureEngineer(steps_1))
assert opt_0.feature_engineer != opt_1.feature_engineer
# Repeat above, but switch which steps are wrapped in `FeatureEngineer`
opt_2.forge_experiment(Ridge, feature_engineer=steps_1)
opt_3.forge_experiment(Ridge, feature_engineer=FeatureEngineer(steps_0))
assert opt_2.feature_engineer != opt_3.feature_engineer
def test_sentinels_optimization(env_0):
optimizer = GBRT(iterations=2)
optimizer.set_experiment_guidelines(
model_initializer=XGBClassifier,
model_init_params=dict(objective="reg:linear",
max_depth=Integer(2, 20),
subsample=0.5),
model_extra_params=dict(fit=dict(
eval_set=get_all_sentinels(env_0),
early_stopping_rounds=5,
eval_metric=Categorical(["auc", "mae"]),
)),
)
optimizer.go()
cv_type="KFold",
cv_params=dict(n_splits=12, shuffle=True, random_state=32),
runs=2,
)
# Now that HyperparameterHunter has an active `Environment`, we can do two things:
#################### 1. Perform Experiments ####################
experiment = CVExperiment(
model_initializer=XGBRegressor,
model_init_params=dict(max_depth=4, n_estimators=400, subsample=0.5),
model_extra_params=dict(fit=dict(eval_metric="mae")),
)
# And/or...
#################### 2. Hyperparameter Optimization ####################
optimizer = GBRT(iterations=20, random_state=32)
optimizer.forge_experiment(
model_initializer=XGBRegressor,
model_init_params=dict(
max_depth=Integer(2, 20),
n_estimators=Integer(100, 900),
learning_rate=Real(0.0001, 0.5),
subsample=0.5,
booster=Categorical(["gbtree", "gblinear"]),
),
model_extra_params=dict(fit=dict(eval_metric=Categorical(["rmse", "mae"]))),
)
optimizer.go()
# Notice, `optimizer` recognizes our earlier `experiment`'s hyperparameters fit inside the search
# space/guidelines set for `optimizer`.
experiment = CVExperiment(
model_initializer=CatBoostClassifier,
model_init_params=dict(
iterations=100,
learning_rate=0.03,
depth=6,
save_snapshot=False,
allow_writing_files=False,
loss_function="MultiClass",
classes_count=7,
),
)
# And/or...
#################### 2. Hyperparameter Optimization ####################
optimizer = GBRT(iterations=8, random_state=42)
optimizer.set_experiment_guidelines(
model_initializer=CatBoostClassifier,
model_init_params=dict(
iterations=100,
learning_rate=Real(low=0.0001, high=0.5),
depth=Integer(4, 15),
save_snapshot=False,
allow_writing_files=False,
loss_function="MultiClass",
classes_count=7,
),
)
optimizer.go()
# Notice, `optimizer` recognizes our earlier `experiment`'s hyperparameters fit inside the search