def execute(): #################### Environment #################### env = Environment( train_dataset=get_boston_data(), results_path="HyperparameterHunterAssets", holdout_dataset=get_holdout_data, target_column="DIS", metrics=["r2_score", "median_absolute_error"], cv_type="KFold", cv_params=dict(n_splits=10, random_state=1), ) #################### CVExperiment #################### exp_0 = CVExperiment( model_initializer=Ridge, model_init_params=dict(), feature_engineer=FeatureEngineer([quantile_transform]), ) #################### Optimization #################### # `opt_0` recognizes `exp_0`'s `feature_engineer` and its results as valid learning material # This is because `opt_0` marks the engineer step functions omitted by `exp_0` as `optional=True` opt_0 = DummyOptPro(iterations=10) opt_0.forge_experiment( model_initializer=Ridge, model_init_params=dict(), feature_engineer=FeatureEngineer([ Categorical([quantile_transform, log_transform], optional=True), Categorical([standard_scale, standard_scale_BAD], optional=True), Categorical([square_sum_feature], optional=True), ]), ) opt_0.go()
def run_initialization_matching_optimization_0(build_fn): optimizer = DummyOptPro(iterations=1) optimizer.forge_experiment( model_initializer=KerasClassifier, model_init_params=dict(build_fn=build_fn), model_extra_params=dict(epochs=1, batch_size=128, verbose=0), ) optimizer.go() return optimizer
def opt_regressor(): optimizer = DummyOptPro(iterations=1) optimizer.forge_experiment( model_initializer=KerasRegressor, model_init_params=_build_fn_regressor, model_extra_params=dict( callbacks=[ReduceLROnPlateau(patience=Integer(5, 10))], batch_size=Categorical([32, 64], transform="onehot"), epochs=10, verbose=0, ), ) optimizer.go()
def test_optional_step_matching(env_boston, feature_engineer): """Tests that a Space containing `optional` `Categorical` Feature Engineering steps matches with the expected saved Experiments. This regression test is focused on issues that arise when `EngineerStep`s other than the last one in the `FeatureEngineer` are `optional`. The simplified version of this test below, :func:`test_limited_optional_step_matching`, demonstrates that result matching works properly when only the final `EngineerStep` is `optional`""" opt_0 = DummyOptPro(iterations=20, random_state=32) opt_0.forge_experiment(XGBRegressor, feature_engineer=feature_engineer) opt_0.go() opt_1 = ExtraTreesOptPro(iterations=20, random_state=32) opt_1.forge_experiment(XGBRegressor, feature_engineer=feature_engineer) opt_1.get_ready() # Assert `opt_1` matched with all Experiments executed by `opt_0` assert len(opt_1.similar_experiments) == opt_0.successful_iterations
def test_limited_optional_step_matching(env_boston, feature_engineer): """Simplified counterpart to above :func:`test_optional_step_matching`. Tests that a Space containing `Categorical` Feature Engineering steps -- of which only the last ones may be `optional` -- matches with the expected saved Experiments. These test cases do not demonstrate the same bug being regression-tested by `test_optional_step_matching`. Instead, this test function exists to ensure that the areas close to the above bug are behaving properly and to help define the bug being tested by `test_optional_step_matching`. This function demonstrates that `optional` is not problematic when used only in the final `EngineerStep`""" opt_0 = DummyOptPro(iterations=20, random_state=32) opt_0.forge_experiment(XGBRegressor, feature_engineer=feature_engineer) opt_0.go() opt_1 = ExtraTreesOptPro(iterations=20, random_state=32) opt_1.forge_experiment(XGBRegressor, feature_engineer=feature_engineer) opt_1.get_ready() # Assert `opt_1` matched with all Experiments executed by `opt_0` assert len(opt_1.similar_experiments) == opt_0.successful_iterations
allow_writing_files=False, ), model_extra_params=dict( fit=dict(verbose=50, eval_set=[(env.validation_input, env.validation_target)]) ), ) # Notice above that CatBoost printed scores for our `eval_set` every 50 iterations just like we said # ... in `model_extra_params["fit"]`; although, it made our results rather difficult to read, so # ... we'll switch back to `verbose=False` during optimization. # And/or... #################### 2. Hyperparameter Optimization #################### # Notice below that `optimizer` still recognizes the results of `experiment` as valid learning material even # ... though their `verbose` values differ. This is because it knows that `verbose` has no effect on actual results. optimizer = DummyOptPro(iterations=10, random_state=777) optimizer.forge_experiment( model_initializer=CatBoostRegressor, model_init_params=dict( iterations=100, learning_rate=Real(0.001, 0.2), depth=Integer(3, 7), bootstrap_type=Categorical(["Bayesian", "Bernoulli"]), save_snapshot=False, allow_writing_files=False, ), model_extra_params=dict( fit=dict(verbose=False, eval_set=[(env.validation_input, env.validation_target)]) ), ) optimizer.go()