def test_SE_retraining_fails_when_param_disabled(): print("\n=== disabling "+kcvp+" and retraining ===") total_runs = 4 aml = setup_and_train(False) # first run first_models = get_partitioned_model_names(aml.leaderboard) first_bof = next(m for m in first_models.se if re.search(r'_BestOfFamily_', m)) ds = import_dataset() for i in range(total_runs - 1): aml.train(y=ds.target, training_frame=ds.train) models = get_partitioned_model_names(aml.leaderboard) first_se_all_models = [m for m in first_models.se if re.search(r'_AllModels_', m)] se_all_models = [m for m in models.se if re.search(r'_AllModels_', m)] se_best_of_family = [m for m in models.se if re.search(r'_BestOfFamily_', m)] lb = aml.leaderboard print(lb.head(lb.nrows)) assert len(models.se) == len(se_all_models) + len(se_best_of_family) assert len(se_all_models) == len(first_se_all_models), \ "expecting only the {} first StackedEnsemble_AllModels, but got {}".format(len(first_se_all_models), len(se_all_models)) assert se_all_models[0] in first_models.se, "first StackedEnsemble_AllModels got replaced by new one" if len(se_best_of_family) > 1: assert first_bof in se_best_of_family, "first StackedEnsemble_BestOfFamily disappeared after multiple runs" row_of = lambda id: lb[lb['model_id'] == id] first_bof_row = row_of(first_bof) assert all(all(row[i] == first_bof_row[i] for i in range(1, lb.ncols)) for row in [row_of(se) for se in se_best_of_family]), \ "expecting possibly 2+ similar StackedEnsemble_BestOfFamily (corner case), but managed to obtain 2 different ones!" else: assert len(se_best_of_family) == 1, "expecting only the first StackedEnsemble_BestOfFamily, but got {}".format(len(se_best_of_family)) assert se_best_of_family[0] == first_bof, "first StackedEnsemble_Best_of_Family got replaced by new one"
def test_AUTO_stopping_metric_with_no_sorting_metric_regression(): print( "Check leaderboard with AUTO stopping metric and no sorting metric for regression" ) ds = import_dataset('regression', split=False) exclude_algos = ["DeepLearning", "GLM"] aml = H2OAutoML( project_name= "py_aml_lb_test_auto_stopping_metric_no_sorting_regression", exclude_algos=exclude_algos, max_models=10, nfolds=2, stopping_rounds=1, stopping_tolerance=0.5, seed=automl_seed) aml.train(y=ds.target, training_frame=ds.train) check_leaderboard( aml, exclude_algos, ["rmse", "mse", "mae", "rmsle", "mean_residual_deviance"], "rmse") base = get_partitioned_model_names(aml.leaderboard).base first = [m for m in base if 'XGBoost_1' in m] others = [m for m in base if m not in first] check_model_property( first, 'stopping_metric', True, None ) #if stopping_rounds == 0, actual value of stopping_metric is set to None check_model_property(others, 'stopping_metric', True, "deviance")
def test_algo_parameter_can_be_applied_only_to_a_specific_algo(): ds = import_dataset() aml = H2OAutoML( project_name="py_specific_algo_param", algo_parameters=dict(GBM__monotone_constraints=dict(AGE=1)), max_models=6, seed=1) aml.train(y=ds.target, training_frame=ds.train) model_names = get_partitioned_model_names(aml.leaderboard).all models_supporting_monotone_constraints = [ n for n in model_names if re.match(r"GBM|XGBoost", n) ] assert next((m for m in models_supporting_monotone_constraints if m.startswith('GBM')), None), "There should be at least one GBM model" for m in models_supporting_monotone_constraints: model = h2o.get_model(m) mc_value = next(v['actual'] for n, v in model.params.items() if n == 'monotone_constraints') if m.startswith('GBM'): assert isinstance(mc_value, list) age = next((v for v in mc_value if v['key'] == 'AGE'), None) assert age is not None assert age['value'] == 1.0 else: assert mc_value is None
def test_stacked_ensembles_are_trained_with_blending_frame_even_if_nfolds_eq_0( ): print( "Check that we can disable cross-validation when passing a blending frame and that Stacked Ensembles are trained using this frame." ) max_models = 5 ds = import_dataset() aml = H2OAutoML(project_name="py_aml_blending_frame", seed=1, max_models=max_models, nfolds=0) aml.train(y=ds.target, training_frame=ds.train, blending_frame=ds.valid, leaderboard_frame=ds.test) se = get_partitioned_model_names(aml.leaderboard).se assert len( se ) > 3, "In blending mode, StackedEnsemble should still be trained in spite of nfolds=0." for m in se: model = h2o.get_model(m) assert model.params['blending_frame']['actual'][ 'name'] == ds.valid.frame_id assert model._model_json['output']['stacking_strategy'] == 'blending'
def test_AUTO_stopping_metric_with_no_sorting_metric_binary(): print( "Check leaderboard with AUTO stopping metric and no sorting metric for binary" ) ds = import_dataset('binary', split=False) exclude_algos = ["DeepLearning", "GLM", "StackedEnsemble"] aml = H2OAutoML( project_name="py_aml_lb_test_auto_stopping_metric_no_sorting_binary", seed=automl_seed, max_models=10, nfolds=2, stopping_rounds=1, stopping_tolerance=0.5, exclude_algos=exclude_algos) aml.train(y=ds.target, training_frame=ds.train) check_leaderboard( aml, exclude_algos, ["auc", "logloss", "aucpr", "mean_per_class_error", "rmse", "mse"], "auc", True) base = get_partitioned_model_names(aml.leaderboard).base first = [m for m in base if 'XGBoost_1' in m] others = [m for m in base if m not in first] check_model_property( first, 'stopping_metric', True, None ) #if stopping_rounds == 0, actual value of stopping_metric is set to None check_model_property(others, 'stopping_metric', True, "logloss")
def test_stacked_ensembles_are_trained_after_max_models(): print("Check that Stacked Ensembles are still trained after max models have been trained") ds = import_dataset() aml = H2OAutoML(project_name="py_aml_SE_after_max_models", seed=1, max_models=5) aml.train(y=ds.target, training_frame=ds.train) se = get_partitioned_model_names(aml.leaderboard).se assert len(se) == 2, "StackedEnsemble should still be trained after max models have been reached"
def test_automl_stops_after_max_models(): print("Check that automl gets interrupted after `max_models`") ds = import_dataset() max_models = 5 aml = H2OAutoML(project_name="py_aml_max_models", seed=1, max_models=max_models) aml.train(y=ds.target, training_frame=ds.train) base_models = get_partitioned_model_names(aml.leaderboard).base assert len(base_models) == max_models, "obtained {} base models when {} are expected".format(len(base_models), max_models)
def test_param_disabled(): print("\n=== disabling "+kcvp+" ===") aml = setup_and_train(False) models = get_partitioned_model_names(aml.leaderboard) keys = list_keys_in_memory() preds = len(keys['cv_predictions']) assert preds == 0, "{preds} CV predictions were not cleaned from memory".format(preds=preds) for m in models.base: assert_cv_predictions_on_model(m, False) for m in models.se: assert not h2o.get_model(h2o.get_model(m).metalearner().model_id).cross_validation_predictions()
def test_nfolds_eq_0(): print("Check nfolds = 0 works properly") ds = import_dataset() aml = H2OAutoML(project_name="py_aml_nfolds0", nfolds=0, max_models=3, seed=1) aml.train(y=ds.target, training_frame=ds.train) base_models = get_partitioned_model_names(aml.leaderboard).base amodel = h2o.get_model(base_models[0]) assert amodel.params['nfolds']['actual'] == 0
def test_exclude_algos_is_applied_on_top_of_modeling_plan(): ds = import_dataset() aml = H2OAutoML(project_name="py_modeling_plan_minimal_syntax", max_models=5, modeling_plan=['DRF', 'GLM', ('GBM', 'grids'), 'StackedEnsemble'], exclude_algos=['GBM', 'StackedEnsemble'], seed=1) aml.train(y=ds.target, training_frame=ds.train) models = get_partitioned_model_names(aml.leaderboard) assert len(models.base) == 3 assert len(models.se) == 0
def test_include_algos(): print("AutoML trains only models for algos listed in include_algos") ds = import_dataset() aml = H2OAutoML(project_name="py_include_algos", include_algos=['GBM'], max_models=max_models, seed=1) aml.train(y=ds.target, training_frame=ds.train, validation_frame=ds.valid) models = get_partitioned_model_names(aml.leaderboard) assert all(['GBM' in name for name in models.base]) assert len(models.se) == 0, "No StackedEnsemble should have been trained if not explicitly included to the existing include_algos"
def test_exclude_algos(): print("AutoML doesn't train models for algos listed in exclude_algos") ds = import_dataset() aml = H2OAutoML(project_name="py_exclude_algos", exclude_algos=['DRF', 'GLM'], max_models=max_models, seed=1) aml.train(y=ds.target, training_frame=ds.train, validation_frame=ds.valid) models = get_partitioned_model_names(aml.leaderboard) assert not any(['DRF' in name or 'GLM' in name for name in models.base]) assert len(models.se) >= 1
def test_param_enabled(): print("\n=== enabling "+kcvp+" ===") aml = setup_and_train(True) models = get_partitioned_model_names(aml.leaderboard) keys = list_keys_in_memory() preds = len(keys['cv_predictions']) expected = len(models.all) * (nfolds + 1) # +1 for holdout prediction assert preds == expected, "missing CV predictions in memory, got {actual}, expected {expected}".format(actual=preds, expected=expected) for m in models.base: assert_cv_predictions_on_model(m) for m in models.se: assert_cv_predictions_on_model(h2o.get_model(m).metalearner().model_id)
def test_SE_retraining_works_when_param_enabled(): print("\n=== enabling "+kcvp+" and retraining ===") total_runs = 4 aml = setup_and_train(True) # first run ds = import_dataset() for i in range(total_runs - 1): aml.train(y=ds.target, training_frame=ds.train) models = get_partitioned_model_names(aml.leaderboard) se_all_models = [m for m in models.se if re.search(r'_AllModels_', m)] se_best_of_family = [m for m in models.se if re.search(r'_BestOfFamily_', m)] assert len(models.se) == len(se_all_models) + len(se_best_of_family) assert len(se_best_of_family) + len(se_all_models) >= total_runs, "some StackedEnsembles are missing"
def test_actual_default_input_stopping_rounds(): train = h2o.import_file(path=pu.locate("smalldata/extdata/australia.csv")) target = 'runoffnew' exclude_algos = ["DeepLearning", "GLM"] aml = H2OAutoML(project_name="actual_default_input_stopping_rounds", exclude_algos=exclude_algos, max_models=10, seed=automl_seed) aml.train(y=target, training_frame=train) base_models = get_partitioned_model_names(aml.leaderboard).base # when using cv, all cv models are trained with the stopping_rounds = 3 (default), but the final model resets stopping_rounds to 0 and use e. g. average ntrees, iterations... check_model_property(base_models, 'stopping_rounds', True, 0, 0, 3)
def test_weights_column(): print("Check weights_column") ds = import_dataset() nrows = ds.train.nrows weights_column = "weight" train = ds.train.concat( h2o.H2OFrame(list(map(lambda _: uniform(0, 5), range(nrows))), column_names=[weights_column])) aml = H2OAutoML(project_name="py_aml_weights_column", max_models=3, seed=1) aml.train(y=ds.target, training_frame=train, weights_column=weights_column) base_models = get_partitioned_model_names(aml.leaderboard).base amodel = h2o.get_model(base_models[0]) assert amodel.params['weights_column']['actual'][ 'column_name'] == weights_column
def test_exploitation_doesnt_impact_max_models(): ds = import_dataset() aml = H2OAutoML(project_name="py_exploitation_ratio_max_models", exploitation_ratio=.1, max_models=6, seed=1, verbosity='debug') aml.train(y=ds.target, training_frame=ds.train) print(aml.leaderboard) models = get_partitioned_model_names(aml.leaderboard) assert len(models.base) == 6 assert len(models.se) > 3 print(aml.training_info) assert 'start_GBM_lr_annealing' in aml.training_info assert 'start_XGBoost_lr_search' in aml.training_info
def test_nfolds_default_and_fold_assignements_skipped_by_default(): print("Check that fold assignments were skipped by default and nfolds > 1") ds = import_dataset() aml = H2OAutoML( project_name="py_aml_keep_cross_validation_fold_assignment_0", nfolds=3, max_models=3, seed=1) aml.train(y=ds.target, training_frame=ds.train) base_models = get_partitioned_model_names(aml.leaderboard).base amodel = h2o.get_model(base_models[0]) assert amodel.params['keep_cross_validation_fold_assignment'][ 'actual'] == False assert amodel._model_json["output"][ "cross_validation_fold_assignment_frame_id"] == None
def test_target_encoding_regression(): ds = import_dataset(mode='regression') aml = H2OAutoML(project_name="automl_with_te_regression", max_models=5, preprocessing=['target_encoding'], seed=1) aml.train(y=ds.target, training_frame=ds.train, leaderboard_frame=ds.test) lb = aml.leaderboard print(lb) # we can't really verify from client if TE was correctly applied... so just using a poor man's check: mem_keys = h2o.ls().key # print(mem_keys) assert any(k.startswith("TargetEncoding_AutoML") for k in mem_keys) for mid in get_partitioned_model_names(lb).all: check_mojo_pojo_availability(mid)
def test_param_disabled(): print("\n=== disabling "+kcvm+" ===") aml = setup_and_train(False) models = get_partitioned_model_names(aml.leaderboard) check_model_property(models.se, kcvm, False) check_model_property(models.base, kcvm, True, False, True) keys = list_keys_in_memory() tot, cv = len(keys['models_all']), len(keys['cv_models']) print("total models in memory = {tot}, among which {cv} CV models".format(tot=tot, cv=cv)) assert tot > 0, "no models left in memory" assert cv == 0, "{cv} CV models were not cleaned from memory".format(cv=cv) for m in models.base: assert not h2o.get_model(m).cross_validation_models(), "unexpected cv models for model "+m for m in models.se: metal = h2o.get_model(h2o.get_model(m).metalearner().model_id) assert not metal.cross_validation_models(), "unexpected cv models for metalearner of model "+m
def test_modeling_plan_using_minimal_syntax(): ds = import_dataset() aml = H2OAutoML(project_name="py_modeling_plan_minimal_syntax", max_models=5, modeling_plan=['DRF', 'GLM', ('GBM', 'grids'), 'StackedEnsemble'], seed=1) aml.train(y=ds.target, training_frame=ds.train) models = get_partitioned_model_names(aml.leaderboard) assert len(models.base) == 5 assert len(models.se) > 2 assert any('DRF' in name for name in models.base) assert any('XRT' in name for name in models.base) assert any('GLM' in name for name in models.base) assert any('GBM_grid' in name for name in models.base) assert any('BestOfFamily' in name for name in models.se) assert any('AllModels' in name for name in models.se)
def test_param_enabled(): print("\n=== enabling "+kcvm+" ===") aml = setup_and_train(True) models = get_partitioned_model_names(aml.leaderboard) check_model_property(models.se, kcvm, False) check_model_property(models.base, kcvm, True, True, True) keys = list_keys_in_memory() tot, cv = len(keys['models_all']), len(keys['cv_models']) print("total models in memory = {tot}, among which {cv} CV models".format(tot=tot, cv=cv)) assert tot > 0, "no models left in memory" expected = len(models.all) * nfolds assert cv == expected, "missing CV models in memory, got {actual}, expected {expected}".format(actual=cv, expected=expected) for m in models.base: assert h2o.get_model(m).cross_validation_models(), "missing cv models for model "+m for m in models.se: metal = h2o.get_model(h2o.get_model(m).metalearner().model_id) assert metal.cross_validation_models(), "missing cv models for metalearner of model "+m
def test_AUTO_stopping_metric_with_custom_sorting_metric_regression(): print("Check leaderboard with AUTO stopping metric and rmse sorting metric") ds = import_dataset('regression', split=False) exclude_algos = ["DeepLearning", "GLM"] aml = H2OAutoML(project_name="py_aml_lb_test_auto_stopping_metric_custom_sorting", exclude_algos=exclude_algos, max_models=10, nfolds=2, stopping_rounds=1, stopping_tolerance=0.5, seed=automl_seed, sort_metric="rmse") aml.train(y=ds.target, training_frame=ds.train) check_leaderboard(aml, exclude_algos, ["rmse", "mean_residual_deviance", "mse", "mae", "rmsle"], "rmse") base = get_partitioned_model_names(aml.leaderboard).base check_model_property(base, 'stopping_metric', True, "RMSE")
def test_balance_classes(): print("Check balance_classes & related args work properly") ds = import_dataset() aml = H2OAutoML( project_name="py_aml_balance_classes_etc", exclude_algos=['XGBoost'], # XGB doesn't support balance_classes max_models=3, balance_classes=True, class_sampling_factors=[0.2, 1.4], max_after_balance_size=3.0, seed=1) aml.train(y=ds.target, training_frame=ds.train) base_models = get_partitioned_model_names(aml.leaderboard).base amodel = h2o.get_model(base_models[0]) assert amodel.params['balance_classes']['actual'] is True assert amodel.params['max_after_balance_size']['actual'] == 3.0 assert amodel.params['class_sampling_factors']['actual'] == [0.2, 1.4]
def test_keep_cross_validation_fold_assignment_enabled_with_nfolds_eq_0(): print( "Check that fold assignments were skipped when `keep_cross_validation_fold_assignment` = True and nfolds = 0" ) ds = import_dataset() aml = H2OAutoML( project_name="py_aml_keep_cross_validation_fold_assignment_2", nfolds=0, max_models=3, seed=1, keep_cross_validation_fold_assignment=True) aml.train(y=ds.target, training_frame=ds.train) base_models = get_partitioned_model_names(aml.leaderboard).base amodel = h2o.get_model(base_models[0]) assert amodel.params['keep_cross_validation_fold_assignment'][ 'actual'] == False assert amodel._model_json["output"][ "cross_validation_fold_assignment_frame_id"] == None
def test_modeling_plan_using_simplified_syntax(): ds = import_dataset() aml = H2OAutoML(project_name="py_modeling_plan_simple_syntax", max_models=3, modeling_plan=[ ('DRF', ['XRT', 'def_1']), ('GBM', 'grids'), ('StackedEnsemble',) ], seed=1) aml.train(y=ds.target, training_frame=ds.train) print(aml.leaderboard) models = get_partitioned_model_names(aml.leaderboard) assert len(models.base) == 3 assert len(models.se) > 2 assert any('DRF' in name for name in models.base) assert any('XRT' in name for name in models.base) assert any('GBM_grid' in name for name in models.base) assert len([name for name in models.se if 'BestOfFamily' in name]) > 2 # we should get a BoF for group1 + one after GBM grid group.
def test_modeling_plan_using_full_syntax(): ds = import_dataset() aml = H2OAutoML(project_name="py_modeling_plan_full_syntax", max_models=3, modeling_plan=[ dict(name='GLM', steps=[dict(id='def_1')]), dict(name='GBM', alias='grids'), dict(name='DRF', steps=[dict(id='def_1', group=5, weight=333)]), # just testing that it is parsed correctly on backend (no model will be built due to the priority group + max_models) dict(name='GBM', steps=[dict(id="def_1")]), ], seed=1) aml.train(y=ds.target, training_frame=ds.train) print(aml.leaderboard) models = get_partitioned_model_names(aml.leaderboard) assert len(models.base) == 3 assert len(models.se) == 0 assert any('GLM' in name for name in models.base) assert any('GBM' in name for name in models.base) assert any('GBM_grid' in name for name in models.base)
def test_fold_column(): print("Check fold_column param") ds = import_dataset() fold_column = "fold_id" nrows = ds.train.nrows train = ds.train.concat( h2o.H2OFrame(list(islice(cycle(range(3)), 0, nrows)), column_names=[fold_column])) aml = H2OAutoML(project_name="py_aml_fold_column", max_models=3, seed=1, keep_cross_validation_models=True) aml.train(y=ds.target, training_frame=train, fold_column=fold_column) models = get_partitioned_model_names(aml.leaderboard) amodel = h2o.get_model(models.base[0]) assert amodel.params['fold_column']['actual']['column_name'] == fold_column ensemble = h2o.get_model(models.se[0]) metalearner = h2o.get_model(ensemble.metalearner()['name']) assert metalearner.params['fold_column']['actual'][ 'column_name'] == fold_column assert len(metalearner.cross_validation_models()) == 3
def test_AUTO_stopping_metric_with_auc_sorting_metric(): print("Check leaderboard with AUTO stopping metric and auc sorting metric") ds = import_dataset('binary', split=False) exclude_algos = ["DeepLearning", "GLM", "StackedEnsemble"] aml = H2OAutoML( project_name="py_aml_lb_test_auto_stopping_metric_auc_sorting", seed=automl_seed, max_models=10, nfolds=2, stopping_rounds=1, stopping_tolerance=0.5, exclude_algos=exclude_algos, sort_metric='auc') aml.train(y=ds.target, training_frame=ds.train) check_leaderboard( aml, exclude_algos, ["auc", "logloss", "aucpr", "mean_per_class_error", "rmse", "mse"], "auc", True) base = get_partitioned_model_names(aml.leaderboard).base check_model_property(base, 'stopping_metric', True, "logloss")