def test_given_hyperparameters_with_spaces_and_custom_model_when_trainer_called_then_hpo_is_performed( temp_model_path, ): hyperparameters = { GenericGluonTSModelFactory(MQRNNEstimator): { "epochs": ag.Int(1, 4) } } # mock the default hps factory to prevent preset hyperparameter configurations from # creeping into the test case with mock.patch("autogluon.timeseries.models.presets.get_default_hps" ) as default_hps_mock: default_hps_mock.return_value = defaultdict(dict) trainer = AutoTimeSeriesTrainer(path=temp_model_path) trainer.fit( train_data=DUMMY_TS_DATAFRAME, hyperparameters=hyperparameters, val_data=DUMMY_TS_DATAFRAME, hyperparameter_tune=True, ) leaderboard = trainer.leaderboard() assert len(leaderboard) == 4 + 1 # ensemble config_history = next(iter(trainer.hpo_results.values()))["config_history"] assert len(config_history) == 4 assert all(1 <= model["epochs"] <= 4 for model in config_history.values())
def test_given_hyperparameters_when_trainer_called_then_leaderboard_is_correct( temp_model_path, eval_metric, hyperparameters, expected_board_length): trainer = AutoTimeSeriesTrainer(path=temp_model_path, eval_metric=eval_metric) trainer.fit( train_data=DUMMY_TS_DATAFRAME, hyperparameters=hyperparameters, val_data=DUMMY_TS_DATAFRAME, ) leaderboard = trainer.leaderboard() expected_board_length += int(trainer.enable_ensemble) assert len(leaderboard) == expected_board_length assert np.all(leaderboard["score_val"] < 0) # all MAPEs should be negative
def test_given_hyperparameters_with_spaces_to_prophet_when_trainer_called_then_hpo_is_performed( temp_model_path, ): hyperparameters = {"Prophet": {"n_changepoints": ag.Int(1, 4)}} # mock the default hps factory to prevent preset hyperparameter configurations from # creeping into the test case with mock.patch("autogluon.timeseries.models.presets.get_default_hps" ) as default_hps_mock: default_hps_mock.return_value = defaultdict(dict) trainer = AutoTimeSeriesTrainer(path=temp_model_path) trainer.fit( train_data=DUMMY_TS_DATAFRAME, hyperparameters=hyperparameters, val_data=DUMMY_TS_DATAFRAME, hyperparameter_tune=True, ) leaderboard = trainer.leaderboard() assert len(leaderboard) == 4 + 1 config_history = trainer.hpo_results["Prophet"]["config_history"] assert len(config_history) == 4 assert all(1 <= model["n_changepoints"] <= 4 for model in config_history.values())