def test_given_hyperparameters_with_spaces_and_custom_model_when_trainer_called_then_hpo_is_performed(
temp_model_path, ):
hyperparameters = {
GenericGluonTSModelFactory(MQRNNEstimator): {
"epochs": ag.Int(1, 4)
}
}
# mock the default hps factory to prevent preset hyperparameter configurations from
# creeping into the test case
with mock.patch("autogluon.timeseries.models.presets.get_default_hps"
) as default_hps_mock:
default_hps_mock.return_value = defaultdict(dict)
trainer = AutoTimeSeriesTrainer(path=temp_model_path)
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
hyperparameters=hyperparameters,
val_data=DUMMY_TS_DATAFRAME,
hyperparameter_tune=True,
)
leaderboard = trainer.leaderboard()
assert len(leaderboard) == 4 + 1 # ensemble
config_history = next(iter(trainer.hpo_results.values()))["config_history"]
assert len(config_history) == 4
assert all(1 <= model["epochs"] <= 4 for model in config_history.values())
def test_given_hyperparameters_when_trainer_called_then_leaderboard_is_correct(
temp_model_path, eval_metric, hyperparameters, expected_board_length):
trainer = AutoTimeSeriesTrainer(path=temp_model_path,
eval_metric=eval_metric)
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
hyperparameters=hyperparameters,
val_data=DUMMY_TS_DATAFRAME,
)
leaderboard = trainer.leaderboard()
expected_board_length += int(trainer.enable_ensemble)
assert len(leaderboard) == expected_board_length
assert np.all(leaderboard["score_val"] < 0) # all MAPEs should be negative
def test_given_hyperparameters_with_spaces_to_prophet_when_trainer_called_then_hpo_is_performed(
temp_model_path, ):
hyperparameters = {"Prophet": {"n_changepoints": ag.Int(1, 4)}}
# mock the default hps factory to prevent preset hyperparameter configurations from
# creeping into the test case
with mock.patch("autogluon.timeseries.models.presets.get_default_hps"
) as default_hps_mock:
default_hps_mock.return_value = defaultdict(dict)
trainer = AutoTimeSeriesTrainer(path=temp_model_path)
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
hyperparameters=hyperparameters,
val_data=DUMMY_TS_DATAFRAME,
hyperparameter_tune=True,
)
leaderboard = trainer.leaderboard()
assert len(leaderboard) == 4 + 1
config_history = trainer.hpo_results["Prophet"]["config_history"]
assert len(config_history) == 4
assert all(1 <= model["n_changepoints"] <= 4
for model in config_history.values())