def test_given_repeating_model_when_trainer_called_incrementally_then_name_collisions_are_prevented(
temp_model_path,
hyperparameter_list,
expected_number_of_unique_names,
expected_suffixes,
):
trainer = AutoTimeSeriesTrainer(path=temp_model_path)
# incrementally train with new hyperparameters
for hp in hyperparameter_list:
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
hyperparameters=hp,
val_data=DUMMY_TS_DATAFRAME,
)
model_names = trainer.get_model_names()
if trainer.enable_ensemble:
expected_number_of_unique_names += 1
assert len(model_names) == expected_number_of_unique_names
for suffix in expected_suffixes:
assert any(name.endswith(suffix) for name in model_names)
if not trainer.enable_ensemble:
# there should be no edges in the model graph without ensembling
assert not trainer.model_graph.edges
def test_given_hyperparameters_with_spaces_and_custom_model_when_trainer_called_then_hpo_is_performed(
temp_model_path, ):
hyperparameters = {
GenericGluonTSModelFactory(MQRNNEstimator): {
"epochs": ag.Int(1, 4)
}
}
# mock the default hps factory to prevent preset hyperparameter configurations from
# creeping into the test case
with mock.patch("autogluon.timeseries.models.presets.get_default_hps"
) as default_hps_mock:
default_hps_mock.return_value = defaultdict(dict)
trainer = AutoTimeSeriesTrainer(path=temp_model_path)
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
hyperparameters=hyperparameters,
val_data=DUMMY_TS_DATAFRAME,
hyperparameter_tune=True,
)
leaderboard = trainer.leaderboard()
assert len(leaderboard) == 4 + 1 # ensemble
config_history = next(iter(trainer.hpo_results.values()))["config_history"]
assert len(config_history) == 4
assert all(1 <= model["epochs"] <= 4 for model in config_history.values())
def test_given_hyperparameters_to_prophet_when_trainer_model_templates_called_then_hyperparameters_set_correctly(
temp_model_path, hyperparameters):
trainer = AutoTimeSeriesTrainer(path=temp_model_path, eval_metric="MAPE")
models = trainer.construct_model_templates(
hyperparameters=hyperparameters, )
for model in models:
for k, v in hyperparameters[model.name].items():
assert model._user_params[k] == v
def test_given_hyperparameters_when_trainer_called_then_leaderboard_is_correct(
temp_model_path, eval_metric, hyperparameters, expected_board_length):
trainer = AutoTimeSeriesTrainer(path=temp_model_path,
eval_metric=eval_metric)
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
hyperparameters=hyperparameters,
val_data=DUMMY_TS_DATAFRAME,
)
leaderboard = trainer.leaderboard()
expected_board_length += int(trainer.enable_ensemble)
assert len(leaderboard) == expected_board_length
assert np.all(leaderboard["score_val"] < 0) # all MAPEs should be negative
def test_when_trainer_fit_and_deleted_models_load_back_correctly_and_can_predict(
temp_model_path, hyperparameters, low_memory):
trainer = AutoTimeSeriesTrainer(path=temp_model_path,
eval_metric="MAPE",
prediction_length=2)
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
hyperparameters=hyperparameters,
val_data=DUMMY_TS_DATAFRAME,
)
model_names = copy.copy(trainer.get_model_names())
trainer.save()
del trainer
loaded_trainer = AutoTimeSeriesTrainer.load(path=temp_model_path)
for m in model_names:
loaded_model = loaded_trainer.load_model(m)
if isinstance(loaded_model, TimeSeriesEnsembleWrapper):
continue
predictions = loaded_model.predict(DUMMY_TS_DATAFRAME)
assert isinstance(predictions, TimeSeriesDataFrame)
predicted_item_index = predictions.index.levels[0]
assert all(
predicted_item_index == DUMMY_TS_DATAFRAME.index.levels[0]) # noqa
assert all(len(predictions.loc[i]) == 2 for i in predicted_item_index)
assert not np.any(np.isnan(predictions))
def test_given_hyperparameters_and_custom_models_when_trainer_model_templates_called_then_hyperparameters_set_correctly(
temp_model_path, hyperparameters):
trainer = AutoTimeSeriesTrainer(path=temp_model_path, eval_metric="MAPE")
models = trainer.construct_model_templates(
hyperparameters=hyperparameters, )
for model in models:
if isinstance(model, GenericGluonTSModel):
model_hyperparam = next(
hyperparameters[m] for m in hyperparameters
if isinstance(m, GenericGluonTSModelFactory))
else:
model_hyperparam = hyperparameters[model.name]
for k, v in model_hyperparam.items():
assert model._user_params[k] == v
def trained_trainers():
trainers = {}
model_paths = []
for hp in TEST_HYPERPARAMETER_SETTINGS:
temp_model_path = tempfile.mkdtemp()
trainer = AutoTimeSeriesTrainer(
path=temp_model_path + os.path.sep,
eval_metric="MAPE",
prediction_length=3,
)
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
val_data=DUMMY_TS_DATAFRAME,
hyperparameters=hp,
)
trainers[repr(hp)] = trainer
model_paths.append(temp_model_path)
yield trainers
for td in model_paths:
shutil.rmtree(td)
def test_given_hyperparameters_with_spaces_to_prophet_when_trainer_called_then_hpo_is_performed(
temp_model_path, ):
hyperparameters = {"Prophet": {"n_changepoints": ag.Int(1, 4)}}
# mock the default hps factory to prevent preset hyperparameter configurations from
# creeping into the test case
with mock.patch("autogluon.timeseries.models.presets.get_default_hps"
) as default_hps_mock:
default_hps_mock.return_value = defaultdict(dict)
trainer = AutoTimeSeriesTrainer(path=temp_model_path)
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
hyperparameters=hyperparameters,
val_data=DUMMY_TS_DATAFRAME,
hyperparameter_tune=True,
)
leaderboard = trainer.leaderboard()
assert len(leaderboard) == 4 + 1
config_history = trainer.hpo_results["Prophet"]["config_history"]
assert len(config_history) == 4
assert all(1 <= model["n_changepoints"] <= 4
for model in config_history.values())
def test_given_hyperparameters_when_trainer_fit_then_freq_set_correctly(
temp_model_path, hyperparameters):
trainer = AutoTimeSeriesTrainer(path=temp_model_path, eval_metric="MAPE")
trainer.fit(
train_data=DUMMY_TS_DATAFRAME,
hyperparameters=hyperparameters,
val_data=DUMMY_TS_DATAFRAME,
)
for model_name in trainer.get_model_names():
model = trainer.load_model(model_name)
assert model.freq == DUMMY_TS_DATAFRAME.freq
def test_trainer_can_be_initialized(temp_model_path):
model = AutoTimeSeriesTrainer(path=temp_model_path, prediction_length=24)
assert isinstance(model, AutoTimeSeriesTrainer)