def test_scheduler_factory__reaises_exception_on_missing_searcher():
with pytest.raises(
ValueError,
match=
'Required key \'searcher\' is not present in hyperparameter_tune_kwargs'
):
scheduler_factory(hyperparameter_tune_kwargs={'scheduler': 'local'})
def test_scheduler_factory__reaises_exception_on_unknown_str_scheduler():
with pytest.raises(
ValueError,
match=
'Required key \'scheduler\' in hyperparameter_tune_kwargs must be one of the values dict_keys'
):
scheduler_factory(hyperparameter_tune_kwargs={
'scheduler': '_some_value_',
'searcher': 'bayesopt'
})
def test_scheduler_factory__can_construct_valid_config_with_str_scheduler():
scheduler_cls, scheduler_params = scheduler_factory(
hyperparameter_tune_kwargs={
'scheduler': 'local',
'searcher': 'bayesopt',
'custom_option': 'value'
})
assert scheduler_cls == LocalSequentialScheduler, 'scheduler_cls must be correct'
assert scheduler_params['resource'][
'num_cpus'] is not None, 'resources/num_cpus must be present'
assert scheduler_params['resource'][
'num_gpus'] is not None, 'resources/num_cpus must be present'
expected_values = {
'searcher': 'bayesopt',
'search_options': {},
'checkpoint': None,
'resume': False,
'num_trials': None,
'reward_attr': 'validation_performance',
'time_attr': 'epoch',
'visualizer': 'none',
'dist_ip_addrs': [],
'scheduler': 'local',
'custom_option': 'value'
}
for k, v in expected_values.items():
assert scheduler_params[k] == v, f'{k} must be {v}'
def test_scheduler_factory__can_construct_valid_config_with_class_scheduler():
scheduler_cls, scheduler_params = scheduler_factory(
hyperparameter_tune_kwargs={
'scheduler': LocalSequentialScheduler,
'searcher': 'bayesopt'
})
assert scheduler_cls == LocalSequentialScheduler, 'scheduler_cls must be correct'
def test_when_hyperparameter_tune_called_on_prophet_then_hyperparameters_are_passed_to_underlying_model(
temp_model_path, ):
scheduler_options = scheduler_factory(hyperparameter_tune_kwargs="auto")
model = ProphetModel(
path=temp_model_path,
freq="H",
prediction_length=4,
hyperparameters={
"growth": "linear",
"n_changepoints": ag.Int(3, 4)
},
)
_, _, results = model.hyperparameter_tune(
scheduler_options=scheduler_options,
time_limit=100,
train_data=DUMMY_TS_DATAFRAME,
val_data=DUMMY_TS_DATAFRAME,
)
assert len(results["config_history"]) == 2
assert results["config_history"][0]["n_changepoints"] == 3
assert results["config_history"][1]["n_changepoints"] == 4
assert all(c["growth"] == "linear"
for c in results["config_history"].values())
def tune_model_hyperparameters(
self,
model: AbstractTimeSeriesModel,
train_data: TimeSeriesDataFrame,
time_limit: Optional[float] = None,
val_data: Optional[TimeSeriesDataFrame] = None,
hyperparameter_tune_kwargs: Union[str, dict] = "auto",
):
scheduler_cls, scheduler_options = scheduler_factory(
hyperparameter_tune_kwargs, time_out=time_limit)
if all(
scheduler_options.get(s) is None
for s in ["num_trials", "time_out"]):
scheduler_options["num_trials"] = 10
logger.debug(
f"\tTuning hyperparameters of {model.name}. scheduler "
f"options: {scheduler_cls.__name__}, {pprint.pformat(scheduler_options, indent=10)}. "
)
with disable_tqdm():
hpo_models, hpo_model_performances, hpo_results = model.hyperparameter_tune(
train_data=train_data,
val_data=val_data,
scheduler_options=(scheduler_cls, scheduler_options),
time_limit=time_limit,
)
hpo_results.pop("search_strategy", None)
self.hpo_results[model.name] = hpo_results
model_names_trained = []
# add each of the trained HPO configurations to the trained models
for model_hpo_name, model_path in hpo_models.items():
model_hpo = self.load_model(model_hpo_name,
path=model_path,
model_type=type(model))
self._add_model(model_hpo)
model_names_trained.append(model_hpo.name)
logger.info(
f"\tTrained {len(model_names_trained)} models while tuning {model.name}."
)
if hpo_results:
if TimeSeriesEvaluator.METRIC_COEFFICIENTS[self.eval_metric] == -1:
sign_str = '-'
else:
sign_str = ''
logger.info(f"\t{hpo_results.get('best_reward'):<7.4f}".ljust(15) +
f"= Validation score ({sign_str}{self.eval_metric})")
logger.info(
f"\t{hpo_results.get('total_time'):<7.2f} s".ljust(15) +
"= Total tuning time")
logger.debug(
f"\tBest hyperparameter configuration: {hpo_results.get('best_config')}"
)
return model_names_trained
def _get_scheduler_options(
self,
hyperparameter_tune_kwargs: Optional[Union[str, Dict]],
time_limit: Optional[int] = None,
) -> Tuple[Optional[Type], Optional[Dict[str, Any]]]:
"""Validation logic for ``hyperparameter_tune_kwargs``. Returns True if ``hyperparameter_tune_kwargs`` is None
or can construct a valid scheduler. Returns False if hyperparameter_tune_kwargs results in an invalid scheduler.
"""
if hyperparameter_tune_kwargs is None:
return None, None
num_trials: Optional[int] = None
if isinstance(hyperparameter_tune_kwargs, dict):
num_trials = hyperparameter_tune_kwargs.get("num_trials")
if time_limit is None and num_trials is None:
logger.warning(
"None of time_limit and num_trials are set, defaulting to num_trials=2",
)
num_trials = 2
else:
num_trials = hyperparameter_tune_kwargs.get("num_trials", 999)
elif isinstance(hyperparameter_tune_kwargs, str):
num_trials = 999
scheduler_cls, scheduler_params = scheduler_factory(
hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,
time_out=time_limit,
nthreads_per_trial="auto",
ngpus_per_trial="auto",
num_trials=num_trials,
)
if scheduler_params["num_trials"] == 1:
logger.warning(
"Warning: Specified num_trials == 1 for hyperparameter tuning, disabling HPO. "
)
return None, None
scheduler_ngpus = scheduler_params["resource"].get("num_gpus", 0)
if (
scheduler_ngpus is not None
and isinstance(scheduler_ngpus, int)
and scheduler_ngpus > 1
):
logger.warning(
f"Warning: TimeSeriesPredictor currently doesn't use >1 GPU per training run. "
f"Detected {scheduler_ngpus} GPUs."
)
return scheduler_cls, scheduler_params
def test_given_hyperparameter_spaces_when_tune_called_then_tuning_output_correct(
model_class, temp_model_path):
scheduler_options = scheduler_factory(hyperparameter_tune_kwargs="auto")
model = model_class(
path=temp_model_path,
freq="H",
quantile_levels=[0.1, 0.9],
hyperparameters={
"epochs": ag.Int(3, 4),
},
)
_, _, results = model.hyperparameter_tune(
scheduler_options=scheduler_options,
time_limit=300,
train_data=DUMMY_TS_DATAFRAME,
val_data=DUMMY_TS_DATAFRAME,
)
assert len(results["config_history"]) == 2
assert results["config_history"][0]["epochs"] == 3
assert results["config_history"][1]["epochs"] == 4
