def test_create_stepwise_study(self, direction: str, overall_best: int) -> None:
tuner = LightGBMTuner({}, None, valid_sets=lgb.Dataset(np.zeros((10, 10))))
def objective(trial: optuna.trial.Trial, value: float) -> float:
trial.set_system_attr(
optuna.integration.lightgbm_tuner.optimize._STEP_NAME_KEY,
"step{:.0f}".format(value),
)
return trial.suggest_uniform("x", value, value)
study = optuna.create_study(direction=direction)
study_step1 = tuner._create_stepwise_study(study, "step1")
with pytest.raises(ValueError):
study_step1.best_trial
study_step1.optimize(lambda t: objective(t, 1), n_trials=1)
study_step2 = tuner._create_stepwise_study(study, "step2")
# `study` has a trial, but `study_step2` has no trials.
with pytest.raises(ValueError):
study_step2.best_trial
study_step2.optimize(lambda t: objective(t, 2), n_trials=2)
assert len(study_step1.trials) == 1
assert len(study_step2.trials) == 2
assert len(study.trials) == 3
assert study_step1.best_trial.value == 1
assert study_step2.best_trial.value == 2
assert study.best_trial.value == overall_best
def train(*args, **kwargs):
# type: (List[Any], Optional[Dict[Any, Any]]) -> Any
"""Wrapper function of LightGBM API: train()
Arguments and keyword arguments for `lightgbm.train()` can be passed.
"""
auto_booster = LightGBMTuner(*args, **kwargs)
booster = auto_booster.run()
return booster
def train(*args: Any, **kwargs: Any) -> Any:
"""Wrapper of LightGBM Training API to tune hyperparameters.
It tunes important hyperparameters (e.g., `min_child_samples` and `feature_fraction`) in a
stepwise manner. Arguments and keyword arguments for `lightgbm.train()
<https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.train.html>`_ can be passed.
"""
auto_booster = LightGBMTuner(*args, **kwargs)
booster = auto_booster.run()
return booster
def test_best_booster_with_model_dir(self) -> None:
params = {"verbose": -1} # type: Dict
dataset = lgb.Dataset(np.zeros((10, 10)))
study = optuna.create_study()
with TemporaryDirectory() as tmpdir:
tuner = LightGBMTuner(params,
dataset,
valid_sets=dataset,
study=study,
model_dir=tmpdir)
with mock.patch.object(BaseTuner,
"_get_booster_best_score",
return_value=0.0):
tuner.tune_regularization_factors()
best_booster = tuner.get_best_booster()
tuner2 = LightGBMTuner(params,
dataset,
valid_sets=dataset,
study=study,
model_dir=tmpdir)
best_booster2 = tuner2.get_best_booster()
assert best_booster.params == best_booster2.params
def test_optuna_callback(self) -> None:
params = {"verbose": -1} # type: Dict[str, Any]
dataset = lgb.Dataset(np.zeros((10, 10)))
callback_mock = mock.MagicMock()
study = optuna.create_study()
tuner = LightGBMTuner(
params, dataset, valid_sets=dataset, study=study, optuna_callbacks=[callback_mock],
)
with mock.patch.object(BaseTuner, "_get_booster_best_score", return_value=1.0):
tuner.tune_params(["num_leaves"], 10, optuna.samplers.TPESampler(), "num_leaves")
assert callback_mock.call_count == 10
def train(*args, **kwargs):
# type: (Any, Any) -> Any
"""Wrapper of LightGBM Training API to tune hyperparameters.
.. warning::
This feature is experimental. The interface may be changed in the future.
It tunes important hyperparameters (e.g., `min_child_samples` and `feature_fraction`) in a
stepwise manner. Arguments and keyword arguments for `lightgbm.train()
<https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.train.html>`_ can be passed.
"""
auto_booster = LightGBMTuner(*args, **kwargs)
booster = auto_booster.run()
return booster
def test_resume_run(self) -> None:
params = {"verbose": -1} # type: Dict
dataset = lgb.Dataset(np.zeros((10, 10)))
study = optuna.create_study()
tuner = LightGBMTunerCV(params, dataset, study=study)
with mock.patch.object(OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]):
tuner.tune_regularization_factors()
n_trials = len(study.trials)
assert n_trials == len(study.trials)
tuner2 = LightGBMTuner(params, dataset, valid_sets=dataset, study=study)
with mock.patch.object(OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]):
tuner2.tune_regularization_factors()
assert n_trials == len(study.trials)
def test_when_a_step_does_not_improve_best_score(self):
# type: () -> None
params = {} # type: Dict
valid_data = np.zeros((10, 10))
valid_sets = lgb.Dataset(valid_data)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=DeprecationWarning)
tuner = LightGBMTuner(params, None, valid_sets=valid_sets)
assert not tuner.higher_is_better()
with mock.patch("lightgbm.train"), mock.patch.object(
BaseTuner, "_get_booster_best_score", return_value=0.9
):
tuner.tune_feature_fraction()
assert "feature_fraction" in tuner.best_params
assert tuner.best_score == 0.9
# Assume that tuning `num_leaves` doesn't improve the `best_score`.
with mock.patch("lightgbm.train"), mock.patch.object(
BaseTuner, "_get_booster_best_score", return_value=1.1
):
tuner.tune_num_leaves()
# `num_leaves` should be same as default.
assert tuner.best_params["num_leaves"] == 31
assert tuner.best_score == 0.9
def train(*args: Any, **kwargs: Any) -> Any:
"""Wrapper of LightGBM Training API to tune hyperparameters.
It tunes important hyperparameters (e.g., ``min_child_samples`` and ``feature_fraction``) in a
stepwise manner. It is a drop-in replacement for `lightgbm.train()`_. See
`a simple example of LightGBM Tuner <https://github.com/optuna/optuna/blob/master/examples/lig
htgbm_tuner_simple.py>`_ which optimizes the validation log loss of cancer detection.
:func:`~optuna.integration.lightgbm.train` is a wrapper function of
:class:`~optuna.integration.lightgbm_tuner.LightGBMTuner`. To use feature in Optuna such as
suspended/resumed optimization and/or parallelization, refer to
:class:`~optuna.integration.lightgbm_tuner.LightGBMTuner` instead of this function.
Arguments and keyword arguments for `lightgbm.train()`_ can be passed.
.. _lightgbm.train(): https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.train.html
"""
_imports.check()
auto_booster = LightGBMTuner(*args, **kwargs)
auto_booster.run()
return auto_booster.get_best_booster()
def test_when_a_step_does_not_improve_best_score(self):
# type: () -> None
params = {} # type: Dict
valid_data = np.zeros((10, 10))
valid_sets = lgb.Dataset(valid_data)
tuner = LightGBMTuner(params, None, valid_sets=valid_sets)
assert not tuner.higher_is_better()
with mock.patch("lightgbm.train"), mock.patch.object(
BaseTuner, "_get_booster_best_score", return_value=0.9
):
tuner.tune_feature_fraction()
assert "feature_fraction" in tuner.best_params
assert tuner.best_score == 0.9
# Assume that tuning `num_leaves` doesn't improve the `best_score`.
with mock.patch("lightgbm.train"), mock.patch.object(
BaseTuner, "_get_booster_best_score", return_value=1.1
):
tuner.tune_num_leaves()
def test_get_best_booster(self) -> None:
unexpected_value = 20 # out of scope.
params = {"verbose": -1, "lambda_l1": unexpected_value} # type: Dict
dataset = lgb.Dataset(np.zeros((10, 10)))
study = optuna.create_study()
tuner = LightGBMTuner(params, dataset, valid_sets=dataset, study=study)
with pytest.raises(ValueError):
tuner.get_best_booster()
with mock.patch.object(BaseTuner, "_get_booster_best_score", return_value=0.0):
tuner.tune_regularization_factors()
best_booster = tuner.get_best_booster()
assert best_booster.params["lambda_l1"] != unexpected_value
# TODO(toshihikoyanase): Remove this check when LightGBMTuner.best_booster is removed.
with pytest.warns(DeprecationWarning):
tuner.best_booster
tuner2 = LightGBMTuner(params, dataset, valid_sets=dataset, study=study)
# Resumed study does not have the best booster.
with pytest.raises(ValueError):
tuner2.get_best_booster()
def test_when_a_step_does_not_improve_best_score(self):
# type: () -> None
params = {} # type: Dict
valid_data = np.zeros((10, 10))
valid_sets = lgb.Dataset(valid_data)
tuner = LightGBMTuner(params, None, valid_sets=valid_sets)
assert not tuner.higher_is_better()
objective_class_name = 'optuna.integration.lightgbm_tuner.optimize.OptunaObjective'
with mock.patch(objective_class_name) as objective_mock,\
mock.patch('optuna.study.Study') as study_mock,\
mock.patch('optuna.trial.Trial') as trial_mock:
fake_objective = mock.MagicMock(spec=OptunaObjective)
fake_objective.report = []
fake_objective.best_booster = None
objective_mock.return_value = fake_objective
fake_study = mock.MagicMock(spec=Study)
fake_study._storage = mock.MagicMock()
fake_study.best_value = 0.9
study_mock.return_value = fake_study
fake_trial = mock.MagicMock(spec=Trial)
fake_trial.best_params = {
'feature_fraction': 0.2,
}
trial_mock.return_value = fake_trial
tuner.tune_feature_fraction()
fake_study.optimize.assert_called()
assert 'feature_fraction' in tuner.best_params
assert tuner.best_score == 0.9
with mock.patch(objective_class_name) as objective_mock,\
mock.patch('optuna.study.Study') as study_mock,\
mock.patch('optuna.trial.Trial') as trial_mock:
fake_objective = mock.MagicMock(spec=OptunaObjective)
fake_objective.report = []
fake_objective.best_booster = None
objective_mock.return_value = fake_objective
# Assume that tuning `num_leaves` doesn't improve the `best_score`.
fake_study = mock.MagicMock(spec=Study)
fake_study._storage = mock.MagicMock()
fake_study.best_value = 1.1
study_mock.return_value = fake_study
fake_trial = mock.MagicMock(spec=Trial)
fake_trial.best_params = {
'num_leaves': 128,
}
trial_mock.return_value = fake_trial
tuner.tune_num_leaves()
fake_study.optimize.assert_called()
# `num_leaves` should not be same as default.
assert tuner.best_params['num_leaves'] == 31
assert tuner.best_score == 0.9