def test_create_stepwise_study(self, direction: str, overall_best: int) -> None: tuner = LightGBMTuner({}, None, valid_sets=lgb.Dataset(np.zeros((10, 10)))) def objective(trial: optuna.trial.Trial, value: float) -> float: trial.set_system_attr( optuna.integration._lightgbm_tuner.optimize._STEP_NAME_KEY, "step{:.0f}".format(value), ) return trial.suggest_float("x", value, value) study = optuna.create_study(direction=direction) study_step1 = tuner._create_stepwise_study(study, "step1") with pytest.raises(ValueError): study_step1.best_trial study_step1.optimize(lambda t: objective(t, 1), n_trials=1) study_step2 = tuner._create_stepwise_study(study, "step2") # `study` has a trial, but `study_step2` has no trials. with pytest.raises(ValueError): study_step2.best_trial study_step2.optimize(lambda t: objective(t, 2), n_trials=2) assert len(study_step1.trials) == 1 assert len(study_step2.trials) == 2 assert len(study.trials) == 3 assert study_step1.best_trial.value == 1 assert study_step2.best_trial.value == 2 assert study.best_trial.value == overall_best
def test_run_verbosity(self, verbosity: int, level: int) -> None: # We need to reconstruct our default handler to properly capture stderr. optuna.logging._reset_library_root_logger() optuna.logging.set_verbosity(optuna.logging.INFO) params = {"verbose": -1} # type: Dict dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=FutureWarning) tuner = LightGBMTuner( params, dataset, valid_sets=dataset, study=study, verbosity=verbosity, time_budget=1, ) with mock.patch.object(_BaseTuner, "_get_booster_best_score", return_value=1.0): tuner.run() assert optuna.logging.get_verbosity() == level assert tuner.lgbm_params["verbose"] == -1
def test_best_booster_with_model_dir(self) -> None: params = {"verbose": -1} # type: Dict dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() with TemporaryDirectory() as tmpdir: tuner = LightGBMTuner(params, dataset, valid_sets=dataset, study=study, model_dir=tmpdir) with mock.patch.object(_BaseTuner, "_get_booster_best_score", return_value=0.0): tuner.tune_regularization_factors() best_booster = tuner.get_best_booster() tuner2 = LightGBMTuner(params, dataset, valid_sets=dataset, study=study, model_dir=tmpdir) best_booster2 = tuner2.get_best_booster() assert best_booster.params == best_booster2.params
def test_resume_run(self) -> None: params: Dict = {"verbose": -1} dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() tuner = LightGBMTuner(params, dataset, valid_sets=dataset, study=study, callbacks=[log_evaluation(-1)]) with mock.patch.object(_BaseTuner, "_get_booster_best_score", return_value=1.0): tuner.tune_regularization_factors() n_trials = len(study.trials) assert n_trials == len(study.trials) tuner2 = LightGBMTuner(params, dataset, valid_sets=dataset, study=study) with mock.patch.object(_BaseTuner, "_get_booster_best_score", return_value=1.0): tuner2.tune_regularization_factors() assert n_trials == len(study.trials)
def test_optuna_callback(self) -> None: params: Dict[str, Any] = {"verbose": -1} dataset = lgb.Dataset(np.zeros((10, 10))) callback_mock = mock.MagicMock() study = optuna.create_study() tuner = LightGBMTuner( params, dataset, valid_sets=dataset, study=study, optuna_callbacks=[callback_mock] ) with mock.patch.object(_BaseTuner, "_get_booster_best_score", return_value=1.0): tuner._tune_params(["num_leaves"], 10, optuna.samplers.TPESampler(), "num_leaves") assert callback_mock.call_count == 10
def test_deprecated_args(self) -> None: dummy_dataset = lgb.Dataset(None) with pytest.warns(FutureWarning): LightGBMTuner({}, dummy_dataset, valid_sets=[dummy_dataset], verbosity=1)
def test_resume_run(self) -> None: params: Dict = {"verbose": -1} dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() tuner = LightGBMTunerCV(params, dataset, study=study) with mock.patch.object(_OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]): tuner.tune_regularization_factors() n_trials = len(study.trials) assert n_trials == len(study.trials) tuner2 = LightGBMTuner(params, dataset, valid_sets=dataset, study=study) with mock.patch.object(_OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]): tuner2.tune_regularization_factors() assert n_trials == len(study.trials)
def test_model_dir(self, dir_exists: bool, expected: bool) -> None: params: Dict = {"verbose": -1} dataset = lgb.Dataset(np.zeros((10, 10))) with mock.patch("optuna.integration._lightgbm_tuner.optimize.os.mkdir") as m: with mock.patch("os.path.exists", return_value=dir_exists): LightGBMTuner(params, dataset, valid_sets=dataset, model_dir="./booster") assert m.called == expected
def test_run_show_progress_bar(self, show_progress_bar: bool, expected: int) -> None: params: Dict = {"verbose": -1} dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() tuner = LightGBMTuner( params, dataset, valid_sets=dataset, study=study, time_budget=1, show_progress_bar=show_progress_bar, ) with mock.patch.object( _BaseTuner, "_get_booster_best_score", return_value=1.0 ), mock.patch("tqdm.tqdm") as mock_tqdm: tuner.run() assert mock_tqdm.call_count == expected
def train(*args: Any, **kwargs: Any) -> Any: """Wrapper of LightGBM Training API to tune hyperparameters. It tunes important hyperparameters (e.g., ``min_child_samples`` and ``feature_fraction``) in a stepwise manner. It is a drop-in replacement for `lightgbm.train()`_. See `a simple example of LightGBM Tuner <https://github.com/optuna/optuna/blob/master/examples/lig htgbm_tuner_simple.py>`_ which optimizes the validation log loss of cancer detection. :func:`~optuna.integration.lightgbm.train` is a wrapper function of :class:`~optuna.integration.lightgbm.LightGBMTuner`. To use feature in Optuna such as suspended/resumed optimization and/or parallelization, refer to :class:`~optuna.integration.lightgbm.LightGBMTuner` instead of this function. Arguments and keyword arguments for `lightgbm.train()`_ can be passed. .. _lightgbm.train(): https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.train.html """ _imports.check() auto_booster = LightGBMTuner(*args, **kwargs) auto_booster.run() return auto_booster.get_best_booster()
def test_when_a_step_does_not_improve_best_score(self) -> None: params = {} # type: Dict valid_data = np.zeros((10, 10)) valid_sets = lgb.Dataset(valid_data) tuner = LightGBMTuner(params, None, valid_sets=valid_sets) assert not tuner.higher_is_better() with mock.patch("lightgbm.train"), mock.patch.object( _BaseTuner, "_get_booster_best_score", return_value=0.9): tuner.tune_feature_fraction() assert "feature_fraction" in tuner.best_params assert tuner.best_score == 0.9 # Assume that tuning `num_leaves` doesn't improve the `best_score`. with mock.patch("lightgbm.train"), mock.patch.object( _BaseTuner, "_get_booster_best_score", return_value=1.1): tuner.tune_num_leaves()
def test_get_best_booster(self) -> None: unexpected_value = 20 # out of scope. params = {"verbose": -1, "lambda_l1": unexpected_value} # type: Dict dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() tuner = LightGBMTuner(params, dataset, valid_sets=dataset, study=study) with pytest.raises(ValueError): tuner.get_best_booster() with mock.patch.object(_BaseTuner, "_get_booster_best_score", return_value=0.0): tuner.tune_regularization_factors() best_booster = tuner.get_best_booster() assert best_booster.params["lambda_l1"] != unexpected_value # TODO(toshihikoyanase): Remove this check when LightGBMTuner.best_booster is removed. with pytest.warns(FutureWarning): tuner.best_booster tuner2 = LightGBMTuner(params, dataset, valid_sets=dataset, study=study) # Resumed study does not have the best booster. with pytest.raises(ValueError): tuner2.get_best_booster()
def test_get_best_booster(self) -> None: unexpected_value = 20 # out of scope. params: Dict = {"verbose": -1, "lambda_l1": unexpected_value} dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() tuner = LightGBMTuner(params, dataset, valid_sets=dataset, study=study) with pytest.raises(ValueError): tuner.get_best_booster() with mock.patch.object(_BaseTuner, "_get_booster_best_score", return_value=0.0): tuner.tune_regularization_factors() best_booster = tuner.get_best_booster() assert best_booster.params["lambda_l1"] != unexpected_value tuner2 = LightGBMTuner(params, dataset, valid_sets=dataset, study=study) # Resumed study does not have the best booster. with pytest.raises(ValueError): tuner2.get_best_booster()