def test_get_best_booster(self) -> None: unexpected_value = 20 # out of scope. params = {"verbose": -1, "lambda_l1": unexpected_value} # type: Dict dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() with TemporaryDirectory() as tmpdir: tuner = LightGBMTunerCV(params, dataset, study=study, model_dir=tmpdir, return_cvbooster=True) with pytest.raises(ValueError): tuner.get_best_booster() with mock.patch.object(_OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]): tuner.tune_regularization_factors() best_boosters = tuner.get_best_booster().boosters for booster in best_boosters: assert booster.params["lambda_l1"] != unexpected_value tuner2 = LightGBMTunerCV(params, dataset, study=study, model_dir=tmpdir, return_cvbooster=True) best_boosters2 = tuner2.get_best_booster().boosters for booster, booster2 in zip(best_boosters, best_boosters2): assert booster.params == booster2.params
def test_optuna_callback(self) -> None: params: Dict[str, Any] = {"verbose": -1} dataset = lgb.Dataset(np.zeros((10, 10))) callback_mock = mock.MagicMock() study = optuna.create_study() tuner = LightGBMTunerCV(params, dataset, study=study, optuna_callbacks=[callback_mock]) with mock.patch.object(_OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]): tuner._tune_params(["num_leaves"], 10, optuna.samplers.TPESampler(), "num_leaves") assert callback_mock.call_count == 10
def test_run_show_progress_bar(self, show_progress_bar: bool, expected: int) -> None: params: Dict = {"verbose": -1} dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() tuner = LightGBMTunerCV( params, dataset, study=study, time_budget=1, show_progress_bar=show_progress_bar ) with mock.patch.object( _OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0] ), mock.patch("tqdm.tqdm") as mock_tqdm: tuner.run() assert mock_tqdm.call_count == expected
def test_resume_run(self) -> None: params: Dict = {"verbose": -1} dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() tuner = LightGBMTunerCV(params, dataset, study=study) with mock.patch.object(_OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]): tuner.tune_regularization_factors() n_trials = len(study.trials) assert n_trials == len(study.trials) tuner2 = LightGBMTuner(params, dataset, valid_sets=dataset, study=study) with mock.patch.object(_OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]): tuner2.tune_regularization_factors() assert n_trials == len(study.trials)
def test_model_dir(self, dir_exists: bool, expected: bool) -> None: unexpected_value = 20 # out of scope. params: Dict = {"verbose": -1, "lambda_l1": unexpected_value} dataset = lgb.Dataset(np.zeros((10, 10))) with mock.patch("os.mkdir") as m: with mock.patch("os.path.exists", return_value=dir_exists): LightGBMTunerCV(params, dataset, model_dir="./booster") assert m.called == expected
def test_run_verbosity(self, verbosity: int, level: int) -> None: # We need to reconstruct our default handler to properly capture stderr. optuna.logging._reset_library_root_logger() optuna.logging.set_verbosity(optuna.logging.INFO) params: Dict = {"verbose": -1} dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() with warnings.catch_warnings(): warnings.simplefilter("ignore", category=FutureWarning) tuner = LightGBMTunerCV( params, dataset, study=study, verbosity=verbosity, time_budget=1 ) with mock.patch.object(_OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]): tuner.run() assert optuna.logging.get_verbosity() == level assert tuner.lgbm_params["verbose"] == -1
def test_inconsistent_study_direction(self, metric: str, study_direction: str) -> None: params: Dict[str, Any] = {} if metric is not None: params["metric"] = metric train_set = lgb.Dataset(None) study = optuna.create_study(direction=study_direction) with pytest.raises(ValueError) as excinfo: LightGBMTunerCV( params, train_set, num_boost_round=5, early_stopping_rounds=2, study=study ) assert excinfo.type == ValueError assert str(excinfo.value).startswith("Study direction is inconsistent with the metric")
def _get_tunercv_object( self, params: Dict[str, Any] = {}, train_set: Optional[lgb.Dataset] = None, kwargs_options: Dict[str, Any] = {}, study: Optional[optuna.study.Study] = None, ) -> LightGBMTunerCV: # Required keyword arguments. kwargs: Dict[str, Any] = dict(num_boost_round=5, early_stopping_rounds=2, study=study) kwargs.update(kwargs_options) runner = LightGBMTunerCV(params, train_set, **kwargs) return runner
def test_deprecated_args(self) -> None: dummy_dataset = lgb.Dataset(None) with pytest.warns(FutureWarning): LightGBMTunerCV({}, dummy_dataset, verbosity=1)
def test_get_best_booster_with_error(self) -> None: params = {"verbose": -1} # type: Dict dataset = lgb.Dataset(np.zeros((10, 10))) study = optuna.create_study() tuner = LightGBMTunerCV(params, dataset, study=study, model_dir=None, return_cvbooster=True) # No trial is completed yet. with pytest.raises(ValueError): tuner.get_best_booster() with mock.patch.object(_OptunaObjectiveCV, "_get_cv_scores", return_value=[1.0]): tuner.tune_regularization_factors() tuner2 = LightGBMTunerCV(params, dataset, study=study, model_dir=None, return_cvbooster=True) # Resumed the study does not have the best booster. with pytest.raises(ValueError): tuner2.get_best_booster() with TemporaryDirectory() as tmpdir: tuner3 = LightGBMTunerCV(params, dataset, study=study, model_dir=tmpdir, return_cvbooster=True) # The booster was not saved hence not found in the `model_dir`. with pytest.raises(ValueError): tuner3.get_best_booster()