def test_normalize_integer(): for dtype in [ "int", "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64", ]: a = Integer(2, 30, transform="normalize", dtype=dtype) for X in range(2, 31): X_orig = a.inverse_transform(a.transform(X)) assert_array_equal(X_orig, X) for dtype in [ int, np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, ]: a = Integer(2, 30, transform="normalize", dtype=dtype) for X in range(2, 31): X_orig = a.inverse_transform(a.transform(X)) assert_array_equal(X_orig, X) assert isinstance(X_orig, dtype)
def test_partly_categorical_space(): dims = Space([Categorical(["a", "b", "c"]), Categorical(["A", "B", "C"])]) assert dims.is_partly_categorical dims = Space([Categorical(["a", "b", "c"]), Integer(1, 2)]) assert dims.is_partly_categorical assert not dims.is_categorical dims = Space([Integer(1, 2), Integer(1, 2)]) assert not dims.is_partly_categorical
def test_searchcv_sklearn_compatibility(): """ Test whether the BayesSearchCV is compatible with base sklearn methods such as clone, set_params, get_params. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, random_state=0) # used to try different model classes pipe = Pipeline([("model", SVC())]) # single categorical value of 'model' parameter sets the model class lin_search = { "model": Categorical([LinearSVC()]), "model__C": Real(1e-6, 1e6, prior="log-uniform"), } dtc_search = { "model": Categorical([DecisionTreeClassifier()]), "model__max_depth": Integer(1, 32), "model__min_samples_split": Real(1e-3, 1.0, prior="log-uniform"), } svc_search = { "model": Categorical([SVC()]), "model__C": Real(1e-6, 1e6, prior="log-uniform"), "model__gamma": Real(1e-6, 1e1, prior="log-uniform"), "model__degree": Integer(1, 8), "model__kernel": Categorical(["linear", "poly", "rbf"]), } opt = BayesSearchCV(pipe, [(lin_search, 1), svc_search], n_iter=2) opt_clone = clone(opt) params, params_clone = opt.get_params(), opt_clone.get_params() assert params.keys() == params_clone.keys() for param, param_clone in zip(params.items(), params_clone.items()): assert param[0] == param_clone[0] assert isinstance(param[1], type(param_clone[1])) opt.set_params(search_spaces=[(dtc_search, 1)]) opt.fit(X_train, y_train) opt_clone.fit(X_train, y_train) total_evaluations = len(opt.cv_results_["mean_test_score"]) total_evaluations_clone = len(opt_clone.cv_results_["mean_test_score"]) # test if expected number of subspaces is explored assert total_evaluations == 1 assert total_evaluations_clone == 1 + 2
def _fit_svc(n_jobs=1, n_points=1, cv=None): """ Utility function to fit a larger classification task with SVC """ X, y = make_classification( n_samples=1000, n_features=20, n_redundant=0, n_informative=18, random_state=1, n_clusters_per_class=1, ) opt = BayesSearchCV( SVC(), { "C": Real(1e-3, 1e3, prior="log-uniform"), "gamma": Real(1e-3, 1e1, prior="log-uniform"), "degree": Integer(1, 3), }, n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, random_state=42, ) opt.fit(X, y) assert opt.score(X, y) > 0.9 opt2 = BayesSearchCV( SVC(), { "C": Real(1e-3, 1e3, prior="log-uniform"), "gamma": Real(1e-3, 1e1, prior="log-uniform"), "degree": Integer(1, 3), }, n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, random_state=42, ) opt2.fit(X, y) assert opt.score(X, y) == opt2.score(X, y)
def test_normalize_integer(): a = Integer(2, 30, transform="normalize") for i in range(50): check_limits(a.rvs(random_state=i), 2, 30) assert_array_equal(a.transformed_bounds, (0, 1)) rng = np.random.RandomState(0) X = rng.randint(2, 31, dtype=np.int64) # Check transformed values are in [0, 1] assert np.all(a.transform(X) <= np.ones_like(X)) assert np.all(np.zeros_like(X) <= a.transform(X)) # Check inverse transform X_orig = a.inverse_transform(a.transform(X)) assert isinstance(X_orig, np.int64) assert_array_equal(X_orig, X) a = Integer(2, 30, transform="normalize", dtype=int) X = rng.randint(2, 31, dtype=int) # Check inverse transform X_orig = a.inverse_transform(a.transform(X)) assert isinstance(X_orig, int) a = Integer(2, 30, transform="normalize", dtype="int") X = rng.randint(2, 31, dtype=int) # Check inverse transform X_orig = a.inverse_transform(a.transform(X)) assert isinstance(X_orig, int) a = Integer(2, 30, prior="log-uniform", base=2, transform="normalize", dtype=int) for i in range(50): check_limits(a.rvs(random_state=i), 2, 30) assert_array_equal(a.transformed_bounds, (0, 1)) X = rng.randint(2, 31, dtype=int) # Check transformed values are in [0, 1] assert np.all(a.transform(X) <= np.ones_like(X)) assert np.all(np.zeros_like(X) <= a.transform(X)) # Check inverse transform X_orig = a.inverse_transform(a.transform(X)) assert isinstance(X_orig, int) assert_array_equal(X_orig, X)
def test_searchcv_runs_multiple_subspaces(): """ Test whether the BayesSearchCV runs without exceptions when multiple subspaces are given. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, random_state=0) # used to try different model classes pipe = Pipeline([("model", SVC())]) # single categorical value of 'model' parameter sets the model class lin_search = { "model": Categorical([LinearSVC()]), "model__C": Real(1e-6, 1e6, prior="log-uniform"), } dtc_search = { "model": Categorical([DecisionTreeClassifier()]), "model__max_depth": Integer(1, 32), "model__min_samples_split": Real(1e-3, 1.0, prior="log-uniform"), } svc_search = { "model": Categorical([SVC()]), "model__C": Real(1e-6, 1e6, prior="log-uniform"), "model__gamma": Real(1e-6, 1e1, prior="log-uniform"), "model__degree": Integer(1, 8), "model__kernel": Categorical(["linear", "poly", "rbf"]), } opt = BayesSearchCV(pipe, [(lin_search, 1), (dtc_search, 1), svc_search], n_iter=2) opt.fit(X_train, y_train) # test if all subspaces are explored total_evaluations = len(opt.cv_results_["mean_test_score"]) assert total_evaluations == 1 + 1 + 2, "Not all spaces were explored!" assert len(opt.optimizer_results_) == 3 assert isinstance(opt.optimizer_results_[0].x[0], LinearSVC) assert isinstance(opt.optimizer_results_[1].x[0], DecisionTreeClassifier) assert isinstance(opt.optimizer_results_[2].x[0], SVC)
def test_searchcv_refit(): """ Test whether results of BayesSearchCV can be reproduced with a fixed random state. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, random_state=0) random_state = 42 opt = BayesSearchCV( SVC(random_state=random_state), { "C": Real(1e-6, 1e6, prior="log-uniform"), "gamma": Real(1e-6, 1e1, prior="log-uniform"), "degree": Integer(1, 8), "kernel": Categorical(["linear", "poly", "rbf"]), }, n_iter=11, random_state=random_state, ) opt2 = BayesSearchCV( SVC(random_state=random_state), { "C": Real(1e-6, 1e6, prior="log-uniform"), "gamma": Real(1e-6, 1e1, prior="log-uniform"), "degree": Integer(1, 8), "kernel": Categorical(["linear", "poly", "rbf"]), }, n_iter=11, random_state=random_state, refit=True, ) opt.fit(X_train, y_train) opt2.best_estimator_ = opt.best_estimator_ opt2.fit(X_train, y_train) # this normally does not hold only if something is wrong # with the optimizaiton procedure as such assert opt2.score(X_test, y_test) > 0.9
def test_space_names_in_use_named_args(): space = [Integer(250, 2000, name="n_estimators")] @use_named_args(space) def objective(n_estimators): return n_estimators res = gp_minimize(objective, space, n_calls=10, random_state=0) best_params = dict(zip((s.name for s in res.space), res.x)) assert "n_estimators" in best_params assert res.space.dimensions[0].name == "n_estimators"
def test_normalize_types(): # can you pass a Space instance to the Space constructor? space = Space([(0.0, 1.0), Integer(-5, 5, dtype=int), (True, False)]) space.set_transformer("normalize") X = [[0.0, -5, False]] Xt = np.zeros((1, 3)) assert_array_equal(space.transform(X), Xt) assert_array_equal(space.inverse_transform(Xt), X) assert_array_equal(space.inverse_transform(space.transform(X)), X) assert isinstance(space.inverse_transform(Xt)[0][0], float) assert isinstance(space.inverse_transform(Xt)[0][1], int) assert isinstance(space.inverse_transform(Xt)[0][2], (np.bool_, bool))
def test_integer(): a = Integer(1, 10) for i in range(50): r = a.rvs(random_state=i) assert 1 <= r assert 11 >= r assert r in a random_values = a.rvs(random_state=0, n_samples=10) assert_array_equal(random_values.shape, (10)) assert_array_equal(a.transform(random_values), random_values) assert_array_equal(a.inverse_transform(random_values), random_values)
def test_searchcv_runs(surrogate, n_jobs, n_points, cv=None): """ Test whether the cross validation search wrapper around sklearn models runs properly with available surrogates and with single or multiple workers and different number of parameter settings to ask from the optimizer in parallel. Parameters ---------- * `surrogate` [str or None]: A class of the scikit-optimize surrogate used. None means to use default surrogate. * `n_jobs` [int]: Number of parallel processes to use for computations. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, random_state=0) # create an instance of a surrogate if it is not a string if surrogate is not None: optimizer_kwargs = {"base_estimator": surrogate} else: optimizer_kwargs = None opt = BayesSearchCV( SVC(), { "C": Real(1e-6, 1e6, prior="log-uniform"), "gamma": Real(1e-6, 1e1, prior="log-uniform"), "degree": Integer(1, 8), "kernel": Categorical(["linear", "poly", "rbf"]), }, n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, optimizer_kwargs=optimizer_kwargs, ) opt.fit(X_train, y_train) # this normally does not hold only if something is wrong # with the optimizaiton procedure as such assert opt.score(X_test, y_test) > 0.9
def test_searchcv_reproducibility(): """ Test whether results of BayesSearchCV can be reproduced with a fixed random state. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, random_state=0) random_state = 42 opt = BayesSearchCV( SVC(random_state=random_state), { "C": Real(1e-6, 1e6, prior="log-uniform"), "gamma": Real(1e-6, 1e1, prior="log-uniform"), "degree": Integer(1, 8), "kernel": Categorical(["linear", "poly", "rbf"]), }, n_iter=11, random_state=random_state, ) opt.fit(X_train, y_train) best_est = opt.best_estimator_ optim_res = opt.optimizer_results_[0].x opt2 = clone(opt).fit(X_train, y_train) best_est2 = opt2.best_estimator_ optim_res2 = opt2.optimizer_results_[0].x assert getattr(best_est, "C") == getattr(best_est2, "C") assert getattr(best_est, "gamma") == getattr(best_est2, "gamma") assert getattr(best_est, "degree") == getattr(best_est2, "degree") assert getattr(best_est, "kernel") == getattr(best_est2, "kernel") # dict is sorted by alphabet assert optim_res[0] == getattr(best_est, "C") assert optim_res[2] == getattr(best_est, "gamma") assert optim_res[1] == getattr(best_est, "degree") assert optim_res[3] == getattr(best_est, "kernel") assert optim_res2[0] == getattr(best_est, "C") assert optim_res2[2] == getattr(best_est, "gamma") assert optim_res2[1] == getattr(best_est, "degree") assert optim_res2[3] == getattr(best_est, "kernel")
def test_dimension_name(): notnames = [1, 1.0, True] for n in notnames: with pytest.raises(ValueError) as exc: real = Real(1, 2, name=n) assert ("Dimension's name must be either string or" "None." == exc.value.args[0]) s = Space([ Real(1, 2, name="a"), Integer(1, 100, name="b"), Categorical(["red, blue"], name="c"), ]) assert s["a"] == (0, s.dimensions[0]) assert s["a", "c"] == [(0, s.dimensions[0]), (2, s.dimensions[2])] assert s[["a", "c"]] == [(0, s.dimensions[0]), (2, s.dimensions[2])] assert s[("a", "c")] == [(0, s.dimensions[0]), (2, s.dimensions[2])] assert s[0] == (0, s.dimensions[0]) assert s[0, "c"] == [(0, s.dimensions[0]), (2, s.dimensions[2])] assert s[0, 2] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
def test_dimensions_names(): from deephyper.skopt.space import Real, Categorical, Integer # create search space and optimizer space = [ Real(0, 1, name="real"), Categorical(["a", "b", "c"], name="cat"), Integer(0, 1, name="int"), ] opt = Optimizer(space, n_initial_points=2) # result of the optimizer missing dimension names result = opt.tell([(0.5, "a", 0.5)], [3]) names = [] for d in result.space.dimensions: names.append(d.name) assert len(names) == 3 assert "real" in names assert "cat" in names assert "int" in names assert None not in names
def test_searchcv_rank(): """ Test whether results of BayesSearchCV can be reproduced with a fixed random state. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, random_state=0) random_state = 42 opt = BayesSearchCV( SVC(random_state=random_state), { "C": Real(1e-6, 1e6, prior="log-uniform"), "gamma": Real(1e-6, 1e1, prior="log-uniform"), "degree": Integer(1, 8), "kernel": Categorical(["linear", "poly", "rbf"]), }, n_iter=11, random_state=random_state, return_train_score=True, ) opt.fit(X_train, y_train) results = opt.cv_results_ test_rank = np.asarray(rankdata(-np.array(results["mean_test_score"]), method="min"), dtype=np.int32) train_rank = np.asarray(rankdata(-np.array(results["mean_train_score"]), method="min"), dtype=np.int32) assert_array_equal(np.array(results["rank_test_score"]), test_rank) assert_array_equal(np.array(results["rank_train_score"]), train_rank)
def test_space_consistency(): # Reals (uniform) s1 = Space([Real(0.0, 1.0)]) s2 = Space([Real(0.0, 1.0)]) s3 = Space([Real(0, 1)]) s4 = Space([(0.0, 1.0)]) s5 = Space([(0.0, 1.0, "uniform")]) s6 = Space([(0, 1.0)]) s7 = Space([(np.float64(0.0), 1.0)]) s8 = Space([(0, np.float64(1.0))]) a1 = s1.rvs(n_samples=10, random_state=0) a2 = s2.rvs(n_samples=10, random_state=0) a3 = s3.rvs(n_samples=10, random_state=0) a4 = s4.rvs(n_samples=10, random_state=0) a5 = s5.rvs(n_samples=10, random_state=0) assert_equal(s1, s2) assert_equal(s1, s3) assert_equal(s1, s4) assert_equal(s1, s5) assert_equal(s1, s6) assert_equal(s1, s7) assert_equal(s1, s8) assert_array_equal(a1, a2) assert_array_equal(a1, a3) assert_array_equal(a1, a4) assert_array_equal(a1, a5) # Reals (log-uniform) s1 = Space([Real(10**-3.0, 10**3.0, prior="log-uniform", base=10)]) s2 = Space([Real(10**-3.0, 10**3.0, prior="log-uniform", base=10)]) s3 = Space([Real(10**-3, 10**3, prior="log-uniform", base=10)]) s4 = Space([(10**-3.0, 10**3.0, "log-uniform", 10)]) s5 = Space([(np.float64(10**-3.0), 10**3.0, "log-uniform", 10)]) a1 = s1.rvs(n_samples=10, random_state=0) a2 = s2.rvs(n_samples=10, random_state=0) a3 = s3.rvs(n_samples=10, random_state=0) a4 = s4.rvs(n_samples=10, random_state=0) assert_equal(s1, s2) assert_equal(s1, s3) assert_equal(s1, s4) assert_equal(s1, s5) assert_array_equal(a1, a2) assert_array_equal(a1, a3) assert_array_equal(a1, a4) # Integers s1 = Space([Integer(1, 5)]) s2 = Space([Integer(1.0, 5.0)]) s3 = Space([(1, 5)]) s4 = Space([(np.int64(1.0), 5)]) s5 = Space([(1, np.int64(5.0))]) a1 = s1.rvs(n_samples=10, random_state=0) a2 = s2.rvs(n_samples=10, random_state=0) a3 = s3.rvs(n_samples=10, random_state=0) assert_equal(s1, s2) assert_equal(s1, s3) assert_equal(s1, s4) assert_equal(s1, s5) assert_array_equal(a1, a2) assert_array_equal(a1, a3) # Integers (log-uniform) s1 = Space([Integer(16, 512, prior="log-uniform", base=2)]) s2 = Space([Integer(16.0, 512.0, prior="log-uniform", base=2)]) s3 = Space([(16, 512, "log-uniform", 2)]) s4 = Space([(np.int64(16.0), 512, "log-uniform", 2)]) s5 = Space([(16, np.int64(512.0), "log-uniform", 2)]) a1 = s1.rvs(n_samples=10, random_state=0) a2 = s2.rvs(n_samples=10, random_state=0) a3 = s3.rvs(n_samples=10, random_state=0) assert_equal(s1, s2) assert_equal(s1, s3) assert_equal(s1, s4) assert_equal(s1, s5) assert_array_equal(a1, a2) assert_array_equal(a1, a3) # Categoricals s1 = Space([Categorical(["a", "b", "c"])]) s2 = Space([Categorical(["a", "b", "c"])]) s3 = Space([["a", "b", "c"]]) a1 = s1.rvs(n_samples=10, random_state=0) a2 = s2.rvs(n_samples=10, random_state=0) a3 = s3.rvs(n_samples=10, random_state=0) assert_equal(s1, s2) assert_array_equal(a1, a2) assert_equal(s1, s3) assert_array_equal(a1, a3) s1 = Space([(True, False)]) s2 = Space([Categorical([True, False])]) s3 = Space([np.array([True, False])]) assert s1 == s2 == s3 # Categoricals Integer s1 = Space([Categorical([1, 2, 3])]) s2 = Space([Categorical([1, 2, 3])]) s3 = Space([[1, 2, 3]]) a1 = s1.rvs(n_samples=10, random_state=0) a2 = s2.rvs(n_samples=10, random_state=0) a3 = s3.rvs(n_samples=10, random_state=0) assert_equal(s1, s2) assert_array_equal(a1, a2) assert_equal(s1, s3) assert_array_equal(a1, a3) s1 = Space([(True, False)]) s2 = Space([Categorical([True, False])]) s3 = Space([np.array([True, False])]) assert s1 == s2 == s3
def test_integer_distance(): ints = Integer(1, 10) for i in range(1, 10 + 1): assert_equal(ints.distance(4, i), abs(4 - i))
def test_integer_distance_out_of_range(): ints = Integer(1, 10) assert_raises_regex(RuntimeError, "compute distance for values within", ints.distance, 11, 10)
(((1, 3), (1.0, 3.0)), ("normalize", "normalize")), (((1, 3), ("a", "b", "c")), ("normalize", "onehot")), ], ) def test_normalize_dimensions(dimensions, normalizations): space = normalize_dimensions(dimensions) for dimension, normalization in zip(space, normalizations): assert dimension.transform_ == normalization @pytest.mark.hps_fast_test @pytest.mark.parametrize( "dimension, name", [ (Real(1, 2, name="learning rate"), "learning rate"), (Integer(1, 100, name="no of trees"), "no of trees"), (Categorical(["red, blue"], name="colors"), "colors"), ], ) def test_normalize_dimensions(dimension, name): space = normalize_dimensions([dimension]) assert space.dimensions[0].name == name @pytest.mark.hps_fast_test def test_use_named_args(): """ Test the function wrapper @use_named_args which is used for wrapping an objective function with named args so it can be called by the optimizers which only pass a single list as the arg.
@pytest.mark.parametrize( "dimension, bounds", [(Real, (2, 1)), (Integer, (2, 1)), (Real, (2, 2)), (Integer, (2, 2))], ) def test_dimension_bounds(dimension, bounds): with pytest.raises(ValueError) as exc: dim = dimension(*bounds) assert "has to be less than the upper bound " in exc.value.args[0] @pytest.mark.parametrize( "dimension, name", [ (Real(1, 2, name="learning_rate"), "learning_rate"), (Integer(1, 100, name="n_trees"), "n_trees"), (Categorical(["red, blue"], name="colors"), "colors"), ], ) def test_dimension_name(dimension, name): assert dimension.name == name def test_dimension_name(): notnames = [1, 1.0, True] for n in notnames: with pytest.raises(ValueError) as exc: real = Real(1, 2, name=n) assert ("Dimension's name must be either string or" "None." == exc.value.args[0]) s = Space([