def test_partly_categorical_space():
dims = Space([Categorical(["a", "b", "c"]), Categorical(["A", "B", "C"])])
assert dims.is_partly_categorical
dims = Space([Categorical(["a", "b", "c"]), Integer(1, 2)])
assert dims.is_partly_categorical
assert not dims.is_categorical
dims = Space([Integer(1, 2), Integer(1, 2)])
assert not dims.is_partly_categorical
def test_purely_categorical_space():
# Test reproduces the bug in #908, make sure it doesn't come back
dims = [Categorical(["a", "b", "c"]), Categorical(["A", "B", "C"])]
optimizer = Optimizer(dims, n_initial_points=2, random_state=3)
for _ in range(2):
x = optimizer.ask()
# before the fix this call raised an exception
optimizer.tell(x, np.random.uniform())
def test_searchcv_sklearn_compatibility():
"""
Test whether the BayesSearchCV is compatible with base sklearn methods
such as clone, set_params, get_params.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
train_size=0.75,
random_state=0)
# used to try different model classes
pipe = Pipeline([("model", SVC())])
# single categorical value of 'model' parameter sets the model class
lin_search = {
"model": Categorical([LinearSVC()]),
"model__C": Real(1e-6, 1e6, prior="log-uniform"),
}
dtc_search = {
"model": Categorical([DecisionTreeClassifier()]),
"model__max_depth": Integer(1, 32),
"model__min_samples_split": Real(1e-3, 1.0, prior="log-uniform"),
}
svc_search = {
"model": Categorical([SVC()]),
"model__C": Real(1e-6, 1e6, prior="log-uniform"),
"model__gamma": Real(1e-6, 1e1, prior="log-uniform"),
"model__degree": Integer(1, 8),
"model__kernel": Categorical(["linear", "poly", "rbf"]),
}
opt = BayesSearchCV(pipe, [(lin_search, 1), svc_search], n_iter=2)
opt_clone = clone(opt)
params, params_clone = opt.get_params(), opt_clone.get_params()
assert params.keys() == params_clone.keys()
for param, param_clone in zip(params.items(), params_clone.items()):
assert param[0] == param_clone[0]
assert isinstance(param[1], type(param_clone[1]))
opt.set_params(search_spaces=[(dtc_search, 1)])
opt.fit(X_train, y_train)
opt_clone.fit(X_train, y_train)
total_evaluations = len(opt.cv_results_["mean_test_score"])
total_evaluations_clone = len(opt_clone.cv_results_["mean_test_score"])
# test if expected number of subspaces is explored
assert total_evaluations == 1
assert total_evaluations_clone == 1 + 2
def test_categorical_distance():
categories = ["car", "dog", "orange"]
cat = Categorical(categories)
for cat1 in categories:
for cat2 in categories:
delta = cat.distance(cat1, cat2)
if cat1 == cat2:
assert delta == 0
else:
assert delta == 1
def test_categorical_transform():
categories = ["apple", "orange", "banana", None, True, False, 3]
cat = Categorical(categories)
apple = [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
orange = [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0]
banana = [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0]
none = [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0]
true = [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0]
false = [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0]
three = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]
assert_equal(cat.transformed_size, 7)
assert_equal(cat.transformed_size, cat.transform(["apple"]).size)
assert_array_equal(cat.transform(categories),
[apple, orange, banana, none, true, false, three])
assert_array_equal(cat.transform(["apple", "orange"]), [apple, orange])
assert_array_equal(cat.transform(["apple", "banana"]), [apple, banana])
assert_array_equal(cat.inverse_transform([apple, orange]),
["apple", "orange"])
assert_array_equal(cat.inverse_transform([apple, banana]),
["apple", "banana"])
ent_inverse = cat.inverse_transform(
[apple, orange, banana, none, true, false, three])
assert_array_equal(ent_inverse, categories)
def test_searchcv_runs_multiple_subspaces():
"""
Test whether the BayesSearchCV runs without exceptions when
multiple subspaces are given.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
train_size=0.75,
random_state=0)
# used to try different model classes
pipe = Pipeline([("model", SVC())])
# single categorical value of 'model' parameter sets the model class
lin_search = {
"model": Categorical([LinearSVC()]),
"model__C": Real(1e-6, 1e6, prior="log-uniform"),
}
dtc_search = {
"model": Categorical([DecisionTreeClassifier()]),
"model__max_depth": Integer(1, 32),
"model__min_samples_split": Real(1e-3, 1.0, prior="log-uniform"),
}
svc_search = {
"model": Categorical([SVC()]),
"model__C": Real(1e-6, 1e6, prior="log-uniform"),
"model__gamma": Real(1e-6, 1e1, prior="log-uniform"),
"model__degree": Integer(1, 8),
"model__kernel": Categorical(["linear", "poly", "rbf"]),
}
opt = BayesSearchCV(pipe, [(lin_search, 1), (dtc_search, 1), svc_search],
n_iter=2)
opt.fit(X_train, y_train)
# test if all subspaces are explored
total_evaluations = len(opt.cv_results_["mean_test_score"])
assert total_evaluations == 1 + 1 + 2, "Not all spaces were explored!"
assert len(opt.optimizer_results_) == 3
assert isinstance(opt.optimizer_results_[0].x[0], LinearSVC)
assert isinstance(opt.optimizer_results_[1].x[0], DecisionTreeClassifier)
assert isinstance(opt.optimizer_results_[2].x[0], SVC)
def test_searchcv_refit():
"""
Test whether results of BayesSearchCV can be reproduced with a fixed
random state.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
train_size=0.75,
random_state=0)
random_state = 42
opt = BayesSearchCV(
SVC(random_state=random_state),
{
"C": Real(1e-6, 1e6, prior="log-uniform"),
"gamma": Real(1e-6, 1e1, prior="log-uniform"),
"degree": Integer(1, 8),
"kernel": Categorical(["linear", "poly", "rbf"]),
},
n_iter=11,
random_state=random_state,
)
opt2 = BayesSearchCV(
SVC(random_state=random_state),
{
"C": Real(1e-6, 1e6, prior="log-uniform"),
"gamma": Real(1e-6, 1e1, prior="log-uniform"),
"degree": Integer(1, 8),
"kernel": Categorical(["linear", "poly", "rbf"]),
},
n_iter=11,
random_state=random_state,
refit=True,
)
opt.fit(X_train, y_train)
opt2.best_estimator_ = opt.best_estimator_
opt2.fit(X_train, y_train)
# this normally does not hold only if something is wrong
# with the optimizaiton procedure as such
assert opt2.score(X_test, y_test) > 0.9
def test_categorical_only2():
from numpy import linalg
from deephyper.skopt.space import Categorical
from deephyper.skopt.learning import GaussianProcessRegressor
space = [Categorical([1, 2, 3]), Categorical([4, 5, 6])]
opt = Optimizer(
space,
base_estimator=GaussianProcessRegressor(alpha=1e-7),
acq_optimizer="lbfgs",
n_initial_points=10,
n_jobs=2,
)
next_x = opt.ask(n_points=4)
assert len(next_x) == 4
opt.tell(next_x, [linalg.norm(x) for x in next_x])
next_x = opt.ask(n_points=4)
assert len(next_x) == 4
opt.tell(next_x, [linalg.norm(x) for x in next_x])
next_x = opt.ask(n_points=4)
assert len(next_x) == 4
def test_normalize_bounds():
bounds = [(-999, 189000), Categorical((True, False))]
space = Space(normalize_dimensions(bounds))
for a in np.linspace(1e-9, 0.4999, 1000):
x = space.inverse_transform([[a, a]])
check_limits(x[0][0], -999, 189000)
y = space.transform(x)
check_limits(y, 0.0, 1.0)
for a in np.linspace(0.50001, 1e-9 + 1.0, 1000):
x = space.inverse_transform([[a, a]])
check_limits(x[0][0], -999, 189000)
y = space.transform(x)
check_limits(y, 0.0, 1.0)
def test_searchcv_runs(surrogate, n_jobs, n_points, cv=None):
"""
Test whether the cross validation search wrapper around sklearn
models runs properly with available surrogates and with single
or multiple workers and different number of parameter settings
to ask from the optimizer in parallel.
Parameters
----------
* `surrogate` [str or None]:
A class of the scikit-optimize surrogate used. None means
to use default surrogate.
* `n_jobs` [int]:
Number of parallel processes to use for computations.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
train_size=0.75,
random_state=0)
# create an instance of a surrogate if it is not a string
if surrogate is not None:
optimizer_kwargs = {"base_estimator": surrogate}
else:
optimizer_kwargs = None
opt = BayesSearchCV(
SVC(),
{
"C": Real(1e-6, 1e6, prior="log-uniform"),
"gamma": Real(1e-6, 1e1, prior="log-uniform"),
"degree": Integer(1, 8),
"kernel": Categorical(["linear", "poly", "rbf"]),
},
n_jobs=n_jobs,
n_iter=11,
n_points=n_points,
cv=cv,
optimizer_kwargs=optimizer_kwargs,
)
opt.fit(X_train, y_train)
# this normally does not hold only if something is wrong
# with the optimizaiton procedure as such
assert opt.score(X_test, y_test) > 0.9
def test_categorical_repr():
small_cat = Categorical([1, 2, 3, 4, 5])
assert small_cat.__repr__(
) == "Categorical(categories=(1, 2, 3, 4, 5), prior=None)"
big_cat = Categorical([1, 2, 3, 4, 5, 6, 7, 8])
assert (big_cat.__repr__() ==
"Categorical(categories=(1, 2, 3, ..., 6, 7, 8), prior=None)")
def test_categorical_identity():
categories = ["cat", "dog", "rat"]
cat = Categorical(categories, transform="identity")
samples = cat.rvs(100)
assert all([t in categories for t in cat.rvs(100)])
transformed = cat.transform(samples)
assert_array_equal(transformed, samples)
assert_array_equal(samples, cat.inverse_transform(transformed))
def test_categorical_string():
categories = [1, 2, 3]
categories_transformed = ["1", "2", "3"]
cat = Categorical(categories, transform="string")
samples = cat.rvs(100)
assert all([t in categories for t in cat.rvs(100)])
transformed = cat.transform(samples)
assert all([t in categories_transformed for t in transformed])
assert_array_equal(samples, cat.inverse_transform(transformed))
def test_categorical_only():
from deephyper.skopt.space import Categorical
cat1 = Categorical([2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
cat2 = Categorical([2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
opt = Optimizer([cat1, cat2])
for n in range(15):
x = opt.ask()
res = opt.tell(x, 12 * n)
assert len(res.x_iters) == 15
next_x = opt.ask(n_points=4)
assert len(next_x) == 4
cat3 = Categorical(["2", "3", "4", "5", "6", "7", "8", "9", "10", "11"])
cat4 = Categorical(["2", "3", "4", "5", "6", "7", "8", "9", "10", "11"])
opt = Optimizer([cat3, cat4])
for n in range(15):
x = opt.ask()
res = opt.tell(x, 12 * n)
assert len(res.x_iters) == 15
next_x = opt.ask(n_points=4)
assert len(next_x) == 4
def test_searchcv_reproducibility():
"""
Test whether results of BayesSearchCV can be reproduced with a fixed
random state.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
train_size=0.75,
random_state=0)
random_state = 42
opt = BayesSearchCV(
SVC(random_state=random_state),
{
"C": Real(1e-6, 1e6, prior="log-uniform"),
"gamma": Real(1e-6, 1e1, prior="log-uniform"),
"degree": Integer(1, 8),
"kernel": Categorical(["linear", "poly", "rbf"]),
},
n_iter=11,
random_state=random_state,
)
opt.fit(X_train, y_train)
best_est = opt.best_estimator_
optim_res = opt.optimizer_results_[0].x
opt2 = clone(opt).fit(X_train, y_train)
best_est2 = opt2.best_estimator_
optim_res2 = opt2.optimizer_results_[0].x
assert getattr(best_est, "C") == getattr(best_est2, "C")
assert getattr(best_est, "gamma") == getattr(best_est2, "gamma")
assert getattr(best_est, "degree") == getattr(best_est2, "degree")
assert getattr(best_est, "kernel") == getattr(best_est2, "kernel")
# dict is sorted by alphabet
assert optim_res[0] == getattr(best_est, "C")
assert optim_res[2] == getattr(best_est, "gamma")
assert optim_res[1] == getattr(best_est, "degree")
assert optim_res[3] == getattr(best_est, "kernel")
assert optim_res2[0] == getattr(best_est, "C")
assert optim_res2[2] == getattr(best_est, "gamma")
assert optim_res2[1] == getattr(best_est, "degree")
assert optim_res2[3] == getattr(best_est, "kernel")
def test_categorical_transform_binary():
categories = ["apple", "orange"]
cat = Categorical(categories)
apple = [0.0]
orange = [1.0]
assert_equal(cat.transformed_size, 1)
assert_equal(cat.transformed_size, cat.transform(["apple"]).size)
assert_array_equal(cat.transform(categories), [apple, orange])
assert_array_equal(cat.transform(["apple", "orange"]), [apple, orange])
assert_array_equal(cat.inverse_transform([apple, orange]),
["apple", "orange"])
ent_inverse = cat.inverse_transform([apple, orange])
assert_array_equal(ent_inverse, categories)
def test_dimension_name():
notnames = [1, 1.0, True]
for n in notnames:
with pytest.raises(ValueError) as exc:
real = Real(1, 2, name=n)
assert ("Dimension's name must be either string or"
"None." == exc.value.args[0])
s = Space([
Real(1, 2, name="a"),
Integer(1, 100, name="b"),
Categorical(["red, blue"], name="c"),
])
assert s["a"] == (0, s.dimensions[0])
assert s["a", "c"] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
assert s[["a", "c"]] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
assert s[("a", "c")] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
assert s[0] == (0, s.dimensions[0])
assert s[0, "c"] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
assert s[0, 2] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
def test_dimensions_names():
from deephyper.skopt.space import Real, Categorical, Integer
# create search space and optimizer
space = [
Real(0, 1, name="real"),
Categorical(["a", "b", "c"], name="cat"),
Integer(0, 1, name="int"),
]
opt = Optimizer(space, n_initial_points=2)
# result of the optimizer missing dimension names
result = opt.tell([(0.5, "a", 0.5)], [3])
names = []
for d in result.space.dimensions:
names.append(d.name)
assert len(names) == 3
assert "real" in names
assert "cat" in names
assert "int" in names
assert None not in names
def test_searchcv_rank():
"""
Test whether results of BayesSearchCV can be reproduced with a fixed
random state.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
train_size=0.75,
random_state=0)
random_state = 42
opt = BayesSearchCV(
SVC(random_state=random_state),
{
"C": Real(1e-6, 1e6, prior="log-uniform"),
"gamma": Real(1e-6, 1e1, prior="log-uniform"),
"degree": Integer(1, 8),
"kernel": Categorical(["linear", "poly", "rbf"]),
},
n_iter=11,
random_state=random_state,
return_train_score=True,
)
opt.fit(X_train, y_train)
results = opt.cv_results_
test_rank = np.asarray(rankdata(-np.array(results["mean_test_score"]),
method="min"),
dtype=np.int32)
train_rank = np.asarray(rankdata(-np.array(results["mean_train_score"]),
method="min"),
dtype=np.int32)
assert_array_equal(np.array(results["rank_test_score"]), test_rank)
assert_array_equal(np.array(results["rank_train_score"]), train_rank)
def test_normalize_categorical():
categories = ["cat", "dog", "rat"]
a = Categorical(categories, transform="normalize")
for i in range(len(categories)):
assert a.rvs(random_state=i)[0] in categories
assert a.inverse_transform([0.0]) == [categories[0]]
assert a.inverse_transform([0.5]) == [categories[1]]
assert a.inverse_transform([1.0]) == [categories[2]]
assert_array_equal(categories, a.inverse_transform([0.0, 0.5, 1]))
categories = [1, 2, 3]
a = Categorical(categories, transform="normalize")
assert_array_equal(categories,
np.sort(np.unique(a.rvs(100, random_state=1))))
assert_array_equal(categories, a.inverse_transform([0.0, 0.5, 1.0]))
categories = [1.0, 2.0, 3.0]
a = Categorical(categories, transform="normalize")
assert_array_equal(categories,
np.sort(np.unique(a.rvs(100, random_state=1))))
assert_array_equal(categories, a.inverse_transform([0.0, 0.5, 1.0]))
categories = [1, 2, 3]
a = Categorical(categories, transform="string")
a.set_transformer("normalize")
assert_array_equal(categories,
np.sort(np.unique(a.rvs(100, random_state=1))))
assert_array_equal(categories, a.inverse_transform([0.0, 0.5, 1.0]))
(((1, 3), ("a", "b", "c")), ("normalize", "onehot")),
],
)
def test_normalize_dimensions(dimensions, normalizations):
space = normalize_dimensions(dimensions)
for dimension, normalization in zip(space, normalizations):
assert dimension.transform_ == normalization
@pytest.mark.hps_fast_test
@pytest.mark.parametrize(
"dimension, name",
[
(Real(1, 2, name="learning rate"), "learning rate"),
(Integer(1, 100, name="no of trees"), "no of trees"),
(Categorical(["red, blue"], name="colors"), "colors"),
],
)
def test_normalize_dimensions(dimension, name):
space = normalize_dimensions([dimension])
assert space.dimensions[0].name == name
@pytest.mark.hps_fast_test
def test_use_named_args():
"""
Test the function wrapper @use_named_args which is used
for wrapping an objective function with named args so it
can be called by the optimizers which only pass a single
list as the arg.
def test_space_consistency():
# Reals (uniform)
s1 = Space([Real(0.0, 1.0)])
s2 = Space([Real(0.0, 1.0)])
s3 = Space([Real(0, 1)])
s4 = Space([(0.0, 1.0)])
s5 = Space([(0.0, 1.0, "uniform")])
s6 = Space([(0, 1.0)])
s7 = Space([(np.float64(0.0), 1.0)])
s8 = Space([(0, np.float64(1.0))])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
a4 = s4.rvs(n_samples=10, random_state=0)
a5 = s5.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_equal(s1, s6)
assert_equal(s1, s7)
assert_equal(s1, s8)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
assert_array_equal(a1, a4)
assert_array_equal(a1, a5)
# Reals (log-uniform)
s1 = Space([Real(10**-3.0, 10**3.0, prior="log-uniform", base=10)])
s2 = Space([Real(10**-3.0, 10**3.0, prior="log-uniform", base=10)])
s3 = Space([Real(10**-3, 10**3, prior="log-uniform", base=10)])
s4 = Space([(10**-3.0, 10**3.0, "log-uniform", 10)])
s5 = Space([(np.float64(10**-3.0), 10**3.0, "log-uniform", 10)])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
a4 = s4.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
assert_array_equal(a1, a4)
# Integers
s1 = Space([Integer(1, 5)])
s2 = Space([Integer(1.0, 5.0)])
s3 = Space([(1, 5)])
s4 = Space([(np.int64(1.0), 5)])
s5 = Space([(1, np.int64(5.0))])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
# Integers (log-uniform)
s1 = Space([Integer(16, 512, prior="log-uniform", base=2)])
s2 = Space([Integer(16.0, 512.0, prior="log-uniform", base=2)])
s3 = Space([(16, 512, "log-uniform", 2)])
s4 = Space([(np.int64(16.0), 512, "log-uniform", 2)])
s5 = Space([(16, np.int64(512.0), "log-uniform", 2)])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_equal(s1, s3)
assert_equal(s1, s4)
assert_equal(s1, s5)
assert_array_equal(a1, a2)
assert_array_equal(a1, a3)
# Categoricals
s1 = Space([Categorical(["a", "b", "c"])])
s2 = Space([Categorical(["a", "b", "c"])])
s3 = Space([["a", "b", "c"]])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_array_equal(a1, a2)
assert_equal(s1, s3)
assert_array_equal(a1, a3)
s1 = Space([(True, False)])
s2 = Space([Categorical([True, False])])
s3 = Space([np.array([True, False])])
assert s1 == s2 == s3
# Categoricals Integer
s1 = Space([Categorical([1, 2, 3])])
s2 = Space([Categorical([1, 2, 3])])
s3 = Space([[1, 2, 3]])
a1 = s1.rvs(n_samples=10, random_state=0)
a2 = s2.rvs(n_samples=10, random_state=0)
a3 = s3.rvs(n_samples=10, random_state=0)
assert_equal(s1, s2)
assert_array_equal(a1, a2)
assert_equal(s1, s3)
assert_array_equal(a1, a3)
s1 = Space([(True, False)])
s2 = Space([Categorical([True, False])])
s3 = Space([np.array([True, False])])
assert s1 == s2 == s3
def check_categorical(vals, random_val):
x = Categorical(vals)
assert_equal(x, Categorical(vals))
assert x != Categorical(vals[:-1] + ("zzz", ))
assert_equal(x.rvs(random_state=1), random_val)