def test_searchcv_callback():
# Test whether callback is used in WeightedBayesSearchCV and
# whether is can be used to interrupt the search loop
X, y = load_iris(True)
opt = WeightedBayesSearchCV(
DecisionTreeClassifier(),
{
'max_depth': [3], # additional test for single dimension
'min_samples_split': Real(0.1, 0.9),
},
n_iter=5
)
total_iterations = [0]
def callback(opt_result):
# this simply counts iterations
total_iterations[0] += 1
# break the optimization loop at some point
if total_iterations[0] > 2:
return True # True == stop optimization
return False
opt.fit(X, y, callback=callback)
assert total_iterations[0] == 3
# test whether final model was fit
opt.score(X, y)
def test_searchcv_rank():
"""
Test whether results of WeightedBayesSearchCV can be reproduced with a fixed
random state.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(
X, y, train_size=0.75, random_state=0
)
random_state = 42
opt = WeightedBayesSearchCV(
SVC(random_state=random_state),
{
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly', 'rbf']),
},
n_iter=11, random_state=random_state, return_train_score=True
)
opt.fit(X_train, y_train)
results = opt.cv_results_
test_rank = np.asarray(rankdata(-np.array(results["mean_test_score"]),
method='min'), dtype=np.int32)
train_rank = np.asarray(rankdata(-np.array(results["mean_train_score"]),
method='min'), dtype=np.int32)
assert_array_equal(np.array(results['rank_test_score']), test_rank)
assert_array_equal(np.array(results['rank_train_score']), train_rank)
def _fit_svc(n_jobs=1, n_points=1, cv=None):
"""
Utility function to fit a larger classification task with SVC
"""
X, y = make_classification(n_samples=1000, n_features=20, n_redundant=0,
n_informative=18, random_state=1,
n_clusters_per_class=1)
opt = WeightedBayesSearchCV(
SVC(),
{
'C': Real(1e-3, 1e+3, prior='log-uniform'),
'gamma': Real(1e-3, 1e+1, prior='log-uniform'),
'degree': Integer(1, 3),
},
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, random_state=42
)
opt.fit(X, y)
assert opt.score(X, y) > 0.9
opt2 = WeightedBayesSearchCV(
SVC(),
{
'C': Real(1e-3, 1e+3, prior='log-uniform'),
'gamma': Real(1e-3, 1e+1, prior='log-uniform'),
'degree': Integer(1, 3),
},
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv,
random_state=42,
)
opt2.fit(X, y)
assert opt.score(X, y) == opt2.score(X, y)
def test_bayes(X_train, y_train):
opt = WeightedBayesSearchCV(
SVC(), {
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly', 'rbf']),
},
n_iter=32,
cv=10,
scoring="accuracy")
opt.fit(X_train, y_train)
print("val. score: %s" % opt.best_score_)
def test_searchcv_runs_multiple_subspaces():
"""
Test whether the WeightedBayesSearchCV runs without exceptions when
multiple subspaces are given.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(
X, y, train_size=0.75, random_state=0
)
# used to try different model classes
pipe = Pipeline([
('model', SVC())
])
# single categorical value of 'model' parameter sets the model class
lin_search = {
'model': Categorical([LinearSVC()]),
'model__C': Real(1e-6, 1e+6, prior='log-uniform'),
}
dtc_search = {
'model': Categorical([DecisionTreeClassifier()]),
'model__max_depth': Integer(1, 32),
'model__min_samples_split': Real(1e-3, 1.0, prior='log-uniform'),
}
svc_search = {
'model': Categorical([SVC()]),
'model__C': Real(1e-6, 1e+6, prior='log-uniform'),
'model__gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'model__degree': Integer(1, 8),
'model__kernel': Categorical(['linear', 'poly', 'rbf']),
}
opt = WeightedBayesSearchCV(
pipe,
[(lin_search, 1), (dtc_search, 1), svc_search],
n_iter=2
)
opt.fit(X_train, y_train)
# test if all subspaces are explored
total_evaluations = len(opt.cv_results_['mean_test_score'])
assert total_evaluations == 1 + 1 + 2, "Not all spaces were explored!"
assert len(opt.optimizer_results_) == 3
assert isinstance(opt.optimizer_results_[0].x[0], LinearSVC)
assert isinstance(opt.optimizer_results_[1].x[0], DecisionTreeClassifier)
assert isinstance(opt.optimizer_results_[2].x[0], SVC)
def test_searchcv_runs(surrogate, n_jobs, n_points, cv=None):
"""
Test whether the cross validation search wrapper around sklearn
models runs properly with available surrogates and with single
or multiple workers and different number of parameter settings
to ask from the optimizer in parallel.
Parameters
----------
* `surrogate` [str or None]:
A class of the scikit-optimize surrogate used. None means
to use default surrogate.
* `n_jobs` [int]:
Number of parallel processes to use for computations.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(
X, y, train_size=0.75, random_state=0
)
# create an instance of a surrogate if it is not a string
if surrogate is not None:
optimizer_kwargs = {'base_estimator': surrogate}
else:
optimizer_kwargs = None
opt = WeightedBayesSearchCV(
SVC(),
{
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly', 'rbf']),
},
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv,
optimizer_kwargs=optimizer_kwargs
)
opt.fit(X_train, y_train)
# this normally does not hold only if something is wrong
# with the optimizaiton procedure as such
assert opt.score(X_test, y_test) > 0.9
def test_searchcv_reproducibility():
"""
Test whether results of WeightedBayesSearchCV can be reproduced with a fixed
random state.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(
X, y, train_size=0.75, random_state=0
)
random_state = 42
opt = WeightedBayesSearchCV(
SVC(random_state=random_state),
{
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly', 'rbf']),
},
n_iter=11, random_state=random_state
)
opt.fit(X_train, y_train)
best_est = opt.best_estimator_
optim_res = opt.optimizer_results_[0].x
opt2 = clone(opt).fit(X_train, y_train)
best_est2 = opt2.best_estimator_
optim_res2 = opt2.optimizer_results_[0].x
assert getattr(best_est, 'C') == getattr(best_est2, 'C')
assert getattr(best_est, 'gamma') == getattr(best_est2, 'gamma')
assert getattr(best_est, 'degree') == getattr(best_est2, 'degree')
assert getattr(best_est, 'kernel') == getattr(best_est2, 'kernel')
# dict is sorted by alphabet
assert optim_res[0] == getattr(best_est, 'C')
assert optim_res[2] == getattr(best_est, 'gamma')
assert optim_res[1] == getattr(best_est, 'degree')
assert optim_res[3] == getattr(best_est, 'kernel')
assert optim_res2[0] == getattr(best_est, 'C')
assert optim_res2[2] == getattr(best_est, 'gamma')
assert optim_res2[1] == getattr(best_est, 'degree')
assert optim_res2[3] == getattr(best_est, 'kernel')
def _fit_svr(n_jobs=1, n_points=1, cv=None):
"""Utility function to fit a larger regression task with SVR
"""
X, y = make_regression(n_samples=1000, n_features=20,
n_informative=18, random_state=1)
opt = WeightedBayesSearchCV(
SVR(),
{
'C': Real(1e-3, 1e+3, prior='log-uniform'),
'gamma': Real(1e-3, 1e+1, prior='log-uniform'),
'degree': Integer(1, 3),
},
scoring=make_scorer(mean_squared_error, greater_is_better=False),
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, random_state=42
)
opt.fit(X, y)
assert opt.score(X,y) > -70000
def test_searchcv_refit():
"""
Test whether results of WeightedBayesSearchCV can be reproduced with a fixed
random state.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(
X, y, train_size=0.75, random_state=0
)
random_state = 42
opt = WeightedBayesSearchCV(
SVC(random_state=random_state),
{
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly', 'rbf']),
},
n_iter=11, random_state=random_state
)
opt2 = WeightedBayesSearchCV(
SVC(random_state=random_state),
{
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly', 'rbf']),
},
n_iter=11, random_state=random_state, refit=True
)
opt.fit(X_train, y_train)
opt2.best_estimator_ = opt.best_estimator_
opt2.fit(X_train, y_train)
# this normally does not hold only if something is wrong
# with the optimizaiton procedure as such
assert opt2.score(X_test, y_test) > 0.9
def _fit_reg_weighted_cv_e(estimator, search_spaces, n_jobs=1, n_points=1, cv=None, random_state=13):
"""
Utility function to fit a larger regression task with the provided estimator, search space and randomly filled sample weight
:return score
"""
X, y = make_regression(n_samples=1000, n_features=20,
n_informative=18, random_state=1)
sample_weight = np.random.rand(len(y))
pipeline = Pipeline([('estimator', estimator)])
opt = WeightedBayesSearchCV(
pipeline,
scoring=make_scorer(mean_squared_error, greater_is_better=False),
random_state=random_state,
search_spaces=search_spaces,
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv
)
opt.fit(X, y, sample_weight=sample_weight, sample_weight_steps=['estimator'])
score = opt.score(X, y)
return score
def _fit_class_weighted_cv_e(estimator, search_spaces, n_jobs=1, n_points=1, cv=None, random_state=13):
"""
Utility function to fit a larger classification task with the provided estimator, search space and randomly filled sample weight
:return score accuracy
"""
X, y = make_classification(n_samples=1000, n_features=20, n_redundant=0,
n_informative=18, random_state=1,
n_clusters_per_class=1)
sample_weight = np.random.rand(len(y))
pipeline = Pipeline([('estimator', estimator)])
opt = WeightedBayesSearchCV(
pipeline,
random_state=random_state,
search_spaces=search_spaces,
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv
)
opt.fit(X, y, sample_weight=sample_weight, sample_weight_steps=['estimator'])
score = opt.score(X, y)
return score
def test_search_cv_internal_parameter_types():
# Test whether the parameters passed to the
# estimator of the WeightedBayesSearchCV are of standard python
# types - float, int, str
# This is estimator is used to check whether the types provided
# are native python types.
class TypeCheckEstimator(BaseEstimator):
def __init__(self, float_param=0.0, int_param=0, str_param=""):
self.float_param = float_param
self.int_param = int_param
self.str_param = str_param
def fit(self, X, y):
assert isinstance(self.float_param, float)
assert isinstance(self.int_param, int)
assert isinstance(self.str_param, str)
return self
def score(self, X, y):
return 0.0
# Below is example code that used to not work.
X, y = make_classification(10, 4)
model = WeightedBayesSearchCV(
estimator=TypeCheckEstimator(),
search_spaces={
'float_param': [0.0, 1.0],
'int_param': [0, 10],
'str_param': ["one", "two", "three"],
},
n_iter=11
)
model.fit(X, y)
test_size=.25,
random_state=0)
# log-uniform: understand as search over p = exp(x) by varying x
opt = WeightedBayesSearchCV(
SVC(),
{
'C': (1e-6, 1e+6, 'log-uniform'),
'gamma': (1e-6, 1e+1, 'log-uniform'),
'degree': (1, 8), # integer valued parameter
'kernel': ['linear', 'poly', 'rbf'], # categorical parameter
},
n_iter=32,
cv=3)
opt.fit(X_train, y_train)
print("val. score: %s" % opt.best_score_)
print("test score: %s" % opt.score(X_test, y_test))
#############################################################################
# Advanced example
# ================
#
# In practice, one wants to enumerate over multiple predictive model classes,
# with different search spaces and number of evaluations per class. An
# example of such search over parameters of Linear SVM, Kernel SVM, and
# decision trees is given below.
from skopt import WeightedBayesSearchCV
from skopt.space import Real, Categorical, Integer
def test_searchcv_sklearn_compatibility():
"""
Test whether the WeightedBayesSearchCV is compatible with base sklearn methods
such as clone, set_params, get_params.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(
X, y, train_size=0.75, random_state=0
)
# used to try different model classes
pipe = Pipeline([
('model', SVC())
])
# single categorical value of 'model' parameter sets the model class
lin_search = {
'model': Categorical([LinearSVC()]),
'model__C': Real(1e-6, 1e+6, prior='log-uniform'),
}
dtc_search = {
'model': Categorical([DecisionTreeClassifier()]),
'model__max_depth': Integer(1, 32),
'model__min_samples_split': Real(1e-3, 1.0, prior='log-uniform'),
}
svc_search = {
'model': Categorical([SVC()]),
'model__C': Real(1e-6, 1e+6, prior='log-uniform'),
'model__gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'model__degree': Integer(1, 8),
'model__kernel': Categorical(['linear', 'poly', 'rbf']),
}
opt = WeightedBayesSearchCV(
pipe,
[(lin_search, 1), svc_search],
n_iter=2
)
opt_clone = clone(opt)
params, params_clone = opt.get_params(), opt_clone.get_params()
assert params.keys() == params_clone.keys()
for param, param_clone in zip(params.items(), params_clone.items()):
assert param[0] == param_clone[0]
assert isinstance(param[1], type(param_clone[1]))
opt.set_params(search_spaces=[(dtc_search, 1)])
opt.fit(X_train, y_train)
opt_clone.fit(X_train, y_train)
total_evaluations = len(opt.cv_results_['mean_test_score'])
total_evaluations_clone = len(opt_clone.cv_results_['mean_test_score'])
# test if expected number of subspaces is explored
assert total_evaluations == 1
assert total_evaluations_clone == 1 + 2
