def _fit_svc(n_jobs=1, n_points=1, cv=None):
"""
Utility function to fit a larger classification task with SVC
"""
X, y = make_classification(n_samples=1000, n_features=20, n_redundant=0,
n_informative=18, random_state=1,
n_clusters_per_class=1)
opt = WeightedBayesSearchCV(
SVC(),
{
'C': Real(1e-3, 1e+3, prior='log-uniform'),
'gamma': Real(1e-3, 1e+1, prior='log-uniform'),
'degree': Integer(1, 3),
},
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, random_state=42
)
opt.fit(X, y)
assert opt.score(X, y) > 0.9
opt2 = WeightedBayesSearchCV(
SVC(),
{
'C': Real(1e-3, 1e+3, prior='log-uniform'),
'gamma': Real(1e-3, 1e+1, prior='log-uniform'),
'degree': Integer(1, 3),
},
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv,
random_state=42,
)
opt2.fit(X, y)
assert opt.score(X, y) == opt2.score(X, y)
def test_searchcv_callback():
# Test whether callback is used in WeightedBayesSearchCV and
# whether is can be used to interrupt the search loop
X, y = load_iris(True)
opt = WeightedBayesSearchCV(
DecisionTreeClassifier(),
{
'max_depth': [3], # additional test for single dimension
'min_samples_split': Real(0.1, 0.9),
},
n_iter=5
)
total_iterations = [0]
def callback(opt_result):
# this simply counts iterations
total_iterations[0] += 1
# break the optimization loop at some point
if total_iterations[0] > 2:
return True # True == stop optimization
return False
opt.fit(X, y, callback=callback)
assert total_iterations[0] == 3
# test whether final model was fit
opt.score(X, y)
def test_searchcv_runs(surrogate, n_jobs, n_points, cv=None):
"""
Test whether the cross validation search wrapper around sklearn
models runs properly with available surrogates and with single
or multiple workers and different number of parameter settings
to ask from the optimizer in parallel.
Parameters
----------
* `surrogate` [str or None]:
A class of the scikit-optimize surrogate used. None means
to use default surrogate.
* `n_jobs` [int]:
Number of parallel processes to use for computations.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(
X, y, train_size=0.75, random_state=0
)
# create an instance of a surrogate if it is not a string
if surrogate is not None:
optimizer_kwargs = {'base_estimator': surrogate}
else:
optimizer_kwargs = None
opt = WeightedBayesSearchCV(
SVC(),
{
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly', 'rbf']),
},
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv,
optimizer_kwargs=optimizer_kwargs
)
opt.fit(X_train, y_train)
# this normally does not hold only if something is wrong
# with the optimizaiton procedure as such
assert opt.score(X_test, y_test) > 0.9
def _fit_svr(n_jobs=1, n_points=1, cv=None):
"""Utility function to fit a larger regression task with SVR
"""
X, y = make_regression(n_samples=1000, n_features=20,
n_informative=18, random_state=1)
opt = WeightedBayesSearchCV(
SVR(),
{
'C': Real(1e-3, 1e+3, prior='log-uniform'),
'gamma': Real(1e-3, 1e+1, prior='log-uniform'),
'degree': Integer(1, 3),
},
scoring=make_scorer(mean_squared_error, greater_is_better=False),
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, random_state=42
)
opt.fit(X, y)
assert opt.score(X,y) > -70000
def test_searchcv_refit():
"""
Test whether results of WeightedBayesSearchCV can be reproduced with a fixed
random state.
"""
X, y = load_iris(True)
X_train, X_test, y_train, y_test = train_test_split(
X, y, train_size=0.75, random_state=0
)
random_state = 42
opt = WeightedBayesSearchCV(
SVC(random_state=random_state),
{
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly', 'rbf']),
},
n_iter=11, random_state=random_state
)
opt2 = WeightedBayesSearchCV(
SVC(random_state=random_state),
{
'C': Real(1e-6, 1e+6, prior='log-uniform'),
'gamma': Real(1e-6, 1e+1, prior='log-uniform'),
'degree': Integer(1, 8),
'kernel': Categorical(['linear', 'poly', 'rbf']),
},
n_iter=11, random_state=random_state, refit=True
)
opt.fit(X_train, y_train)
opt2.best_estimator_ = opt.best_estimator_
opt2.fit(X_train, y_train)
# this normally does not hold only if something is wrong
# with the optimizaiton procedure as such
assert opt2.score(X_test, y_test) > 0.9
def _fit_reg_weighted_cv_e(estimator, search_spaces, n_jobs=1, n_points=1, cv=None, random_state=13):
"""
Utility function to fit a larger regression task with the provided estimator, search space and randomly filled sample weight
:return score
"""
X, y = make_regression(n_samples=1000, n_features=20,
n_informative=18, random_state=1)
sample_weight = np.random.rand(len(y))
pipeline = Pipeline([('estimator', estimator)])
opt = WeightedBayesSearchCV(
pipeline,
scoring=make_scorer(mean_squared_error, greater_is_better=False),
random_state=random_state,
search_spaces=search_spaces,
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv
)
opt.fit(X, y, sample_weight=sample_weight, sample_weight_steps=['estimator'])
score = opt.score(X, y)
return score
def _fit_class_weighted_cv_e(estimator, search_spaces, n_jobs=1, n_points=1, cv=None, random_state=13):
"""
Utility function to fit a larger classification task with the provided estimator, search space and randomly filled sample weight
:return score accuracy
"""
X, y = make_classification(n_samples=1000, n_features=20, n_redundant=0,
n_informative=18, random_state=1,
n_clusters_per_class=1)
sample_weight = np.random.rand(len(y))
pipeline = Pipeline([('estimator', estimator)])
opt = WeightedBayesSearchCV(
pipeline,
random_state=random_state,
search_spaces=search_spaces,
n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv
)
opt.fit(X, y, sample_weight=sample_weight, sample_weight_steps=['estimator'])
score = opt.score(X, y)
return score
# log-uniform: understand as search over p = exp(x) by varying x
opt = WeightedBayesSearchCV(
SVC(),
{
'C': (1e-6, 1e+6, 'log-uniform'),
'gamma': (1e-6, 1e+1, 'log-uniform'),
'degree': (1, 8), # integer valued parameter
'kernel': ['linear', 'poly', 'rbf'], # categorical parameter
},
n_iter=32,
cv=3)
opt.fit(X_train, y_train)
print("val. score: %s" % opt.best_score_)
print("test score: %s" % opt.score(X_test, y_test))
#############################################################################
# Advanced example
# ================
#
# In practice, one wants to enumerate over multiple predictive model classes,
# with different search spaces and number of evaluations per class. An
# example of such search over parameters of Linear SVM, Kernel SVM, and
# decision trees is given below.
from skopt import WeightedBayesSearchCV
from skopt.space import Real, Categorical, Integer
from skopt.plots import plot_objective, plot_histogram
from sklearn.datasets import load_digits