def test_searchcv_callback(): # Test whether callback is used in WeightedBayesSearchCV and # whether is can be used to interrupt the search loop X, y = load_iris(True) opt = WeightedBayesSearchCV( DecisionTreeClassifier(), { 'max_depth': [3], # additional test for single dimension 'min_samples_split': Real(0.1, 0.9), }, n_iter=5 ) total_iterations = [0] def callback(opt_result): # this simply counts iterations total_iterations[0] += 1 # break the optimization loop at some point if total_iterations[0] > 2: return True # True == stop optimization return False opt.fit(X, y, callback=callback) assert total_iterations[0] == 3 # test whether final model was fit opt.score(X, y)
def test_searchcv_rank(): """ Test whether results of WeightedBayesSearchCV can be reproduced with a fixed random state. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split( X, y, train_size=0.75, random_state=0 ) random_state = 42 opt = WeightedBayesSearchCV( SVC(random_state=random_state), { 'C': Real(1e-6, 1e+6, prior='log-uniform'), 'gamma': Real(1e-6, 1e+1, prior='log-uniform'), 'degree': Integer(1, 8), 'kernel': Categorical(['linear', 'poly', 'rbf']), }, n_iter=11, random_state=random_state, return_train_score=True ) opt.fit(X_train, y_train) results = opt.cv_results_ test_rank = np.asarray(rankdata(-np.array(results["mean_test_score"]), method='min'), dtype=np.int32) train_rank = np.asarray(rankdata(-np.array(results["mean_train_score"]), method='min'), dtype=np.int32) assert_array_equal(np.array(results['rank_test_score']), test_rank) assert_array_equal(np.array(results['rank_train_score']), train_rank)
def _fit_svc(n_jobs=1, n_points=1, cv=None): """ Utility function to fit a larger classification task with SVC """ X, y = make_classification(n_samples=1000, n_features=20, n_redundant=0, n_informative=18, random_state=1, n_clusters_per_class=1) opt = WeightedBayesSearchCV( SVC(), { 'C': Real(1e-3, 1e+3, prior='log-uniform'), 'gamma': Real(1e-3, 1e+1, prior='log-uniform'), 'degree': Integer(1, 3), }, n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, random_state=42 ) opt.fit(X, y) assert opt.score(X, y) > 0.9 opt2 = WeightedBayesSearchCV( SVC(), { 'C': Real(1e-3, 1e+3, prior='log-uniform'), 'gamma': Real(1e-3, 1e+1, prior='log-uniform'), 'degree': Integer(1, 3), }, n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, random_state=42, ) opt2.fit(X, y) assert opt.score(X, y) == opt2.score(X, y)
def test_bayes(X_train, y_train): opt = WeightedBayesSearchCV( SVC(), { 'C': Real(1e-6, 1e+6, prior='log-uniform'), 'gamma': Real(1e-6, 1e+1, prior='log-uniform'), 'degree': Integer(1, 8), 'kernel': Categorical(['linear', 'poly', 'rbf']), }, n_iter=32, cv=10, scoring="accuracy") opt.fit(X_train, y_train) print("val. score: %s" % opt.best_score_)
def test_searchcv_runs_multiple_subspaces(): """ Test whether the WeightedBayesSearchCV runs without exceptions when multiple subspaces are given. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split( X, y, train_size=0.75, random_state=0 ) # used to try different model classes pipe = Pipeline([ ('model', SVC()) ]) # single categorical value of 'model' parameter sets the model class lin_search = { 'model': Categorical([LinearSVC()]), 'model__C': Real(1e-6, 1e+6, prior='log-uniform'), } dtc_search = { 'model': Categorical([DecisionTreeClassifier()]), 'model__max_depth': Integer(1, 32), 'model__min_samples_split': Real(1e-3, 1.0, prior='log-uniform'), } svc_search = { 'model': Categorical([SVC()]), 'model__C': Real(1e-6, 1e+6, prior='log-uniform'), 'model__gamma': Real(1e-6, 1e+1, prior='log-uniform'), 'model__degree': Integer(1, 8), 'model__kernel': Categorical(['linear', 'poly', 'rbf']), } opt = WeightedBayesSearchCV( pipe, [(lin_search, 1), (dtc_search, 1), svc_search], n_iter=2 ) opt.fit(X_train, y_train) # test if all subspaces are explored total_evaluations = len(opt.cv_results_['mean_test_score']) assert total_evaluations == 1 + 1 + 2, "Not all spaces were explored!" assert len(opt.optimizer_results_) == 3 assert isinstance(opt.optimizer_results_[0].x[0], LinearSVC) assert isinstance(opt.optimizer_results_[1].x[0], DecisionTreeClassifier) assert isinstance(opt.optimizer_results_[2].x[0], SVC)
def test_searchcv_runs(surrogate, n_jobs, n_points, cv=None): """ Test whether the cross validation search wrapper around sklearn models runs properly with available surrogates and with single or multiple workers and different number of parameter settings to ask from the optimizer in parallel. Parameters ---------- * `surrogate` [str or None]: A class of the scikit-optimize surrogate used. None means to use default surrogate. * `n_jobs` [int]: Number of parallel processes to use for computations. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split( X, y, train_size=0.75, random_state=0 ) # create an instance of a surrogate if it is not a string if surrogate is not None: optimizer_kwargs = {'base_estimator': surrogate} else: optimizer_kwargs = None opt = WeightedBayesSearchCV( SVC(), { 'C': Real(1e-6, 1e+6, prior='log-uniform'), 'gamma': Real(1e-6, 1e+1, prior='log-uniform'), 'degree': Integer(1, 8), 'kernel': Categorical(['linear', 'poly', 'rbf']), }, n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, optimizer_kwargs=optimizer_kwargs ) opt.fit(X_train, y_train) # this normally does not hold only if something is wrong # with the optimizaiton procedure as such assert opt.score(X_test, y_test) > 0.9
def test_searchcv_reproducibility(): """ Test whether results of WeightedBayesSearchCV can be reproduced with a fixed random state. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split( X, y, train_size=0.75, random_state=0 ) random_state = 42 opt = WeightedBayesSearchCV( SVC(random_state=random_state), { 'C': Real(1e-6, 1e+6, prior='log-uniform'), 'gamma': Real(1e-6, 1e+1, prior='log-uniform'), 'degree': Integer(1, 8), 'kernel': Categorical(['linear', 'poly', 'rbf']), }, n_iter=11, random_state=random_state ) opt.fit(X_train, y_train) best_est = opt.best_estimator_ optim_res = opt.optimizer_results_[0].x opt2 = clone(opt).fit(X_train, y_train) best_est2 = opt2.best_estimator_ optim_res2 = opt2.optimizer_results_[0].x assert getattr(best_est, 'C') == getattr(best_est2, 'C') assert getattr(best_est, 'gamma') == getattr(best_est2, 'gamma') assert getattr(best_est, 'degree') == getattr(best_est2, 'degree') assert getattr(best_est, 'kernel') == getattr(best_est2, 'kernel') # dict is sorted by alphabet assert optim_res[0] == getattr(best_est, 'C') assert optim_res[2] == getattr(best_est, 'gamma') assert optim_res[1] == getattr(best_est, 'degree') assert optim_res[3] == getattr(best_est, 'kernel') assert optim_res2[0] == getattr(best_est, 'C') assert optim_res2[2] == getattr(best_est, 'gamma') assert optim_res2[1] == getattr(best_est, 'degree') assert optim_res2[3] == getattr(best_est, 'kernel')
def _fit_svr(n_jobs=1, n_points=1, cv=None): """Utility function to fit a larger regression task with SVR """ X, y = make_regression(n_samples=1000, n_features=20, n_informative=18, random_state=1) opt = WeightedBayesSearchCV( SVR(), { 'C': Real(1e-3, 1e+3, prior='log-uniform'), 'gamma': Real(1e-3, 1e+1, prior='log-uniform'), 'degree': Integer(1, 3), }, scoring=make_scorer(mean_squared_error, greater_is_better=False), n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv, random_state=42 ) opt.fit(X, y) assert opt.score(X,y) > -70000
def test_searchcv_refit(): """ Test whether results of WeightedBayesSearchCV can be reproduced with a fixed random state. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split( X, y, train_size=0.75, random_state=0 ) random_state = 42 opt = WeightedBayesSearchCV( SVC(random_state=random_state), { 'C': Real(1e-6, 1e+6, prior='log-uniform'), 'gamma': Real(1e-6, 1e+1, prior='log-uniform'), 'degree': Integer(1, 8), 'kernel': Categorical(['linear', 'poly', 'rbf']), }, n_iter=11, random_state=random_state ) opt2 = WeightedBayesSearchCV( SVC(random_state=random_state), { 'C': Real(1e-6, 1e+6, prior='log-uniform'), 'gamma': Real(1e-6, 1e+1, prior='log-uniform'), 'degree': Integer(1, 8), 'kernel': Categorical(['linear', 'poly', 'rbf']), }, n_iter=11, random_state=random_state, refit=True ) opt.fit(X_train, y_train) opt2.best_estimator_ = opt.best_estimator_ opt2.fit(X_train, y_train) # this normally does not hold only if something is wrong # with the optimizaiton procedure as such assert opt2.score(X_test, y_test) > 0.9
def _fit_reg_weighted_cv_e(estimator, search_spaces, n_jobs=1, n_points=1, cv=None, random_state=13): """ Utility function to fit a larger regression task with the provided estimator, search space and randomly filled sample weight :return score """ X, y = make_regression(n_samples=1000, n_features=20, n_informative=18, random_state=1) sample_weight = np.random.rand(len(y)) pipeline = Pipeline([('estimator', estimator)]) opt = WeightedBayesSearchCV( pipeline, scoring=make_scorer(mean_squared_error, greater_is_better=False), random_state=random_state, search_spaces=search_spaces, n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv ) opt.fit(X, y, sample_weight=sample_weight, sample_weight_steps=['estimator']) score = opt.score(X, y) return score
def _fit_class_weighted_cv_e(estimator, search_spaces, n_jobs=1, n_points=1, cv=None, random_state=13): """ Utility function to fit a larger classification task with the provided estimator, search space and randomly filled sample weight :return score accuracy """ X, y = make_classification(n_samples=1000, n_features=20, n_redundant=0, n_informative=18, random_state=1, n_clusters_per_class=1) sample_weight = np.random.rand(len(y)) pipeline = Pipeline([('estimator', estimator)]) opt = WeightedBayesSearchCV( pipeline, random_state=random_state, search_spaces=search_spaces, n_jobs=n_jobs, n_iter=11, n_points=n_points, cv=cv ) opt.fit(X, y, sample_weight=sample_weight, sample_weight_steps=['estimator']) score = opt.score(X, y) return score
def test_search_cv_internal_parameter_types(): # Test whether the parameters passed to the # estimator of the WeightedBayesSearchCV are of standard python # types - float, int, str # This is estimator is used to check whether the types provided # are native python types. class TypeCheckEstimator(BaseEstimator): def __init__(self, float_param=0.0, int_param=0, str_param=""): self.float_param = float_param self.int_param = int_param self.str_param = str_param def fit(self, X, y): assert isinstance(self.float_param, float) assert isinstance(self.int_param, int) assert isinstance(self.str_param, str) return self def score(self, X, y): return 0.0 # Below is example code that used to not work. X, y = make_classification(10, 4) model = WeightedBayesSearchCV( estimator=TypeCheckEstimator(), search_spaces={ 'float_param': [0.0, 1.0], 'int_param': [0, 10], 'str_param': ["one", "two", "three"], }, n_iter=11 ) model.fit(X, y)
test_size=.25, random_state=0) # log-uniform: understand as search over p = exp(x) by varying x opt = WeightedBayesSearchCV( SVC(), { 'C': (1e-6, 1e+6, 'log-uniform'), 'gamma': (1e-6, 1e+1, 'log-uniform'), 'degree': (1, 8), # integer valued parameter 'kernel': ['linear', 'poly', 'rbf'], # categorical parameter }, n_iter=32, cv=3) opt.fit(X_train, y_train) print("val. score: %s" % opt.best_score_) print("test score: %s" % opt.score(X_test, y_test)) ############################################################################# # Advanced example # ================ # # In practice, one wants to enumerate over multiple predictive model classes, # with different search spaces and number of evaluations per class. An # example of such search over parameters of Linear SVM, Kernel SVM, and # decision trees is given below. from skopt import WeightedBayesSearchCV from skopt.space import Real, Categorical, Integer
def test_searchcv_sklearn_compatibility(): """ Test whether the WeightedBayesSearchCV is compatible with base sklearn methods such as clone, set_params, get_params. """ X, y = load_iris(True) X_train, X_test, y_train, y_test = train_test_split( X, y, train_size=0.75, random_state=0 ) # used to try different model classes pipe = Pipeline([ ('model', SVC()) ]) # single categorical value of 'model' parameter sets the model class lin_search = { 'model': Categorical([LinearSVC()]), 'model__C': Real(1e-6, 1e+6, prior='log-uniform'), } dtc_search = { 'model': Categorical([DecisionTreeClassifier()]), 'model__max_depth': Integer(1, 32), 'model__min_samples_split': Real(1e-3, 1.0, prior='log-uniform'), } svc_search = { 'model': Categorical([SVC()]), 'model__C': Real(1e-6, 1e+6, prior='log-uniform'), 'model__gamma': Real(1e-6, 1e+1, prior='log-uniform'), 'model__degree': Integer(1, 8), 'model__kernel': Categorical(['linear', 'poly', 'rbf']), } opt = WeightedBayesSearchCV( pipe, [(lin_search, 1), svc_search], n_iter=2 ) opt_clone = clone(opt) params, params_clone = opt.get_params(), opt_clone.get_params() assert params.keys() == params_clone.keys() for param, param_clone in zip(params.items(), params_clone.items()): assert param[0] == param_clone[0] assert isinstance(param[1], type(param_clone[1])) opt.set_params(search_spaces=[(dtc_search, 1)]) opt.fit(X_train, y_train) opt_clone.fit(X_train, y_train) total_evaluations = len(opt.cv_results_['mean_test_score']) total_evaluations_clone = len(opt_clone.cv_results_['mean_test_score']) # test if expected number of subspaces is explored assert total_evaluations == 1 assert total_evaluations_clone == 1 + 2