class TrainableTest(unittest.TestCase):
X_id = None
y_id = None
X = None
y = None
@classmethod
def setUpClass(cls):
ray.init(local_mode=True)
X, y = make_classification(
n_samples=50, n_features=50, n_informative=3, random_state=0)
cls.X_id, cls.y_id = ray.put(X), ray.put(y)
cls.y = y
cls.X = X
@classmethod
def tearDownClass(cls):
ray.shutdown()
def base_params(self, estimator_list):
config = {"estimator_list": estimator_list}
cv = check_cv(
cv=len(estimator_list), y=self.y, classifier=estimator_list[0])
config["X_id"] = self.X_id
config["y_id"] = self.y_id
config["early_stopping"] = False
config["early_stop_type"] = get_early_stop_type(
estimator_list[0], False)
config["max_iters"] = 1
config["groups"] = None
config["cv"] = cv
config["fit_params"] = None
config["scoring"], _ = _check_multimetric_scoring(
estimator_list[0], scoring=None)
config["return_train_score"] = False
config["n_jobs"] = 1
return config
def test_basic_train(self):
config = self.base_params(estimator_list=[SVC(), SVC()])
trainable = _Trainable(config)
trainable.train()
trainable.stop()
@unittest.skipIf(not has_xgboost(), "xgboost not installed")
def testXGBoostEarlyStop(self):
config = self.base_params(
estimator_list=[create_xgboost(),
create_xgboost()])
config["early_stopping"] = True
config["early_stop_type"] = get_early_stop_type(
config["estimator_list"][0], True)
trainable = _Trainable(config)
trainable.train()
assert all(trainable.saved_models)
trainable.train()
assert all(trainable.saved_models)
trainable.stop()
@unittest.skipIf(not has_xgboost(), "xgboost not installed")
def testXGBoostNoEarlyStop(self):
config = self.base_params(
estimator_list=[create_xgboost(),
create_xgboost()])
config["early_stopping"] = False
trainable = _Trainable(config)
trainable.train()
assert not any(trainable.saved_models)
trainable.stop()
@unittest.skipIf(not has_required_lightgbm_version(),
"lightgbm>=3.0.0 not installed")
def testLGBMEarlyStop(self):
config = self.base_params(
estimator_list=[create_lightgbm(),
create_lightgbm()])
config["early_stopping"] = True
config["early_stop_type"] = get_early_stop_type(
config["estimator_list"][0], True)
trainable = _Trainable(config)
trainable.train()
assert all(trainable.saved_models)
trainable.train()
assert all(trainable.saved_models)
trainable.stop()
@unittest.skipIf(not has_required_lightgbm_version(),
"lightgbm>=3.0.0 not installed")
def testLGBMNoEarlyStop(self):
config = self.base_params(
estimator_list=[create_lightgbm(),
create_lightgbm()])
config["early_stopping"] = False
trainable = _Trainable(config)
trainable.train()
assert not any(trainable.saved_models)
trainable.stop()
# @unittest.skipIf(not has_catboost(), "catboost not installed")
@unittest.skip("Catboost needs to be updated.")
def testCatboostEarlyStop(self):
config = self.base_params(
estimator_list=[create_catboost(),
create_catboost()])
config["early_stopping"] = True
config["early_stop_type"] = get_early_stop_type(
config["estimator_list"][0], True)
trainable = _Trainable(config)
trainable.train()
assert all(trainable.saved_models)
trainable.train()
assert all(trainable.saved_models)
trainable.stop()
# @unittest.skipIf(not has_catboost(), "catboost not installed")
@unittest.skip("Catboost needs to be updated.")
def testCatboostNoEarlyStop(self):
config = self.base_params(
estimator_list=[create_catboost(),
create_catboost()])
config["early_stopping"] = False
trainable = _Trainable(config)
trainable.train()
assert not any(trainable.saved_models)
trainable.stop()
def testPartialFit(self):
config = self.base_params([SGDClassifier(), SGDClassifier()])
config["early_stopping"] = True
config["early_stop_type"] = get_early_stop_type(
config["estimator_list"][0], True)
trainable = _Trainable(config)
trainable.train()
assert trainable.estimator_list[0].t_ > 0
previous_t = trainable.estimator_list[0].t_
trainable.train()
assert trainable.estimator_list[0].t_ > previous_t
trainable.stop()
def testNoPartialFit(self):
config = self.base_params([SGDClassifier(), SGDClassifier()])
config["early_stopping"] = False
trainable = _Trainable(config)
trainable.train()
assert not hasattr(trainable.estimator_list[0], "t_")
trainable.train()
assert not hasattr(trainable.estimator_list[0], "t_")
trainable.stop()
def testWarmStart(self):
# Hard to get introspection so we just test that it runs.
config = self.base_params([LogisticRegression(), LogisticRegression()])
config["early_stopping"] = True
config["early_stop_type"] = get_early_stop_type(
config["estimator_list"][0], True)
trainable = _Trainable(config)
trainable.train()
trainable.train()
trainable.stop()
class RandomizedSearchTest(unittest.TestCase):
def test_random_search_cv_results(self):
# Make a dataset with a lot of noise to get various kind of prediction
# errors across CV folds and parameter settings
X, y = make_classification(
n_samples=50, n_features=50, n_informative=3, random_state=0)
# scipy.stats dists now supports `seed` but we still support scipy 0.12
# which doesn't support the seed. Hence the assertions in the test for
# random_search alone should not depend on randomization.
n_splits = 3
n_search_iter = 30
params = dict(C=expon(scale=10), gamma=expon(scale=0.1))
random_search = TuneSearchCV(
SVC(),
n_trials=n_search_iter,
cv=n_splits,
param_distributions=params,
return_train_score=True,
n_jobs=2)
random_search.fit(X, y)
param_keys = ("param_C", "param_gamma")
score_keys = (
"mean_test_score",
"mean_train_score",
"rank_test_score",
"rank_train_score",
"split0_test_score",
"split1_test_score",
"split2_test_score",
"split0_train_score",
"split1_train_score",
"split2_train_score",
"std_test_score",
"std_train_score",
"time_total_s",
)
n_cand = n_search_iter
def test_check_cv_results_array_types(cv_results, param_keys,
score_keys):
# Check if the search `cv_results`'s array are of correct types
self.assertTrue(
all(
isinstance(cv_results[param], np.ma.MaskedArray)
for param in param_keys))
self.assertTrue(
all(cv_results[key].dtype == object for key in param_keys))
self.assertFalse(
any(
isinstance(cv_results[key], np.ma.MaskedArray)
for key in score_keys))
self.assertTrue(
all(cv_results[key].dtype == np.float64 for key in score_keys
if not key.startswith("rank")))
self.assertEquals(cv_results["rank_test_score"].dtype, np.int32)
def test_check_cv_results_keys(cv_results, param_keys, score_keys,
n_cand):
# Test the search.cv_results_ contains all the required results
assert_array_equal(
sorted(cv_results.keys()),
sorted(param_keys + score_keys + ("params", )))
self.assertTrue(
all(cv_results[key].shape == (n_cand, )
for key in param_keys + score_keys))
cv_results = random_search.cv_results_
# Check results structure
test_check_cv_results_array_types(cv_results, param_keys, score_keys)
test_check_cv_results_keys(cv_results, param_keys, score_keys, n_cand)
# For random_search, all the param array vals should be unmasked
self.assertFalse(
any(cv_results["param_C"].mask)
or any(cv_results["param_gamma"].mask))
def test_local_dir(self):
digits = datasets.load_digits()
x = digits.data
y = digits.target
clf = SGDClassifier()
parameter_grid = {
"alpha": Real(1e-4, 1e-1, 1),
"epsilon": Real(0.01, 0.1)
}
scheduler = MedianStoppingRule(grace_period=10.0)
tune_search = TuneSearchCV(
clf,
parameter_grid,
early_stopping=scheduler,
max_iters=10,
local_dir="./test-result")
tune_search.fit(x, y)
self.assertTrue(len(os.listdir("./test-result")) != 0)
def test_local_mode(self):
digits = datasets.load_digits()
x = digits.data
y = digits.target
clf = SGDClassifier()
parameter_grid = {
"alpha": Real(1e-4, 1e-1, 1),
"epsilon": Real(0.01, 0.1)
}
tune_search = TuneSearchCV(
clf,
parameter_grid,
n_jobs=1,
max_iters=10,
local_dir="./test-result")
import ray
with patch.object(ray, "init", wraps=ray.init) as wrapped_init:
tune_search.fit(x, y)
self.assertTrue(wrapped_init.call_args[1]["local_mode"])
def test_multi_best_classification(self):
digits = datasets.load_digits()
x = digits.data
y = digits.target
model = SGDClassifier()
parameter_grid = {"alpha": [1e-4, 1e-1, 1], "epsilon": [0.01, 0.1]}
scoring = ("accuracy", "f1_micro")
search_methods = ["random", "bayesian", "hyperopt", "bohb"]
for search_method in search_methods:
tune_search = TuneSearchCV(
model,
parameter_grid,
scoring=scoring,
search_optimization=search_method,
cv=2,
n_trials=3,
n_jobs=1,
refit="accuracy")
tune_search.fit(x, y)
self.assertAlmostEqual(
tune_search.best_score_,
max(tune_search.cv_results_["mean_test_accuracy"]),
places=10)
p = tune_search.cv_results_["params"]
scores = tune_search.cv_results_["mean_test_accuracy"]
cv_best_param = max(
list(zip(scores, p)), key=lambda pair: pair[0])[1]
self.assertEqual(tune_search.best_params_, cv_best_param)
def test_multi_best_classification_scoring_dict(self):
digits = datasets.load_digits()
x = digits.data
y = digits.target
model = SGDClassifier()
parameter_grid = {"alpha": [1e-4, 1e-1, 1], "epsilon": [0.01, 0.1]}
scoring = {"acc": "accuracy", "f1": "f1_micro"}
search_methods = ["random", "bayesian", "hyperopt", "bohb"]
for search_method in search_methods:
tune_search = TuneSearchCV(
model,
parameter_grid,
scoring=scoring,
search_optimization=search_method,
cv=2,
n_trials=3,
n_jobs=1,
refit="acc")
tune_search.fit(x, y)
self.assertAlmostEqual(
tune_search.best_score_,
max(tune_search.cv_results_["mean_test_acc"]),
places=10)
p = tune_search.cv_results_["params"]
scores = tune_search.cv_results_["mean_test_acc"]
cv_best_param = max(
list(zip(scores, p)), key=lambda pair: pair[0])[1]
self.assertEqual(tune_search.best_params_, cv_best_param)
def test_multi_best_regression(self):
x, y = make_regression(n_samples=100, n_features=10, n_informative=5)
model = SGDRegressor()
parameter_grid = {"alpha": [1e-4, 1e-1, 1], "epsilon": [0.01, 0.1]}
scoring = ("neg_mean_absolute_error", "neg_mean_squared_error")
search_methods = ["random", "bayesian", "hyperopt", "bohb"]
for search_method in search_methods:
tune_search = TuneSearchCV(
model,
parameter_grid,
scoring=scoring,
search_optimization=search_method,
cv=2,
n_trials=3,
n_jobs=1,
refit="neg_mean_absolute_error")
tune_search.fit(x, y)
self.assertAlmostEqual(
tune_search.best_score_,
max(tune_search.cv_results_[
"mean_test_neg_mean_absolute_error"]),
places=10)
p = tune_search.cv_results_["params"]
scores = tune_search.cv_results_[
"mean_test_neg_mean_absolute_error"]
cv_best_param = max(
list(zip(scores, p)), key=lambda pair: pair[0])[1]
self.assertEqual(tune_search.best_params_, cv_best_param)
def test_multi_refit_false(self):
digits = datasets.load_digits()
x = digits.data
y = digits.target
model = SGDClassifier()
parameter_grid = {"alpha": [1e-4, 1e-1, 1], "epsilon": [0.01, 0.1]}
scoring = ("accuracy", "f1_micro")
tune_search = TuneSearchCV(
model,
parameter_grid,
scoring=scoring,
search_optimization="random",
cv=2,
n_trials=3,
n_jobs=1,
refit=False)
tune_search.fit(x, y)
with self.assertRaises(ValueError) as exc:
tune_search.best_score_
self.assertTrue(("instance was initialized with refit=False. "
"For multi-metric evaluation,") in str(exc.exception))
with self.assertRaises(ValueError) as exc:
tune_search.best_index_
self.assertTrue(("instance was initialized with refit=False. "
"For multi-metric evaluation,") in str(exc.exception))
with self.assertRaises(ValueError) as exc:
tune_search.best_params_
self.assertTrue(("instance was initialized with refit=False. "
"For multi-metric evaluation,") in str(exc.exception))
def test_warm_start_detection(self):
parameter_grid = {"alpha": Real(1e-4, 1e-1, 1)}
from sklearn.ensemble import VotingClassifier, RandomForestClassifier
clf = VotingClassifier(estimators=[(
"rf", RandomForestClassifier(n_estimators=50, random_state=0))])
tune_search = TuneSearchCV(
clf,
parameter_grid,
n_jobs=1,
max_iters=10,
local_dir="./test-result")
self.assertEqual(tune_search.early_stop_type,
EarlyStopping.NO_EARLY_STOP)
from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier(random_state=0)
tune_search2 = TuneSearchCV(
clf,
parameter_grid,
n_jobs=1,
max_iters=10,
local_dir="./test-result")
self.assertEqual(tune_search2.early_stop_type,
EarlyStopping.NO_EARLY_STOP)
from sklearn.linear_model import LogisticRegression
clf = LogisticRegression()
tune_search3 = TuneSearchCV(
clf,
parameter_grid,
n_jobs=1,
max_iters=10,
local_dir="./test-result")
self.assertEqual(tune_search3.early_stop_type,
EarlyStopping.NO_EARLY_STOP)
tune_search4 = TuneSearchCV(
clf,
parameter_grid,
early_stopping=True,
n_jobs=1,
max_iters=10,
local_dir="./test-result")
self.assertEqual(tune_search4.early_stop_type,
EarlyStopping.WARM_START_ITER)
clf = RandomForestClassifier()
tune_search5 = TuneSearchCV(
clf,
parameter_grid,
early_stopping=True,
n_jobs=1,
max_iters=10,
local_dir="./test-result")
self.assertEqual(tune_search5.early_stop_type,
EarlyStopping.WARM_START_ENSEMBLE)
def test_warm_start_error(self):
parameter_grid = {"alpha": Real(1e-4, 1e-1, 1)}
from sklearn.ensemble import VotingClassifier, RandomForestClassifier
clf = VotingClassifier(estimators=[(
"rf", RandomForestClassifier(n_estimators=50, random_state=0))])
tune_search = TuneSearchCV(
clf,
parameter_grid,
n_jobs=1,
early_stopping=False,
max_iters=10,
local_dir="./test-result")
self.assertFalse(tune_search._can_early_stop())
with self.assertRaises(ValueError):
tune_search = TuneSearchCV(
clf,
parameter_grid,
n_jobs=1,
early_stopping=True,
max_iters=10,
local_dir="./test-result")
from sklearn.linear_model import LogisticRegression
clf = LogisticRegression()
with self.assertRaises(ValueError):
parameter_grid = {"max_iter": [1, 2]}
TuneSearchCV(
clf,
parameter_grid,
early_stopping=True,
n_jobs=1,
max_iters=10,
local_dir="./test-result")
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier()
with self.assertRaises(ValueError):
parameter_grid = {"n_estimators": [1, 2]}
TuneSearchCV(
clf,
parameter_grid,
early_stopping=True,
n_jobs=1,
max_iters=10,
local_dir="./test-result")
def test_warn_reduce_maxiters(self):
parameter_grid = {"alpha": Real(1e-4, 1e-1, 1)}
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(max_depth=2, random_state=0)
with self.assertWarnsRegex(UserWarning, "max_iters is set"):
TuneSearchCV(
clf, parameter_grid, max_iters=10, local_dir="./test-result")
with self.assertWarnsRegex(UserWarning, "max_iters is set"):
TuneSearchCV(
SGDClassifier(),
parameter_grid,
max_iters=10,
local_dir="./test-result")
def test_warn_early_stop(self):
with self.assertWarnsRegex(UserWarning, "max_iters = 1"):
TuneSearchCV(
LogisticRegression(), {"C": [1, 2]}, early_stopping=True)
with self.assertWarnsRegex(UserWarning, "max_iters = 1"):
TuneSearchCV(
SGDClassifier(), {"epsilon": [0.1, 0.2]}, early_stopping=True)
@unittest.skipIf(not has_xgboost(), "xgboost not installed")
def test_early_stop_xgboost_warn(self):
from xgboost.sklearn import XGBClassifier
with self.assertWarnsRegex(UserWarning, "github.com"):
TuneSearchCV(
XGBClassifier(), {"C": [1, 2]},
early_stopping=True,
max_iters=10)
with self.assertWarnsRegex(UserWarning, "max_iters"):
TuneSearchCV(
XGBClassifier(), {"C": [1, 2]},
early_stopping=True,
max_iters=1)
@unittest.skipIf(not has_required_lightgbm_version(),
"lightgbm not installed")
def test_early_stop_lightgbm_warn(self):
from lightgbm import LGBMClassifier
with self.assertWarnsRegex(UserWarning, "lightgbm"):
TuneSearchCV(
LGBMClassifier(), {"learning_rate": [0.1, 0.5]},
early_stopping=True,
max_iters=10)
with self.assertWarnsRegex(UserWarning, "max_iters"):
TuneSearchCV(
LGBMClassifier(), {"learning_rate": [0.1, 0.5]},
early_stopping=True,
max_iters=1)
@unittest.skipIf(not has_catboost(), "catboost not installed")
def test_early_stop_catboost_warn(self):
from catboost import CatBoostClassifier
with self.assertWarnsRegex(UserWarning, "Catboost"):
TuneSearchCV(
CatBoostClassifier(), {"learning_rate": [0.1, 0.5]},
early_stopping=True,
max_iters=10)
with self.assertWarnsRegex(UserWarning, "max_iters"):
TuneSearchCV(
CatBoostClassifier(), {"learning_rate": [0.1, 0.5]},
early_stopping=True,
max_iters=1)
def test_pipeline_early_stop(self):
digits = datasets.load_digits()
x = digits.data
y = digits.target
pipe = Pipeline([("reduce_dim", PCA()), ("classify", SGDClassifier())])
parameter_grid = [
{
"classify__alpha": [1e-4, 1e-1, 1],
"classify__epsilon": [0.01, 0.1]
},
]
with self.assertRaises(ValueError) as exc:
TuneSearchCV(
pipe,
parameter_grid,
early_stopping=True,
pipeline_auto_early_stop=False,
max_iters=10)
self.assertTrue((
"Early stopping is not supported because the estimator does "
"not have `partial_fit`, does not support warm_start, or "
"is a tree classifier. Set `early_stopping=False`."
) in str(exc.exception))
tune_search = TuneSearchCV(
pipe, parameter_grid, early_stopping=True, max_iters=10)
tune_search.fit(x, y)