def test_plateau(self):
try:
from ray.tune.stopper import TrialPlateauStopper
except ImportError:
self.skipTest("`TrialPlateauStopper` not available in "
"current Ray version.")
return
X, y = make_classification(n_samples=50,
n_features=50,
n_informative=3,
random_state=0)
clf = PlateauClassifier(converge_after=4)
stopper = TrialPlateauStopper(metric="objective")
search = TuneGridSearchCV(clf, {"foo_param": [2.0, 3.0, 4.0]},
cv=2,
max_iters=20,
stopper=stopper,
early_stopping=True)
search.fit(X, y)
print(search.cv_results_)
for iters in search.cv_results_["training_iteration"]:
# Converges after 4 iterations, but the stopper needs another
# 4 to detect it converged.
self.assertLessEqual(iters, 8)
def test_grid_search_groups(self):
# Check if ValueError (when groups is None) propagates to
# dcv.GridSearchCV
# And also check if groups is correctly passed to the cv object
rng = np.random.RandomState(0)
X, y = make_classification(n_samples=15, n_classes=2, random_state=0)
groups = rng.randint(0, 3, 15)
clf = LinearSVC(random_state=0)
grid = {"C": [1]}
group_cvs = [
LeaveOneGroupOut(),
LeavePGroupsOut(2),
GroupKFold(n_splits=3),
GroupShuffleSplit(n_splits=3),
]
for cv in group_cvs:
gs = TuneGridSearchCV(clf, grid, cv=cv)
with self.assertRaises(ValueError) as exc:
gs.fit(X, y)
self.assertTrue(
"parameter should not be None" in str(exc.exception))
gs.fit(X, y, groups=groups)
non_group_cvs = [
StratifiedKFold(n_splits=3),
StratifiedShuffleSplit(n_splits=3)
]
for cv in non_group_cvs:
gs = TuneGridSearchCV(clf, grid, cv=cv)
# Should not raise an error
gs.fit(X, y)
def test_no_refit(self):
# Test that GSCV can be used for model selection alone without
# refitting
clf = MockClassifier()
grid_search = TuneGridSearchCV(clf, {"foo_param": [1, 2, 3]},
refit=False,
cv=3)
grid_search.fit(X, y)
self.assertFalse(hasattr(grid_search, "best_estimator_"))
self.assertFalse(hasattr(grid_search, "best_index_"))
self.assertFalse(hasattr(grid_search, "best_score_"))
self.assertFalse(hasattr(grid_search, "best_params_"))
# Make sure the predict/transform etc fns raise meaningful error msg
for fn_name in (
"predict",
"predict_proba",
"predict_log_proba",
"transform",
"inverse_transform",
):
with self.assertRaises(NotFittedError) as exc:
getattr(grid_search, fn_name)(X)
self.assertTrue(
("refit=False. %s is available only after refitting on the "
"best parameters" % fn_name) in str(exc.exception))
def test_grid_search_bad_param_grid(self):
param_dict = {"C": 1.0}
clf = SVC()
with self.assertRaises(ValueError) as exc:
TuneGridSearchCV(clf, param_dict)
self.assertTrue(("Parameter grid for parameter (C) needs to"
" be a list or numpy array") in str(exc.exception))
param_dict = {"C": []}
clf = SVC()
with self.assertRaises(ValueError) as exc:
TuneGridSearchCV(clf, param_dict)
self.assertTrue(
("Parameter values for parameter (C) need to be a non-empty "
"sequence.") in str(exc.exception))
param_dict = {"C": "1,2,3"}
clf = SVC()
with self.assertRaises(ValueError) as exc:
TuneGridSearchCV(clf, param_dict)
self.assertTrue(("Parameter grid for parameter (C) needs to"
" be a list or numpy array") in str(exc.exception))
param_dict = {"C": np.ones(6).reshape(3, 2)}
clf = SVC()
with self.assertRaises(ValueError):
TuneGridSearchCV(clf, param_dict)
def test_digits(self):
# Loading the Digits dataset
digits = datasets.load_digits()
# To apply an classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.images)
X = digits.images.reshape((n_samples, -1))
y = digits.target
# Split the dataset in two equal parts
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.5,
random_state=0)
# Set the parameters by cross-validation
tuned_parameters = {
"kernel": ["rbf"],
"gamma": [1e-3, 1e-4],
"C": [1, 10, 100, 1000]
}
tune_search = TuneGridSearchCV(SVC(), tuned_parameters, max_iters=20)
tune_search.fit(X_train, y_train)
pred = tune_search.predict(X_test)
print(pred)
accuracy = np.count_nonzero(
np.array(pred) == np.array(y_test)) / len(pred)
print(accuracy)
def test_diabetes(self):
# load the diabetes datasets
dataset = datasets.load_diabetes()
X = dataset.data
y = dataset.target
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.5,
random_state=0)
# prepare a range of alpha values to test
alphas = np.array([1, 0.1, 0.01, 0.001, 0.0001, 0])
param_grid = dict(alpha=alphas)
# create and fit a ridge regression model, testing each alpha
model = linear_model.Ridge()
tune_search = TuneGridSearchCV(
model,
param_grid,
)
tune_search.fit(X_train, y_train)
pred = tune_search.predict(X_test)
print(pred)
error = sum(np.array(pred) - np.array(y_test)) / len(pred)
print(error)
def test_gridsearch_multi_cv_results(self):
parameter_grid = {"alpha": [1e-4, 1e-1, 1], "epsilon": [0.01, 0.1]}
scoring = ("accuracy", "f1_micro")
cv = 2
tune_search = TuneGridSearchCV(
SGDClassifier(),
parameter_grid,
scoring=scoring,
max_iters=20,
refit=False,
cv=cv)
tune_search.fit(X, y)
result = tune_search.cv_results_
keys_to_check = []
for s in scoring:
keys_to_check.append("mean_test_%s" % s)
for i in range(cv):
keys_to_check.append("split%d_test_%s" % (i, s))
for key in keys_to_check:
self.assertIn(key, result)
def test_grid_search_precomputed_kernel_error_nonsquare(self):
# Test that grid search returns an error with a non-square precomputed
# training kernel matrix
K_train = np.zeros((10, 20))
y_train = np.ones((10, ))
clf = SVC(kernel="precomputed")
cv = TuneGridSearchCV(clf, {"C": [0.1, 1.0]})
with self.assertRaises(TuneError):
cv.fit(K_train, y_train)
def test_y_as_list(self):
# Pass y as list in dcv.GridSearchCV
X = np.arange(100).reshape(10, 10)
y = np.array([0] * 5 + [1] * 5)
clf = CheckingClassifier(check_y=lambda x: isinstance(x, list))
cv = KFold(n_splits=3)
grid_search = TuneGridSearchCV(clf, {"foo_param": [1, 2, 3]}, cv=cv)
grid_search.fit(X, y.tolist()).score(X, y)
self.assertTrue(hasattr(grid_search, "cv_results_"))
def test_gridsearch_nd(self):
# Pass X as list in dcv.GridSearchCV
X_4d = np.arange(10 * 5 * 3 * 2).reshape(10, 5, 3, 2)
y_3d = np.arange(10 * 7 * 11).reshape(10, 7, 11)
clf = CheckingClassifier(
check_X=lambda x: x.shape[1:] == (5, 3, 2),
check_y=lambda x: x.shape[1:] == (7, 11),
)
grid_search = TuneGridSearchCV(clf, {"foo_param": [1, 2, 3]}, cv=3)
grid_search.fit(X_4d, y_3d).score(X, y)
self.assertTrue(hasattr(grid_search, "cv_results_"))
def test_grid_search_error(self):
# Test that grid search will capture errors on data with different
# length
X_, y_ = make_classification(n_samples=200,
n_features=100,
random_state=0)
clf = LinearSVC()
cv = TuneGridSearchCV(clf, {"C": [0.1, 1.0]})
with self.assertRaises(TuneError):
cv.fit(X_[:180], y_)
def test_refit(self):
# Regression test for bug in refitting
# Simulates re-fitting a broken estimator; this used to break with
# sparse SVMs.
X = np.arange(100).reshape(10, 10)
y = np.array([0] * 5 + [1] * 5)
clf = TuneGridSearchCV(BrokenClassifier(), {"parameter": [0, 1]},
scoring="accuracy",
refit=True)
clf.fit(X, y)
def test_grid_search_precomputed_kernel_error_nonsquare(self):
# Test that grid search returns an error with a non-square precomputed
# training kernel matrix
K_train = np.zeros((10, 20))
y_train = np.ones((10, ))
clf = SVC(kernel="precomputed")
cv = TuneGridSearchCV(clf, {"C": [0.1, 1.0]})
# with self.assertRaises(TuneError):
with self.assertRaises(ValueError) as exc:
cv.fit(K_train, y_train)
self.assertTrue(
("X should be a square kernel matrix") in str(exc.exception))
def test_grid_search_error(self):
# Test that grid search will capture errors on data with different
# length
X_, y_ = make_classification(
n_samples=200, n_features=100, random_state=0)
clf = LinearSVC()
cv = TuneGridSearchCV(clf, {"C": [0.1, 1.0]})
with self.assertRaises(ValueError) as exc:
cv.fit(X_[:180], y_)
self.assertTrue(("Found input variables with inconsistent numbers of "
"samples: [180, 200]") in str(exc.exception))
def test_trivial_cv_results_attr(self):
# Test search over a "grid" with only one point.
# Non-regression test: grid_scores_ wouldn't be set by
# dcv.GridSearchCV.
clf = MockClassifier()
grid_search = TuneGridSearchCV(clf, {"foo_param": [1]}, cv=3)
grid_search.fit(X, y)
self.assertTrue(hasattr(grid_search, "cv_results_"))
random_search = TuneSearchCV(clf, {"foo_param": [0]}, n_iter=1, cv=3)
random_search.fit(X, y)
self.assertTrue(hasattr(random_search, "cv_results_"))
def test_grid_search_one_grid_point(self):
X_, y_ = make_classification(
n_samples=200, n_features=100, random_state=0)
param_dict = {"C": [1.0], "kernel": ["rbf"], "gamma": [0.1]}
clf = SVC()
cv = TuneGridSearchCV(clf, param_dict)
cv.fit(X_, y_)
clf = SVC(C=1.0, kernel="rbf", gamma=0.1)
clf.fit(X_, y_)
assert_array_equal(clf.dual_coef_, cv.best_estimator_.dual_coef_)
def test_grid_search_error(self):
# Test that grid search will capture errors on data with different
# length
X_, y_ = make_classification(n_samples=200,
n_features=100,
random_state=0)
clf = LinearSVC()
cv = TuneGridSearchCV(clf, {"C": [0.1, 1.0]})
with self.assertLogs("ray.tune") as cm:
cv.fit(X_[:180], y_)
self.assertTrue(("ValueError: Found input variables with inconsistent "
"numbers of samples: [180, 200]") in str(cm.output))
def test_local_mode(self):
# Pass X as list in dcv.GridSearchCV
X = np.arange(100).reshape(10, 10)
y = np.array([0] * 5 + [1] * 5)
clf = CheckingClassifier(check_X=lambda x: isinstance(x, list))
cv = KFold(n_splits=3)
with patch.object(ray, "init", wraps=ray.init) as wrapped_init:
grid_search = TuneGridSearchCV(
clf, {"foo_param": [1, 2, 3]}, n_jobs=1, cv=cv)
grid_search.fit(X.tolist(), y).score(X, y)
self.assertTrue(hasattr(grid_search, "cv_results_"))
self.assertTrue(wrapped_init.call_args[1]["local_mode"])
def test_gridsearch_no_predict(self):
# test grid-search with an estimator without predict.
# slight duplication of a test from KDE
def custom_scoring(estimator, X):
return 42 if estimator.bandwidth == 0.1 else 0
X, _ = make_blobs(
cluster_std=0.1, random_state=1, centers=[[0, 1], [1, 0], [0, 0]])
search = TuneGridSearchCV(
KernelDensity(),
param_grid=dict(bandwidth=[0.01, 0.1, 1]),
scoring=custom_scoring,
)
search.fit(X)
self.assertEqual(search.best_params_["bandwidth"], 0.1)
self.assertEqual(search.best_score_, 42)
def test_tune_search_spaces(self):
# Test mixed search spaces
clf = MockClassifier()
foo = [1, 2, 3]
bar = [1, 2]
grid_search = TuneGridSearchCV(clf, {
"foo_param": tune.grid_search(foo),
"bar_param": bar
},
refit=False,
cv=3)
grid_search.fit(X, y)
params = grid_search.cv_results_["params"]
results_grid = {k: {dic[k] for dic in params} for k in params[0]}
self.assertTrue(len(results_grid["foo_param"]) == len(foo))
self.assertTrue(len(results_grid["bar_param"]) == len(bar))
def sweep(self, X, y):
optimizers = ["rmsprop", "adam"]
kernel_initializer = ["glorot_uniform", "normal"]
epochs = [5, 10]
param_grid = dict(optimizer=optimizers,
nb_epoch=epochs,
kernel_initializer=kernel_initializer)
grid = TuneGridSearchCV(
estimator=self.model,
param_grid=param_grid,
scoring="neg_mean_squared_error",
)
grid_result = grid.fit(X, y)
return grid_result
def test_gridsearch_no_multi_cv_results(self):
parameter_grid = {"alpha": [1e-4, 1e-1, 1], "epsilon": [0.01, 0.1]}
cv = 2
tune_search = TuneGridSearchCV(
SGDClassifier(), parameter_grid, max_iters=20, refit=False, cv=cv)
tune_search.fit(X, y)
result = tune_search.cv_results_
keys_to_check = ["mean_test_score"]
for i in range(cv):
keys_to_check.append("split%d_test_score" % i)
for key in keys_to_check:
self.assertIn(key, result)
def test_grid_search_precomputed_kernel(self):
# Test that grid search works when the input features are given in the
# form of a precomputed kernel matrix
X_, y_ = make_classification(n_samples=200,
n_features=100,
random_state=0)
# compute the training kernel matrix corresponding to the linear kernel
K_train = np.dot(X_[:180], X_[:180].T)
y_train = y_[:180]
clf = SVC(kernel="precomputed")
cv = TuneGridSearchCV(clf, {"C": [0.1, 1.0]})
cv.fit(K_train, y_train)
self.assertTrue(cv.best_score_ >= 0)
# compute the test kernel matrix
K_test = np.dot(X_[180:], X_[:180].T)
y_test = y_[180:]
y_pred = cv.predict(K_test)
self.assertTrue(np.mean(y_pred == y_test) >= 0)
# test error is raised when the precomputed kernel is not array-like
# or sparse
with self.assertRaises(TuneError):
cv.fit(K_train.tolist(), y_train)
def test_timeout(self):
clf = SleepClassifier()
# SleepClassifier sleeps for `foo_param` seconds, `cv` times.
# Thus, the time budget is exhausted after testing the first two
# `foo_param`s.
grid_search = TuneGridSearchCV(clf, {"foo_param": [1.1, 1.2, 2.5]},
time_budget_s=5.0,
cv=2,
max_iters=5,
early_stopping=True)
start = time.time()
grid_search.fit(X, y)
taken = time.time() - start
print(grid_search)
# Without timeout we would need over 50 seconds for this to
# finish. Allow for some initialization overhead
self.assertLess(taken, 18.0)
def test_grid_search_sparse(self):
# Test that grid search works with both dense and sparse matrices
X_, y_ = make_classification(
n_samples=200, n_features=100, random_state=0)
clf = LinearSVC()
cv = TuneGridSearchCV(clf, {"C": [0.1, 1.0]})
cv.fit(X_[:180], y_[:180])
y_pred = cv.predict(X_[180:])
C = cv.best_estimator_.C
X_ = sp.csr_matrix(X_)
clf = LinearSVC()
cv = TuneGridSearchCV(clf, {"C": [0.1, 1.0]})
cv.fit(X_[:180].tocoo(), y_[:180])
y_pred2 = cv.predict(X_[180:])
C2 = cv.best_estimator_.C
self.assertTrue(np.mean(y_pred == y_pred2) >= 0.9)
self.assertEqual(C, C2)
def test_max_iters(self):
X, y = make_classification(n_samples=50,
n_features=50,
n_informative=3,
random_state=0)
clf = PlateauClassifier(converge_after=20)
search = TuneGridSearchCV(clf, {"foo_param": [2.0, 3.0, 4.0]},
cv=2,
max_iters=6,
early_stopping=True)
search.fit(X, y)
print(search.cv_results_)
for iters in search.cv_results_["training_iteration"]:
# Stop after 6 iterations.
self.assertLessEqual(iters, 6)
def test_pandas_input(self):
# check cross_val_score doesn't destroy pandas dataframe
types = [(MockDataFrame, MockDataFrame)]
try:
from pandas import Series, DataFrame
types.append((DataFrame, Series))
except ImportError:
pass
X = np.arange(100).reshape(10, 10)
y = np.array([0] * 5 + [1] * 5)
for InputFeatureType, TargetType in types:
# X dataframe, y series
X_df, y_ser = InputFeatureType(X), TargetType(y)
clf = CheckingClassifier(
check_X=lambda x: isinstance(x, InputFeatureType),
check_y=lambda x: isinstance(x, TargetType),
)
grid_search = TuneGridSearchCV(clf, {"foo_param": [1, 2, 3]})
grid_search.fit(X_df, y_ser).score(X_df, y_ser)
grid_search.predict(X_df)
self.assertTrue(hasattr(grid_search, "cv_results_"))
def test_unsupervised_grid_search(self):
# test grid-search with unsupervised estimator
X, y = make_blobs(random_state=0)
km = KMeans(random_state=0)
grid_search = TuneGridSearchCV(km,
param_grid=dict(n_clusters=[2, 3, 4]),
scoring="adjusted_rand_score")
grid_search.fit(X, y)
# ARI can find the right number :)
self.assertEqual(grid_search.best_params_["n_clusters"], 3)
# Now without a score, and without y
grid_search = TuneGridSearchCV(km,
param_grid=dict(n_clusters=[2, 3, 4]))
grid_search.fit(X)
self.assertEqual(grid_search.best_params_["n_clusters"], 4)
self.dense0 = nn.Linear(20, num_units)
self.nonlin = nonlin
self.dropout = nn.Dropout(0.5)
self.dense1 = nn.Linear(num_units, 10)
self.output = nn.Linear(10, 2)
def forward(self, X, **kwargs):
X = self.nonlin(self.dense0(X))
X = self.dropout(X)
X = F.relu(self.dense1(X))
X = F.softmax(self.output(X))
return X
net = NeuralNetClassifier(
MyModule,
max_epochs=10,
lr=0.1,
# Shuffle training data on each epoch
iterator_train__shuffle=True,
)
params = {
"lr": [0.01, 0.02],
"module__num_units": [10, 20],
}
gs = TuneGridSearchCV(net, params, scoring="accuracy")
gs.fit(X, y)
print(gs.best_score_, gs.best_params_)
Y_test = np_utils.to_categorical(y_test, nb_classes)
def create_model(optimizer="rmsprop", kernel_initializer="glorot_uniform"):
model = Sequential()
model.add(Dense(512, input_shape=(784, )))
model.add(Activation("relu"))
model.add(Dropout(0.2))
model.add(Dense(512, kernel_initializer=kernel_initializer))
model.add(Activation("relu"))
model.add(Dropout(0.2))
model.add(Dense(10, kernel_initializer=kernel_initializer))
model.add(Activation("softmax")) # This special "softmax" a
model.compile(loss="binary_crossentropy",
optimizer=optimizer,
metrics=["accuracy"])
return model
model = KerasClassifier(build_fn=create_model)
optimizers = ["rmsprop", "adam"]
kernel_initializer = ["glorot_uniform", "normal"]
epochs = [5, 10]
param_grid = dict(optimizer=optimizers,
nb_epoch=epochs,
kernel_initializer=kernel_initializer)
grid = TuneGridSearchCV(estimator=model, param_grid=param_grid)
grid_result = grid.fit(X_train, Y_train)
print(grid_result.best_params_)
print(grid_result.cv_results_)
