def test_grid_search(self, grid_search):
model, dataset_loader_train = Mock(), Mock()
dataset_loader_train.return_value = (np.random.random(
(10, 10)), np.random.random(10))
CVIterator = Mock()
def cv_iterator(n, p):
return CVIterator(n=n, p=p)
grid_search_params = {'cv': partial(cv_iterator, p=2)}
scores = {
'mean_test_score': [0.1, 0.2],
'std_test_score': [0.06463643, 0.05073433],
'params': [{
'C': 0.1
}, {
'C': 0.3
}],
}
with patch('palladium.fit.GridSearchCV') as GridSearchCV:
GridSearchCV().cv_results_ = scores
grid_search(dataset_loader_train, model, grid_search_params)
GridSearchCV.assert_called_with(model,
refit=False,
cv=CVIterator.return_value)
CVIterator.assert_called_with(n=10, p=2)
def test_two_scores_raises(self, grid_search):
model, dataset_loader_train = Mock(spec=['fit', 'predict']), Mock()
dataset_loader_train.return_value = object(), object()
with pytest.raises(ValueError):
grid_search(dataset_loader_train,
model, {'scoring': 'f1'},
scoring='accuracy')
def test_two_scores_priority(self, grid_search, GridSearchCVWithScores):
# 'scoring' has higher priority than 'model.score'
model = Mock(spec=['fit', 'predict', 'score'])
dataset_loader_train = Mock()
scoring = Mock()
dataset_loader_train.return_value = object(), object()
grid_search(dataset_loader_train, model, {}, scoring=scoring)
GridSearchCVWithScores.assert_called_with(model,
refit=False,
scoring=scoring)
def test_persist_best_requires_persister(self, grid_search):
model = Mock(spec=['fit', 'predict'])
del model.cv_results_
dataset_loader_train = Mock()
scoring = Mock()
dataset_loader_train.return_value = object(), object()
with pytest.raises(ValueError):
grid_search(dataset_loader_train,
model, {},
scoring=scoring,
persist_best=True)
def test_deprecated_scoring(self, grid_search, GridSearchCVWithScores):
# 'scoring' inside of 'grid_search' is deprecated
model = Mock(spec=['fit', 'predict', 'score'])
dataset_loader_train = Mock()
scoring = Mock()
dataset_loader_train.return_value = object(), object()
with pytest.warns(DeprecationWarning):
grid_search(dataset_loader_train,
model, {'scoring': scoring},
scoring=None)
GridSearchCVWithScores.assert_called_with(model,
refit=False,
scoring=scoring)
def test_it(self, grid_search):
model, dataset_loader_train = Mock(), Mock()
grid_search_params = {'verbose': 4}
X, y = object(), object()
dataset_loader_train.return_value = X, y
scores = {
'mean_test_score': [0.1, 0.2],
'std_test_score': [0.06463643, 0.05073433],
'params': [{
'C': 0.1
}, {
'C': 0.3
}]
}
with patch('palladium.fit.GridSearchCV') as GridSearchCV:
GridSearchCV().cv_results_ = scores
result = grid_search(dataset_loader_train, model,
grid_search_params)
expected = []
expected.append("mean: {0:.5f}, std: {1:.5f}, params: {2}".format(
scores['mean_test_score'][0], scores['std_test_score'][0],
scores['params'][0]))
expected.append("mean: {0:.5f}, std: {1:.5f}, params: {2}".format(
scores['mean_test_score'][1], scores['std_test_score'][1],
scores['params'][1]))
assert result == expected
dataset_loader_train.assert_called_with()
GridSearchCV.assert_called_with(model, refit=False, verbose=4)
GridSearchCV().fit.assert_called_with(X, y)
def test_grid_search_with_instance(self, grid_search):
scores = {
'mean_test_score': [0.1, 0.2],
'std_test_score': [0.06463643, 0.05073433],
'params': [{
'C': 0.1
}, {
'C': 0.3
}],
}
model, dataset_loader_train = Mock(), Mock()
X, y = np.random.random((10, 10)), np.random.random(10)
dataset_loader_train.return_value = X, y
search = Mock()
search.cv_results_ = scores
grid_search(dataset_loader_train, model, grid_search=search)
search.fit.assert_called_with(X, y)
def test_persist_best(self, grid_search, GridSearchCVWithScores):
model = Mock(spec=['fit', 'predict'])
del model.cv_results_
dataset_loader_train = Mock()
scoring = Mock()
model_persister = Mock()
dataset_loader_train.return_value = object(), object()
grid_search(dataset_loader_train,
model, {},
scoring=scoring,
persist_best=True,
model_persister=model_persister)
GridSearchCVWithScores.assert_called_with(model,
refit=True,
scoring=scoring)
model_persister.write.assert_called_with(GridSearchCVWithScores())
def test_grid_search(self, grid_search):
model, dataset_loader_train = Mock(), Mock()
dataset_loader_train.return_value = (
np.random.random((10, 10)), np.random.random(10))
CVIterator = Mock()
def cv_iterator(n, p):
return CVIterator(n=n, p=p)
grid_search_params = {'cv': partial(cv_iterator, p=2)}
scores = [
Mock(mean_validation_score=0.1),
Mock(mean_validation_score=0.2),
]
with patch('palladium.fit.GridSearchCV') as GridSearchCV:
GridSearchCV().grid_scores_ = scores
grid_search(dataset_loader_train, model, grid_search_params)
GridSearchCV.assert_called_with(model, refit=False,
cv=CVIterator.return_value)
CVIterator.assert_called_with(n=10, p=2)
def test_grid_search(self, grid_search):
model, dataset_loader_train = Mock(), Mock()
dataset_loader_train.return_value = (np.random.random(
(10, 10)), np.random.random(10))
CVIterator = Mock()
def cv_iterator(n, p):
return CVIterator(n=n, p=p)
grid_search_params = {'cv': partial(cv_iterator, p=2)}
scores = [
Mock(mean_validation_score=0.1),
Mock(mean_validation_score=0.2),
]
with patch('palladium.fit.GridSearchCV') as GridSearchCV:
GridSearchCV().grid_scores_ = scores
grid_search(dataset_loader_train, model, grid_search_params)
GridSearchCV.assert_called_with(model,
refit=False,
cv=CVIterator.return_value)
CVIterator.assert_called_with(n=10, p=2)
def test_it(self, grid_search):
model, dataset_loader_train = Mock(), Mock()
grid_search_params = {'verbose': 4}
X, y = object(), object()
dataset_loader_train.return_value = X, y
scores = [
Mock(mean_validation_score=0.1),
Mock(mean_validation_score=0.2),
]
with patch('palladium.fit.GridSearchCV') as GridSearchCV:
GridSearchCV().grid_scores_ = scores
result = grid_search(dataset_loader_train, model,
grid_search_params)
assert result == list(reversed(scores))
dataset_loader_train.assert_called_with()
GridSearchCV.assert_called_with(model, refit=False, verbose=4)
GridSearchCV().fit.assert_called_with(X, y)
def test_it(self, grid_search):
model, dataset_loader_train = Mock(), Mock()
grid_search_params = {'verbose': 4}
X, y = object(), object()
dataset_loader_train.return_value = X, y
scores = [
Mock(mean_validation_score=0.1),
Mock(mean_validation_score=0.2),
]
with patch('palladium.fit.GridSearchCV') as GridSearchCV:
GridSearchCV().grid_scores_ = scores
result = grid_search(
dataset_loader_train, model, grid_search_params)
assert result == list(reversed(scores))
dataset_loader_train.assert_called_with()
GridSearchCV.assert_called_with(model, refit=False, verbose=4)
GridSearchCV().fit.assert_called_with(X, y)
def test_it(self, grid_search, GridSearchCVWithScores, capsys, tmpdir):
model, dataset_loader_train = Mock(), Mock()
grid_search_params = {'verbose': 4}
X, y = object(), object()
dataset_loader_train.return_value = X, y
results_csv = tmpdir.join('results.csv')
result = grid_search(
dataset_loader_train=dataset_loader_train,
model=model,
grid_search=grid_search_params,
save_results=str(results_csv),
)
dataset_loader_train.assert_called_with()
GridSearchCVWithScores.assert_called_with(model,
refit=False,
verbose=4)
GridSearchCVWithScores().fit.assert_called_with(X, y)
assert result is GridSearchCVWithScores()
scores = GridSearchCVWithScores().cv_results_
assert (str(pandas.DataFrame(scores)).strip() == capsys.readouterr()
[0].strip())
assert (str(pandas.DataFrame(scores)).strip() == str(
pandas.read_csv(str(results_csv))).strip())
def test_no_score_method_raises(self, grid_search):
model, dataset_loader_train = Mock(spec=['fit', 'predict']), Mock()
dataset_loader_train.return_value = object(), object()
with pytest.raises(ValueError):
grid_search(dataset_loader_train, model, {})
def test_no_score_method_raises(self, grid_search):
model, dataset_loader_train = Mock(spec=['fit', 'predict']), Mock()
dataset_loader_train.return_value = object(), object()
with pytest.raises(ValueError):
grid_search(dataset_loader_train, model, {})
