def test_cross_val_multiscore():
"""Test cross_val_multiscore for computing scores on decoding over time."""
from sklearn.model_selection import KFold, StratifiedKFold, cross_val_score
from sklearn.linear_model import LogisticRegression, LinearRegression
if check_version('sklearn', '0.20'):
logreg = LogisticRegression(solver='liblinear', random_state=0)
else:
logreg = LogisticRegression(random_state=0)
# compare to cross-val-score
X = np.random.rand(20, 3)
y = np.arange(20) % 2
cv = KFold(2, random_state=0, shuffle=True)
clf = logreg
assert_array_equal(cross_val_score(clf, X, y, cv=cv),
cross_val_multiscore(clf, X, y, cv=cv))
# Test with search light
X = np.random.rand(20, 4, 3)
y = np.arange(20) % 2
clf = SlidingEstimator(logreg, scoring='accuracy')
scores_acc = cross_val_multiscore(clf, X, y, cv=cv)
assert_array_equal(np.shape(scores_acc), [2, 3])
# check values
scores_acc_manual = list()
for train, test in cv.split(X, y):
clf.fit(X[train], y[train])
scores_acc_manual.append(clf.score(X[test], y[test]))
assert_array_equal(scores_acc, scores_acc_manual)
# check scoring metric
# raise an error if scoring is defined at cross-val-score level and
# search light, because search light does not return a 1-dimensional
# prediction.
pytest.raises(ValueError,
cross_val_multiscore,
clf,
X,
y,
cv=cv,
scoring='roc_auc')
clf = SlidingEstimator(logreg, scoring='roc_auc')
scores_auc = cross_val_multiscore(clf, X, y, cv=cv, n_jobs=1)
scores_auc_manual = list()
for train, test in cv.split(X, y):
clf.fit(X[train], y[train])
scores_auc_manual.append(clf.score(X[test], y[test]))
assert_array_equal(scores_auc, scores_auc_manual)
# indirectly test that cross_val_multiscore rightly detects the type of
# estimator and generates a StratifiedKFold for classiers and a KFold
# otherwise
X = np.random.randn(1000, 3)
y = np.ones(1000, dtype=int)
y[::2] = 0
clf = logreg
reg = LinearRegression()
for cross_val in (cross_val_score, cross_val_multiscore):
manual = cross_val(clf, X, y, cv=StratifiedKFold(2))
auto = cross_val(clf, X, y, cv=2)
assert_array_equal(manual, auto)
manual = cross_val(reg, X, y, cv=KFold(2))
auto = cross_val(reg, X, y, cv=2)
assert_array_equal(manual, auto)
def test_search_light():
"""Test SlidingEstimator."""
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.metrics import roc_auc_score, make_scorer
with pytest.warns(None): # NumPy module import
from sklearn.ensemble import BaggingClassifier
from sklearn.base import is_classifier
logreg = LogisticRegression(solver='liblinear', multi_class='ovr',
random_state=0)
X, y = make_data()
n_epochs, _, n_time = X.shape
# init
pytest.raises(ValueError, SlidingEstimator, 'foo')
sl = SlidingEstimator(Ridge())
assert (not is_classifier(sl))
sl = SlidingEstimator(LogisticRegression(solver='liblinear'))
assert (is_classifier(sl))
# fit
assert_equal(sl.__repr__()[:18], '<SlidingEstimator(')
sl.fit(X, y)
assert_equal(sl.__repr__()[-28:], ', fitted with 10 estimators>')
pytest.raises(ValueError, sl.fit, X[1:], y)
pytest.raises(ValueError, sl.fit, X[:, :, 0], y)
sl.fit(X, y, sample_weight=np.ones_like(y))
# transforms
pytest.raises(ValueError, sl.predict, X[:, :, :2])
y_pred = sl.predict(X)
assert (y_pred.dtype == int)
assert_array_equal(y_pred.shape, [n_epochs, n_time])
y_proba = sl.predict_proba(X)
assert (y_proba.dtype == float)
assert_array_equal(y_proba.shape, [n_epochs, n_time, 2])
# score
score = sl.score(X, y)
assert_array_equal(score.shape, [n_time])
assert (np.sum(np.abs(score)) != 0)
assert (score.dtype == float)
sl = SlidingEstimator(logreg)
assert_equal(sl.scoring, None)
# Scoring method
for scoring in ['foo', 999]:
sl = SlidingEstimator(logreg, scoring=scoring)
sl.fit(X, y)
pytest.raises((ValueError, TypeError), sl.score, X, y)
# Check sklearn's roc_auc fix: scikit-learn/scikit-learn#6874
# -- 3 class problem
sl = SlidingEstimator(logreg, scoring='roc_auc')
y = np.arange(len(X)) % 3
sl.fit(X, y)
pytest.raises(ValueError, sl.score, X, y)
# -- 2 class problem not in [0, 1]
y = np.arange(len(X)) % 2 + 1
sl.fit(X, y)
score = sl.score(X, y)
assert_array_equal(score, [roc_auc_score(y - 1, _y_pred - 1)
for _y_pred in sl.decision_function(X).T])
y = np.arange(len(X)) % 2
# Cannot pass a metric as a scoring parameter
sl1 = SlidingEstimator(logreg, scoring=roc_auc_score)
sl1.fit(X, y)
pytest.raises(ValueError, sl1.score, X, y)
# Now use string as scoring
sl1 = SlidingEstimator(logreg, scoring='roc_auc')
sl1.fit(X, y)
rng = np.random.RandomState(0)
X = rng.randn(*X.shape) # randomize X to avoid AUCs in [0, 1]
score_sl = sl1.score(X, y)
assert_array_equal(score_sl.shape, [n_time])
assert (score_sl.dtype == float)
# Check that scoring was applied adequately
scoring = make_scorer(roc_auc_score, needs_threshold=True)
score_manual = [scoring(est, x, y) for est, x in zip(
sl1.estimators_, X.transpose(2, 0, 1))]
assert_array_equal(score_manual, score_sl)
# n_jobs
sl = SlidingEstimator(logreg, n_jobs=1, scoring='roc_auc')
score_1job = sl.fit(X, y).score(X, y)
sl.n_jobs = 2
score_njobs = sl.fit(X, y).score(X, y)
assert_array_equal(score_1job, score_njobs)
sl.predict(X)
# n_jobs > n_estimators
sl.fit(X[..., [0]], y)
sl.predict(X[..., [0]])
# pipeline
class _LogRegTransformer(LogisticRegression):
# XXX needs transformer in pipeline to get first proba only
def __init__(self):
super(_LogRegTransformer, self).__init__()
self.multi_class = 'ovr'
self.random_state = 0
self.solver = 'liblinear'
def transform(self, X):
return super(_LogRegTransformer, self).predict_proba(X)[..., 1]
pipe = make_pipeline(SlidingEstimator(_LogRegTransformer()),
logreg)
pipe.fit(X, y)
pipe.predict(X)
# n-dimensional feature space
X = np.random.rand(10, 3, 4, 2)
y = np.arange(10) % 2
y_preds = list()
for n_jobs in [1, 2]:
pipe = SlidingEstimator(
make_pipeline(Vectorizer(), logreg), n_jobs=n_jobs)
y_preds.append(pipe.fit(X, y).predict(X))
features_shape = pipe.estimators_[0].steps[0][1].features_shape_
assert_array_equal(features_shape, [3, 4])
assert_array_equal(y_preds[0], y_preds[1])
# Bagging classifiers
X = np.random.rand(10, 3, 4)
for n_jobs in (1, 2):
pipe = SlidingEstimator(BaggingClassifier(None, 2), n_jobs=n_jobs)
pipe.fit(X, y)
pipe.score(X, y)
assert (isinstance(pipe.estimators_[0], BaggingClassifier))
def test_search_light():
"""Test SlidingEstimator."""
from sklearn.linear_model import Ridge, LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.metrics import roc_auc_score, make_scorer
from sklearn.ensemble import BaggingClassifier
from sklearn.base import is_classifier
X, y = make_data()
n_epochs, _, n_time = X.shape
# init
pytest.raises(ValueError, SlidingEstimator, 'foo')
sl = SlidingEstimator(Ridge())
assert (not is_classifier(sl))
sl = SlidingEstimator(LogisticRegression())
assert (is_classifier(sl))
# fit
assert_equal(sl.__repr__()[:18], '<SlidingEstimator(')
sl.fit(X, y)
assert_equal(sl.__repr__()[-28:], ', fitted with 10 estimators>')
pytest.raises(ValueError, sl.fit, X[1:], y)
pytest.raises(ValueError, sl.fit, X[:, :, 0], y)
sl.fit(X, y, sample_weight=np.ones_like(y))
# transforms
pytest.raises(ValueError, sl.predict, X[:, :, :2])
y_pred = sl.predict(X)
assert (y_pred.dtype == int)
assert_array_equal(y_pred.shape, [n_epochs, n_time])
y_proba = sl.predict_proba(X)
assert (y_proba.dtype == float)
assert_array_equal(y_proba.shape, [n_epochs, n_time, 2])
# score
score = sl.score(X, y)
assert_array_equal(score.shape, [n_time])
assert (np.sum(np.abs(score)) != 0)
assert (score.dtype == float)
sl = SlidingEstimator(LogisticRegression())
assert_equal(sl.scoring, None)
# Scoring method
for scoring in ['foo', 999]:
sl = SlidingEstimator(LogisticRegression(), scoring=scoring)
sl.fit(X, y)
pytest.raises((ValueError, TypeError), sl.score, X, y)
# Check sklearn's roc_auc fix: scikit-learn/scikit-learn#6874
# -- 3 class problem
sl = SlidingEstimator(LogisticRegression(random_state=0),
scoring='roc_auc')
y = np.arange(len(X)) % 3
sl.fit(X, y)
pytest.raises(ValueError, sl.score, X, y)
# -- 2 class problem not in [0, 1]
y = np.arange(len(X)) % 2 + 1
sl.fit(X, y)
score = sl.score(X, y)
assert_array_equal(score, [
roc_auc_score(y - 1, _y_pred - 1)
for _y_pred in sl.decision_function(X).T
])
y = np.arange(len(X)) % 2
# Cannot pass a metric as a scoring parameter
sl1 = SlidingEstimator(LogisticRegression(), scoring=roc_auc_score)
sl1.fit(X, y)
pytest.raises(ValueError, sl1.score, X, y)
# Now use string as scoring
sl1 = SlidingEstimator(LogisticRegression(), scoring='roc_auc')
sl1.fit(X, y)
rng = np.random.RandomState(0)
X = rng.randn(*X.shape) # randomize X to avoid AUCs in [0, 1]
score_sl = sl1.score(X, y)
assert_array_equal(score_sl.shape, [n_time])
assert (score_sl.dtype == float)
# Check that scoring was applied adequately
scoring = make_scorer(roc_auc_score, needs_threshold=True)
score_manual = [
scoring(est, x, y)
for est, x in zip(sl1.estimators_, X.transpose(2, 0, 1))
]
assert_array_equal(score_manual, score_sl)
# n_jobs
sl = SlidingEstimator(LogisticRegression(random_state=0),
n_jobs=1,
scoring='roc_auc')
score_1job = sl.fit(X, y).score(X, y)
sl.n_jobs = 2
score_njobs = sl.fit(X, y).score(X, y)
assert_array_equal(score_1job, score_njobs)
sl.predict(X)
# n_jobs > n_estimators
sl.fit(X[..., [0]], y)
sl.predict(X[..., [0]])
# pipeline
class _LogRegTransformer(LogisticRegression):
# XXX needs transformer in pipeline to get first proba only
def transform(self, X):
return super(_LogRegTransformer, self).predict_proba(X)[..., 1]
pipe = make_pipeline(SlidingEstimator(_LogRegTransformer()),
LogisticRegression())
pipe.fit(X, y)
pipe.predict(X)
# n-dimensional feature space
X = np.random.rand(10, 3, 4, 2)
y = np.arange(10) % 2
y_preds = list()
for n_jobs in [1, 2]:
pipe = SlidingEstimator(make_pipeline(Vectorizer(),
LogisticRegression()),
n_jobs=n_jobs)
y_preds.append(pipe.fit(X, y).predict(X))
features_shape = pipe.estimators_[0].steps[0][1].features_shape_
assert_array_equal(features_shape, [3, 4])
assert_array_equal(y_preds[0], y_preds[1])
# Bagging classifiers
X = np.random.rand(10, 3, 4)
for n_jobs in (1, 2):
pipe = SlidingEstimator(BaggingClassifier(None, 2), n_jobs=n_jobs)
pipe.fit(X, y)
pipe.score(X, y)
assert (isinstance(pipe.estimators_[0], BaggingClassifier))
def test_get_coef():
"""Test the retrieval of linear coefficients (filters and patterns) from
simple and pipeline estimators.
"""
from sklearn.base import TransformerMixin, BaseEstimator
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
# Define a classifier, an invertible transformer and an non-invertible one.
class Clf(BaseEstimator):
def fit(self, X, y):
return self
class NoInv(TransformerMixin):
def fit(self, X, y):
return self
def transform(self, X):
return X
class Inv(NoInv):
def inverse_transform(self, X):
return X
np.random.RandomState(0)
n_samples, n_features = 20, 3
y = (np.arange(n_samples) % 2) * 2 - 1
w = np.random.randn(n_features, 1)
X = w.dot(y[np.newaxis, :]).T + np.random.randn(n_samples, n_features)
# I. Test inverse function
# Check that we retrieve the right number of inverse functions even if
# there are nested pipelines
good_estimators = [
(1, make_pipeline(Inv(), Clf())),
(2, make_pipeline(Inv(), Inv(), Clf())),
(3, make_pipeline(Inv(), make_pipeline(Inv(), Inv()), Clf())),
]
for expected_n, est in good_estimators:
est.fit(X, y)
assert_true(expected_n == len(_get_inverse_funcs(est)))
bad_estimators = [
Clf(), # no preprocessing
Inv(), # final estimator isn't classifier
make_pipeline(NoInv(), Clf()), # first step isn't invertible
make_pipeline(Inv(), make_pipeline(
Inv(), NoInv()), Clf()), # nested step isn't invertible
]
for est in bad_estimators:
est.fit(X, y)
invs = _get_inverse_funcs(est)
assert_equal(invs, list())
# II. Test get coef for simple estimator and pipelines
for clf in (LinearModel(), make_pipeline(StandardScaler(), LinearModel())):
clf.fit(X, y)
# Retrieve final linear model
filters = get_coef(clf, 'filters_', False)
if hasattr(clf, 'steps'):
coefs = clf.steps[-1][-1].model.coef_
else:
coefs = clf.model.coef_
assert_array_equal(filters, coefs[0])
patterns = get_coef(clf, 'patterns_', False)
assert_true(filters[0] != patterns[0])
n_chans = X.shape[1]
assert_array_equal(filters.shape, patterns.shape, [n_chans, n_chans])
# Inverse transform linear model
filters_inv = get_coef(clf, 'filters_', True)
assert_true(filters[0] != filters_inv[0])
patterns_inv = get_coef(clf, 'patterns_', True)
assert_true(patterns[0] != patterns_inv[0])
# Check patterns values
clf = make_pipeline(StandardScaler(), LinearModel(LinearRegression()))
clf.fit(X, y)
patterns = get_coef(clf, 'patterns_', True)
mean, std = X.mean(0), X.std(0)
X = (X - mean) / std
coef = np.linalg.pinv(X.T.dot(X)).dot(X.T.dot(y))
patterns_manual = np.cov(X.T).dot(coef)
assert_array_almost_equal(patterns, patterns_manual * std + mean)
# Check with search_light:
n_samples, n_features, n_times = 20, 3, 5
y = np.arange(n_samples) % 2
X = np.random.rand(n_samples, n_features, n_times)
clf = SlidingEstimator(make_pipeline(StandardScaler(), LinearModel()))
clf.fit(X, y)
for inverse in (True, False):
patterns = get_coef(clf, 'patterns_', inverse)
filters = get_coef(clf, 'filters_', inverse)
assert_array_equal(filters.shape, patterns.shape,
[n_features, n_times])
for t in [0, 1]:
assert_array_equal(get_coef(clf.estimators_[t], 'filters_', False),
filters[:, t])