def test_generalization_across_time():
"""Test time generalization decoding
"""
from sklearn.svm import SVC
from sklearn.base import is_classifier
# KernelRidge is used for testing 1) regression analyses 2) n-dimensional
# predictions.
from sklearn.kernel_ridge import KernelRidge
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import roc_auc_score, mean_squared_error
epochs = make_epochs()
y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1))
if check_version('sklearn', '0.18'):
from sklearn.model_selection import (KFold, StratifiedKFold,
ShuffleSplit, LeaveOneLabelOut)
cv_shuffle = ShuffleSplit()
cv = LeaveOneLabelOut()
# XXX we cannot pass any other parameters than X and y to cv.split
# so we have to build it before hand
cv_lolo = [(train, test) for train, test in cv.split(
X=y_4classes, y=y_4classes, labels=y_4classes)]
# With sklearn >= 0.17, `clf` can be identified as a regressor, and
# the scoring metrics can therefore be automatically assigned.
scorer_regress = None
else:
from sklearn.cross_validation import (KFold, StratifiedKFold,
ShuffleSplit, LeaveOneLabelOut)
cv_shuffle = ShuffleSplit(len(epochs))
cv_lolo = LeaveOneLabelOut(y_4classes)
# With sklearn < 0.17, `clf` cannot be identified as a regressor, and
# therefore the scoring metrics cannot be automatically assigned.
scorer_regress = mean_squared_error
# Test default running
gat = GeneralizationAcrossTime(picks='foo')
assert_equal("<GAT | no fit, no prediction, no score>", "%s" % gat)
assert_raises(ValueError, gat.fit, epochs)
with warnings.catch_warnings(record=True):
# check classic fit + check manual picks
gat.picks = [0]
gat.fit(epochs)
# check optional y as array
gat.picks = None
gat.fit(epochs, y=epochs.events[:, 2])
# check optional y as list
gat.fit(epochs, y=epochs.events[:, 2].tolist())
assert_equal(len(gat.picks_), len(gat.ch_names), 1)
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), no "
"prediction, no score>", '%s' % gat)
assert_equal(gat.ch_names, epochs.ch_names)
# test different predict function:
gat = GeneralizationAcrossTime(predict_method='decision_function')
gat.fit(epochs)
# With classifier, the default cv is StratifiedKFold
assert_true(gat.cv_.__class__ == StratifiedKFold)
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 1))
gat.predict_method = 'predict_proba'
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 2))
gat.predict_method = 'foo'
assert_raises(NotImplementedError, gat.predict, epochs)
gat.predict_method = 'predict'
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 1))
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs, no score>",
"%s" % gat)
gat.score(epochs)
assert_true(gat.scorer_.__name__ == 'accuracy_score')
# check clf / predict_method combinations for which the scoring metrics
# cannot be inferred.
gat.scorer = None
gat.predict_method = 'decision_function'
assert_raises(ValueError, gat.score, epochs)
# Check specifying y manually
gat.predict_method = 'predict'
gat.score(epochs, y=epochs.events[:, 2])
gat.score(epochs, y=epochs.events[:, 2].tolist())
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs,\n scored "
"(accuracy_score)>", "%s" % gat)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=epochs.events[:, 2])
old_mode = gat.predict_mode
gat.predict_mode = 'super-foo-mode'
assert_raises(ValueError, gat.predict, epochs)
gat.predict_mode = old_mode
gat.score(epochs, y=epochs.events[:, 2])
assert_true("accuracy_score" in '%s' % gat.scorer_)
epochs2 = epochs.copy()
# check _DecodingTime class
assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.050 (s), length: 0.050 (s), n_time_windows: 15>",
"%s" % gat.train_times_)
assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.050 (s), length: 0.050 (s), n_time_windows: 15 x 15>",
"%s" % gat.test_times_)
# the y-check
gat.predict_mode = 'mean-prediction'
epochs2.events[:, 2] += 10
gat_ = copy.deepcopy(gat)
with use_log_level('error'):
assert_raises(ValueError, gat_.score, epochs2)
gat.predict_mode = 'cross-validation'
# Test basics
# --- number of trials
assert_true(gat.y_train_.shape[0] ==
gat.y_true_.shape[0] ==
len(gat.y_pred_[0][0]) == 14)
# --- number of folds
assert_true(np.shape(gat.estimators_)[1] == gat.cv)
# --- length training size
assert_true(len(gat.train_times_['slices']) == 15 ==
np.shape(gat.estimators_)[0])
# --- length testing sizes
assert_true(len(gat.test_times_['slices']) == 15 ==
np.shape(gat.scores_)[0])
assert_true(len(gat.test_times_['slices'][0]) == 15 ==
np.shape(gat.scores_)[1])
# Test score_mode
gat.score_mode = 'foo'
assert_raises(ValueError, gat.score, epochs)
gat.score_mode = 'fold-wise'
scores = gat.score(epochs)
assert_array_equal(np.shape(scores), [15, 15, 5])
gat.score_mode = 'mean-sample-wise'
scores = gat.score(epochs)
assert_array_equal(np.shape(scores), [15, 15])
gat.score_mode = 'mean-fold-wise'
scores = gat.score(epochs)
assert_array_equal(np.shape(scores), [15, 15])
gat.predict_mode = 'mean-prediction'
with warnings.catch_warnings(record=True) as w:
gat.score(epochs)
assert_true(any("score_mode changed from " in str(ww.message)
for ww in w))
# Test longer time window
gat = GeneralizationAcrossTime(train_times={'length': .100})
with warnings.catch_warnings(record=True):
gat2 = gat.fit(epochs)
assert_true(gat is gat2) # return self
assert_true(hasattr(gat2, 'cv_'))
assert_true(gat2.cv_ != gat.cv)
with warnings.catch_warnings(record=True): # not vectorizing
scores = gat.score(epochs)
assert_true(isinstance(scores, np.ndarray)) # type check
assert_equal(len(scores[0]), len(scores)) # shape check
assert_equal(len(gat.test_times_['slices'][0][0]), 2)
# Decim training steps
gat = GeneralizationAcrossTime(train_times={'step': .100})
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_true(len(gat.scores_) == len(gat.estimators_) == 8) # training time
assert_equal(len(gat.scores_[0]), 15) # testing time
# Test start stop training & test cv without n_fold params
y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1))
train_times = dict(start=0.090, stop=0.250)
gat = GeneralizationAcrossTime(cv=cv_lolo, train_times=train_times)
# predict without fit
assert_raises(RuntimeError, gat.predict, epochs)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=y_4classes)
gat.score(epochs)
assert_equal(len(gat.scores_), 4)
assert_equal(gat.train_times_['times'][0], epochs.times[6])
assert_equal(gat.train_times_['times'][-1], epochs.times[9])
# Test score without passing epochs & Test diagonal decoding
gat = GeneralizationAcrossTime(test_times='diagonal')
with warnings.catch_warnings(record=True): # not vectorizing
gat.fit(epochs)
assert_raises(RuntimeError, gat.score)
with warnings.catch_warnings(record=True): # not vectorizing
gat.predict(epochs)
scores = gat.score()
assert_true(scores is gat.scores_)
assert_equal(np.shape(gat.scores_), (15, 1))
assert_array_equal([tim for ttime in gat.test_times_['times']
for tim in ttime], gat.train_times_['times'])
# Test generalization across conditions
gat = GeneralizationAcrossTime(predict_mode='mean-prediction', cv=2)
with warnings.catch_warnings(record=True):
gat.fit(epochs[0:6])
with warnings.catch_warnings(record=True):
# There are some empty test folds because of n_trials
gat.predict(epochs[7:])
gat.score(epochs[7:])
# Test training time parameters
gat_ = copy.deepcopy(gat)
# --- start stop outside time range
gat_.train_times = dict(start=-999.)
with use_log_level('error'):
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(start=999.)
assert_raises(ValueError, gat_.fit, epochs)
# --- impossible slices
gat_.train_times = dict(step=.000001)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(length=.000001)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(length=999.)
assert_raises(ValueError, gat_.fit, epochs)
# Test testing time parameters
# --- outside time range
gat.test_times = dict(start=-999.)
with warnings.catch_warnings(record=True): # no epochs in fold
assert_raises(ValueError, gat.predict, epochs)
gat.test_times = dict(start=999.)
with warnings.catch_warnings(record=True): # no test epochs
assert_raises(ValueError, gat.predict, epochs)
# --- impossible slices
gat.test_times = dict(step=.000001)
with warnings.catch_warnings(record=True): # no test epochs
assert_raises(ValueError, gat.predict, epochs)
gat_ = copy.deepcopy(gat)
gat_.train_times_['length'] = .000001
gat_.test_times = dict(length=.000001)
with warnings.catch_warnings(record=True): # no test epochs
assert_raises(ValueError, gat_.predict, epochs)
# --- test time region of interest
gat.test_times = dict(step=.150)
with warnings.catch_warnings(record=True): # not vectorizing
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 5, 14, 1))
# --- silly value
gat.test_times = 'foo'
with warnings.catch_warnings(record=True): # no test epochs
assert_raises(ValueError, gat.predict, epochs)
assert_raises(RuntimeError, gat.score)
# --- unmatched length between training and testing time
gat.test_times = dict(length=.150)
assert_raises(ValueError, gat.predict, epochs)
# --- irregular length training and testing times
# 2 estimators, the first one is trained on two successive time samples
# whereas the second one is trained on a single time sample.
train_times = dict(slices=[[0, 1], [1]])
# The first estimator is tested once, the second estimator is tested on
# two successive time samples.
test_times = dict(slices=[[[0, 1]], [[0], [1]]])
gat = GeneralizationAcrossTime(train_times=train_times,
test_times=test_times)
gat.fit(epochs)
with warnings.catch_warnings(record=True): # not vectorizing
gat.score(epochs)
assert_array_equal(np.shape(gat.y_pred_[0]), [1, len(epochs), 1])
assert_array_equal(np.shape(gat.y_pred_[1]), [2, len(epochs), 1])
# check cannot Automatically infer testing times for adhoc training times
gat.test_times = None
assert_raises(ValueError, gat.predict, epochs)
svc = SVC(C=1, kernel='linear', probability=True)
gat = GeneralizationAcrossTime(clf=svc, predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs)
# sklearn needs it: c.f.
# https://github.com/scikit-learn/scikit-learn/issues/2723
# and http://bit.ly/1u7t8UT
with use_log_level('error'):
assert_raises(ValueError, gat.score, epochs2)
gat.score(epochs)
assert_true(0.0 <= np.min(scores) <= 1.0)
assert_true(0.0 <= np.max(scores) <= 1.0)
# Test that gets error if train on one dataset, test on another, and don't
# specify appropriate cv:
gat = GeneralizationAcrossTime(cv=cv_shuffle)
gat.fit(epochs)
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
assert_raises(ValueError, gat.predict, epochs[:10])
# Make CV with some empty train and test folds:
# --- empty test fold(s) should warn when gat.predict()
gat._cv_splits[0] = [gat._cv_splits[0][0], np.empty(0)]
with warnings.catch_warnings(record=True) as w:
gat.predict(epochs)
assert_true(len(w) > 0)
assert_true(any('do not have any test epochs' in str(ww.message)
for ww in w))
# --- empty train fold(s) should raise when gat.fit()
gat = GeneralizationAcrossTime(cv=[([0], [1]), ([], [0])])
assert_raises(ValueError, gat.fit, epochs[:2])
# Check that still works with classifier that output y_pred with
# shape = (n_trials, 1) instead of (n_trials,)
if check_version('sklearn', '0.17'): # no is_regressor before v0.17
gat = GeneralizationAcrossTime(clf=KernelRidge(), cv=2)
epochs.crop(None, epochs.times[2])
gat.fit(epochs)
# With regression the default cv is KFold and not StratifiedKFold
assert_true(gat.cv_.__class__ == KFold)
gat.score(epochs)
# with regression the default scoring metrics is mean squared error
assert_true(gat.scorer_.__name__ == 'mean_squared_error')
# Test combinations of complex scenarios
# 2 or more distinct classes
n_classes = [2, 4] # 4 tested
# nicely ordered labels or not
le = LabelEncoder()
y = le.fit_transform(epochs.events[:, 2])
y[len(y) // 2:] += 2
ys = (y, y + 1000)
# Univariate and multivariate prediction
svc = SVC(C=1, kernel='linear', probability=True)
reg = KernelRidge()
def scorer_proba(y_true, y_pred):
return roc_auc_score(y_true, y_pred[:, 0])
# We re testing 3 scenario: default, classifier + predict_proba, regressor
scorers = [None, scorer_proba, scorer_regress]
predict_methods = [None, 'predict_proba', None]
clfs = [svc, svc, reg]
# Test all combinations
for clf, predict_method, scorer in zip(clfs, predict_methods, scorers):
for y in ys:
for n_class in n_classes:
for predict_mode in ['cross-validation', 'mean-prediction']:
# Cannot use AUC for n_class > 2
if (predict_method == 'predict_proba' and n_class != 2):
continue
y_ = y % n_class
with warnings.catch_warnings(record=True):
gat = GeneralizationAcrossTime(
cv=2, clf=clf, scorer=scorer,
predict_mode=predict_mode)
gat.fit(epochs, y=y_)
gat.score(epochs, y=y_)
# Check that scorer is correctly defined manually and
# automatically.
scorer_name = gat.scorer_.__name__
if scorer is None:
if is_classifier(clf):
assert_equal(scorer_name, 'accuracy_score')
else:
assert_equal(scorer_name, 'mean_squared_error')
else:
assert_equal(scorer_name, scorer.__name__)
def test_generalization_across_time():
"""Test time generalization decoding
"""
from sklearn.svm import SVC
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import mean_squared_error
raw = io.Raw(raw_fname, preload=False)
events = read_events(event_name)
picks = pick_types(raw.info, meg='mag', stim=False, ecg=False,
eog=False, exclude='bads')
picks = picks[0:2]
decim = 30
# Test on time generalization within one condition
with warnings.catch_warnings(record=True):
epochs = Epochs(raw, events, event_id, tmin, tmax, picks=picks,
baseline=(None, 0), preload=True, decim=decim)
# Test default running
gat = GeneralizationAcrossTime()
assert_equal("<GAT | no fit, no prediction, no score>", "%s" % gat)
assert_raises(ValueError, gat.fit, epochs, picks='foo')
with warnings.catch_warnings(record=True):
# check classic fit + check manual picks
gat.fit(epochs, picks=[0])
# check optional y as array
gat.fit(epochs, y=epochs.events[:, 2])
# check optional y as list
gat.fit(epochs, y=epochs.events[:, 2].tolist())
assert_equal(len(gat.picks_), len(gat.ch_names), 1)
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), no "
"prediction, no score>", '%s' % gat)
assert_equal(gat.ch_names, epochs.ch_names)
gat.predict(epochs)
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs, no score>",
"%s" % gat)
gat.score(epochs)
gat.score(epochs, y=epochs.events[:, 2])
gat.score(epochs, y=epochs.events[:, 2].tolist())
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs,\n scored "
"(accuracy_score)>", "%s" % gat)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=epochs.events[:, 2])
old_mode = gat.predict_mode
gat.predict_mode = 'super-foo-mode'
assert_raises(ValueError, gat.predict, epochs)
gat.predict_mode = old_mode
gat.score(epochs, y=epochs.events[:, 2])
assert_true("accuracy_score" in '%s' % gat.scorer_)
epochs2 = epochs.copy()
# check _DecodingTime class
assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.047 (s), length: 0.047 (s), n_time_windows: 15>",
"%s" % gat.train_times)
assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.047 (s), length: 0.047 (s), n_time_windows: 15 x 15>",
"%s" % gat.test_times_)
# the y-check
gat.predict_mode = 'mean-prediction'
epochs2.events[:, 2] += 10
gat_ = copy.deepcopy(gat)
assert_raises(ValueError, gat_.score, epochs2)
gat.predict_mode = 'cross-validation'
# Test basics
# --- number of trials
assert_true(gat.y_train_.shape[0] ==
gat.y_true_.shape[0] ==
len(gat.y_pred_[0][0]) == 14)
# --- number of folds
assert_true(np.shape(gat.estimators_)[1] == gat.cv)
# --- length training size
assert_true(len(gat.train_times['slices']) == 15 ==
np.shape(gat.estimators_)[0])
# --- length testing sizes
assert_true(len(gat.test_times_['slices']) == 15 ==
np.shape(gat.scores_)[0])
assert_true(len(gat.test_times_['slices'][0]) == 15 ==
np.shape(gat.scores_)[1])
# Test longer time window
gat = GeneralizationAcrossTime(train_times={'length': .100})
with warnings.catch_warnings(record=True):
gat2 = gat.fit(epochs)
assert_true(gat is gat2) # return self
assert_true(hasattr(gat2, 'cv_'))
assert_true(gat2.cv_ != gat.cv)
scores = gat.score(epochs)
assert_true(isinstance(scores, list)) # type check
assert_equal(len(scores[0]), len(scores)) # shape check
assert_equal(len(gat.test_times_['slices'][0][0]), 2)
# Decim training steps
gat = GeneralizationAcrossTime(train_times={'step': .100})
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_true(len(gat.scores_) == len(gat.estimators_) == 8) # training time
assert_equal(len(gat.scores_[0]), 15) # testing time
# Test start stop training
gat = GeneralizationAcrossTime(train_times={'start': 0.090,
'stop': 0.250})
# predict without fit
assert_raises(RuntimeError, gat.predict, epochs)
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_equal(len(gat.scores_), 4)
assert_equal(gat.train_times['times_'][0], epochs.times[6])
assert_equal(gat.train_times['times_'][-1], epochs.times[9])
# Test score without passing epochs
gat = GeneralizationAcrossTime()
with warnings.catch_warnings(record=True):
gat.fit(epochs)
assert_raises(RuntimeError, gat.score)
gat.predict(epochs, test_times='diagonal') # Test diagonal decoding
scores = gat.score()
assert_true(scores is gat.scores_)
assert_equal(np.shape(gat.scores_), (15, 1))
# Test generalization across conditions
gat = GeneralizationAcrossTime(predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs[0:6])
gat.predict(epochs[7:])
assert_raises(ValueError, gat.predict, epochs, test_times='hahahaha')
assert_raises(RuntimeError, gat.score)
gat.score(epochs[7:])
svc = SVC(C=1, kernel='linear', probability=True)
gat = GeneralizationAcrossTime(clf=svc, predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs)
# sklearn needs it: c.f.
# https://github.com/scikit-learn/scikit-learn/issues/2723
# and http://bit.ly/1u7t8UT
assert_raises(ValueError, gat.score, epochs2)
gat.score(epochs)
scores = sum(scores, []) # flatten
assert_true(0.0 <= np.min(scores) <= 1.0)
assert_true(0.0 <= np.max(scores) <= 1.0)
# Test that gets error if train on one dataset, test on another, and don't
# specify appropriate cv:
gat = GeneralizationAcrossTime()
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
assert_raises(ValueError, gat.predict, epochs[:10])
# Test combinations of complex scenarios
# 2 or more distinct classes
n_classes = [2, 4] # 4 tested
# nicely ordered labels or not
le = LabelEncoder()
y = le.fit_transform(epochs.events[:, 2])
y[len(y) // 2:] += 2
ys = (y, y + 1000)
# Univariate and multivariate prediction
svc = SVC(C=1, kernel='linear')
class SVC_proba(SVC):
def predict(self, x):
probas = super(SVC_proba, self).predict_proba(x)
return probas[:, 0]
svcp = SVC_proba(C=1, kernel='linear', probability=True)
clfs = [svc, svcp]
scorers = [None, mean_squared_error]
# Test all combinations
for clf, scorer in zip(clfs, scorers):
for y in ys:
for n_class in n_classes:
y_ = y % n_class
with warnings.catch_warnings(record=True):
gat = GeneralizationAcrossTime(cv=2, clf=clf)
gat.fit(epochs, y=y_)
gat.score(epochs, y=y_, scorer=scorer)
def test_generalization_across_time():
"""Test time generalization decoding
"""
from sklearn.svm import SVC
from sklearn.base import is_classifier
# KernelRidge is used for testing 1) regression analyses 2) n-dimensional
# predictions.
from sklearn.kernel_ridge import KernelRidge
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import roc_auc_score, mean_squared_error
epochs = make_epochs()
y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1))
if check_version('sklearn', '0.18'):
from sklearn.model_selection import (KFold, StratifiedKFold,
ShuffleSplit, LeaveOneLabelOut)
cv_shuffle = ShuffleSplit()
cv = LeaveOneLabelOut()
# XXX we cannot pass any other parameters than X and y to cv.split
# so we have to build it before hand
cv_lolo = [(train, test) for train, test in cv.split(
X=y_4classes, y=y_4classes, labels=y_4classes)]
# With sklearn >= 0.17, `clf` can be identified as a regressor, and
# the scoring metrics can therefore be automatically assigned.
scorer_regress = None
else:
from sklearn.cross_validation import (KFold, StratifiedKFold,
ShuffleSplit, LeaveOneLabelOut)
cv_shuffle = ShuffleSplit(len(epochs))
cv_lolo = LeaveOneLabelOut(y_4classes)
# With sklearn < 0.17, `clf` cannot be identified as a regressor, and
# therefore the scoring metrics cannot be automatically assigned.
scorer_regress = mean_squared_error
# Test default running
gat = GeneralizationAcrossTime(picks='foo')
assert_equal("<GAT | no fit, no prediction, no score>", "%s" % gat)
assert_raises(ValueError, gat.fit, epochs)
with warnings.catch_warnings(record=True):
# check classic fit + check manual picks
gat.picks = [0]
gat.fit(epochs)
# check optional y as array
gat.picks = None
gat.fit(epochs, y=epochs.events[:, 2])
# check optional y as list
gat.fit(epochs, y=epochs.events[:, 2].tolist())
assert_equal(len(gat.picks_), len(gat.ch_names), 1)
assert_equal(
"<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), no "
"prediction, no score>", '%s' % gat)
assert_equal(gat.ch_names, epochs.ch_names)
# test different predict function:
gat = GeneralizationAcrossTime(predict_method='decision_function')
gat.fit(epochs)
# With classifier, the default cv is StratifiedKFold
assert_true(gat.cv_.__class__ == StratifiedKFold)
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 1))
gat.predict_method = 'predict_proba'
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 2))
gat.predict_method = 'foo'
assert_raises(NotImplementedError, gat.predict, epochs)
gat.predict_method = 'predict'
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 1))
assert_equal(
"<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs, no score>", "%s" % gat)
gat.score(epochs)
assert_true(gat.scorer_.__name__ == 'accuracy_score')
# check clf / predict_method combinations for which the scoring metrics
# cannot be inferred.
gat.scorer = None
gat.predict_method = 'decision_function'
assert_raises(ValueError, gat.score, epochs)
# Check specifying y manually
gat.predict_method = 'predict'
gat.score(epochs, y=epochs.events[:, 2])
gat.score(epochs, y=epochs.events[:, 2].tolist())
assert_equal(
"<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs,\n scored "
"(accuracy_score)>", "%s" % gat)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=epochs.events[:, 2])
old_mode = gat.predict_mode
gat.predict_mode = 'super-foo-mode'
assert_raises(ValueError, gat.predict, epochs)
gat.predict_mode = old_mode
gat.score(epochs, y=epochs.events[:, 2])
assert_true("accuracy_score" in '%s' % gat.scorer_)
epochs2 = epochs.copy()
# check _DecodingTime class
assert_equal(
"<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.050 (s), length: 0.050 (s), n_time_windows: 15>",
"%s" % gat.train_times_)
assert_equal(
"<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.050 (s), length: 0.050 (s), n_time_windows: 15 x 15>",
"%s" % gat.test_times_)
# the y-check
gat.predict_mode = 'mean-prediction'
epochs2.events[:, 2] += 10
gat_ = copy.deepcopy(gat)
with use_log_level('error'):
assert_raises(ValueError, gat_.score, epochs2)
gat.predict_mode = 'cross-validation'
# Test basics
# --- number of trials
assert_true(gat.y_train_.shape[0] == gat.y_true_.shape[0] == len(
gat.y_pred_[0][0]) == 14)
# --- number of folds
assert_true(np.shape(gat.estimators_)[1] == gat.cv)
# --- length training size
assert_true(
len(gat.train_times_['slices']) == 15 == np.shape(gat.estimators_)[0])
# --- length testing sizes
assert_true(
len(gat.test_times_['slices']) == 15 == np.shape(gat.scores_)[0])
assert_true(
len(gat.test_times_['slices'][0]) == 15 == np.shape(gat.scores_)[1])
# Test score_mode
gat.score_mode = 'foo'
assert_raises(ValueError, gat.score, epochs)
gat.score_mode = 'fold-wise'
scores = gat.score(epochs)
assert_array_equal(np.shape(scores), [15, 15, 5])
gat.score_mode = 'mean-sample-wise'
scores = gat.score(epochs)
assert_array_equal(np.shape(scores), [15, 15])
gat.score_mode = 'mean-fold-wise'
scores = gat.score(epochs)
assert_array_equal(np.shape(scores), [15, 15])
gat.predict_mode = 'mean-prediction'
with warnings.catch_warnings(record=True) as w:
gat.score(epochs)
assert_true(
any("score_mode changed from " in str(ww.message) for ww in w))
# Test longer time window
gat = GeneralizationAcrossTime(train_times={'length': .100})
with warnings.catch_warnings(record=True):
gat2 = gat.fit(epochs)
assert_true(gat is gat2) # return self
assert_true(hasattr(gat2, 'cv_'))
assert_true(gat2.cv_ != gat.cv)
with warnings.catch_warnings(record=True): # not vectorizing
scores = gat.score(epochs)
assert_true(isinstance(scores, np.ndarray)) # type check
assert_equal(len(scores[0]), len(scores)) # shape check
assert_equal(len(gat.test_times_['slices'][0][0]), 2)
# Decim training steps
gat = GeneralizationAcrossTime(train_times={'step': .100})
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_true(len(gat.scores_) == len(gat.estimators_) == 8) # training time
assert_equal(len(gat.scores_[0]), 15) # testing time
# Test start stop training & test cv without n_fold params
y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1))
train_times = dict(start=0.090, stop=0.250)
gat = GeneralizationAcrossTime(cv=cv_lolo, train_times=train_times)
# predict without fit
assert_raises(RuntimeError, gat.predict, epochs)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=y_4classes)
gat.score(epochs)
assert_equal(len(gat.scores_), 4)
assert_equal(gat.train_times_['times'][0], epochs.times[6])
assert_equal(gat.train_times_['times'][-1], epochs.times[9])
# Test score without passing epochs & Test diagonal decoding
gat = GeneralizationAcrossTime(test_times='diagonal')
with warnings.catch_warnings(record=True): # not vectorizing
gat.fit(epochs)
assert_raises(RuntimeError, gat.score)
with warnings.catch_warnings(record=True): # not vectorizing
gat.predict(epochs)
scores = gat.score()
assert_true(scores is gat.scores_)
assert_equal(np.shape(gat.scores_), (15, 1))
assert_array_equal(
[tim for ttime in gat.test_times_['times'] for tim in ttime],
gat.train_times_['times'])
# Test generalization across conditions
gat = GeneralizationAcrossTime(predict_mode='mean-prediction', cv=2)
with warnings.catch_warnings(record=True):
gat.fit(epochs[0:6])
with warnings.catch_warnings(record=True):
# There are some empty test folds because of n_trials
gat.predict(epochs[7:])
gat.score(epochs[7:])
# Test training time parameters
gat_ = copy.deepcopy(gat)
# --- start stop outside time range
gat_.train_times = dict(start=-999.)
with use_log_level('error'):
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(start=999.)
assert_raises(ValueError, gat_.fit, epochs)
# --- impossible slices
gat_.train_times = dict(step=.000001)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(length=.000001)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(length=999.)
assert_raises(ValueError, gat_.fit, epochs)
# Test testing time parameters
# --- outside time range
gat.test_times = dict(start=-999.)
with warnings.catch_warnings(record=True): # no epochs in fold
assert_raises(ValueError, gat.predict, epochs)
gat.test_times = dict(start=999.)
with warnings.catch_warnings(record=True): # no test epochs
assert_raises(ValueError, gat.predict, epochs)
# --- impossible slices
gat.test_times = dict(step=.000001)
with warnings.catch_warnings(record=True): # no test epochs
assert_raises(ValueError, gat.predict, epochs)
gat_ = copy.deepcopy(gat)
gat_.train_times_['length'] = .000001
gat_.test_times = dict(length=.000001)
with warnings.catch_warnings(record=True): # no test epochs
assert_raises(ValueError, gat_.predict, epochs)
# --- test time region of interest
gat.test_times = dict(step=.150)
with warnings.catch_warnings(record=True): # not vectorizing
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 5, 14, 1))
# --- silly value
gat.test_times = 'foo'
with warnings.catch_warnings(record=True): # no test epochs
assert_raises(ValueError, gat.predict, epochs)
assert_raises(RuntimeError, gat.score)
# --- unmatched length between training and testing time
gat.test_times = dict(length=.150)
assert_raises(ValueError, gat.predict, epochs)
# --- irregular length training and testing times
# 2 estimators, the first one is trained on two successive time samples
# whereas the second one is trained on a single time sample.
train_times = dict(slices=[[0, 1], [1]])
# The first estimator is tested once, the second estimator is tested on
# two successive time samples.
test_times = dict(slices=[[[0, 1]], [[0], [1]]])
gat = GeneralizationAcrossTime(train_times=train_times,
test_times=test_times)
gat.fit(epochs)
with warnings.catch_warnings(record=True): # not vectorizing
gat.score(epochs)
assert_array_equal(np.shape(gat.y_pred_[0]), [1, len(epochs), 1])
assert_array_equal(np.shape(gat.y_pred_[1]), [2, len(epochs), 1])
# check cannot Automatically infer testing times for adhoc training times
gat.test_times = None
assert_raises(ValueError, gat.predict, epochs)
svc = SVC(C=1, kernel='linear', probability=True)
gat = GeneralizationAcrossTime(clf=svc, predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs)
# sklearn needs it: c.f.
# https://github.com/scikit-learn/scikit-learn/issues/2723
# and http://bit.ly/1u7t8UT
with use_log_level('error'):
assert_raises(ValueError, gat.score, epochs2)
gat.score(epochs)
assert_true(0.0 <= np.min(scores) <= 1.0)
assert_true(0.0 <= np.max(scores) <= 1.0)
# Test that gets error if train on one dataset, test on another, and don't
# specify appropriate cv:
gat = GeneralizationAcrossTime(cv=cv_shuffle)
gat.fit(epochs)
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
assert_raises(ValueError, gat.predict, epochs[:10])
# Make CV with some empty train and test folds:
# --- empty test fold(s) should warn when gat.predict()
gat._cv_splits[0] = [gat._cv_splits[0][0], np.empty(0)]
with warnings.catch_warnings(record=True) as w:
gat.predict(epochs)
assert_true(len(w) > 0)
assert_true(
any('do not have any test epochs' in str(ww.message) for ww in w))
# --- empty train fold(s) should raise when gat.fit()
gat = GeneralizationAcrossTime(cv=[([0], [1]), ([], [0])])
assert_raises(ValueError, gat.fit, epochs[:2])
# Check that still works with classifier that output y_pred with
# shape = (n_trials, 1) instead of (n_trials,)
if check_version('sklearn', '0.17'): # no is_regressor before v0.17
gat = GeneralizationAcrossTime(clf=KernelRidge(), cv=2)
epochs.crop(None, epochs.times[2])
gat.fit(epochs)
# With regression the default cv is KFold and not StratifiedKFold
assert_true(gat.cv_.__class__ == KFold)
gat.score(epochs)
# with regression the default scoring metrics is mean squared error
assert_true(gat.scorer_.__name__ == 'mean_squared_error')
# Test combinations of complex scenarios
# 2 or more distinct classes
n_classes = [2, 4] # 4 tested
# nicely ordered labels or not
le = LabelEncoder()
y = le.fit_transform(epochs.events[:, 2])
y[len(y) // 2:] += 2
ys = (y, y + 1000)
# Univariate and multivariate prediction
svc = SVC(C=1, kernel='linear', probability=True)
reg = KernelRidge()
def scorer_proba(y_true, y_pred):
return roc_auc_score(y_true, y_pred[:, 0])
# We re testing 3 scenario: default, classifier + predict_proba, regressor
scorers = [None, scorer_proba, scorer_regress]
predict_methods = [None, 'predict_proba', None]
clfs = [svc, svc, reg]
# Test all combinations
for clf, predict_method, scorer in zip(clfs, predict_methods, scorers):
for y in ys:
for n_class in n_classes:
for predict_mode in ['cross-validation', 'mean-prediction']:
# Cannot use AUC for n_class > 2
if (predict_method == 'predict_proba' and n_class != 2):
continue
y_ = y % n_class
with warnings.catch_warnings(record=True):
gat = GeneralizationAcrossTime(
cv=2,
clf=clf,
scorer=scorer,
predict_mode=predict_mode)
gat.fit(epochs, y=y_)
gat.score(epochs, y=y_)
# Check that scorer is correctly defined manually and
# automatically.
scorer_name = gat.scorer_.__name__
if scorer is None:
if is_classifier(clf):
assert_equal(scorer_name, 'accuracy_score')
else:
assert_equal(scorer_name, 'mean_squared_error')
else:
assert_equal(scorer_name, scorer.__name__)
def test_generalization_across_time():
"""Test time generalization decoding
"""
from sklearn.svm import SVC
from sklearn.linear_model import RANSACRegressor, LinearRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import mean_squared_error
from sklearn.cross_validation import LeaveOneLabelOut
epochs = make_epochs()
# Test default running
gat = GeneralizationAcrossTime(picks='foo')
assert_equal("<GAT | no fit, no prediction, no score>", "%s" % gat)
assert_raises(ValueError, gat.fit, epochs)
with warnings.catch_warnings(record=True):
# check classic fit + check manual picks
gat.picks = [0]
gat.fit(epochs)
# check optional y as array
gat.picks = None
gat.fit(epochs, y=epochs.events[:, 2])
# check optional y as list
gat.fit(epochs, y=epochs.events[:, 2].tolist())
assert_equal(len(gat.picks_), len(gat.ch_names), 1)
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), no "
"prediction, no score>", '%s' % gat)
assert_equal(gat.ch_names, epochs.ch_names)
gat.predict(epochs)
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs, no score>",
"%s" % gat)
gat.score(epochs)
gat.score(epochs, y=epochs.events[:, 2])
gat.score(epochs, y=epochs.events[:, 2].tolist())
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs,\n scored "
"(accuracy_score)>", "%s" % gat)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=epochs.events[:, 2])
old_mode = gat.predict_mode
gat.predict_mode = 'super-foo-mode'
assert_raises(ValueError, gat.predict, epochs)
gat.predict_mode = old_mode
gat.score(epochs, y=epochs.events[:, 2])
assert_true("accuracy_score" in '%s' % gat.scorer_)
epochs2 = epochs.copy()
# check _DecodingTime class
assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.050 (s), length: 0.050 (s), n_time_windows: 15>",
"%s" % gat.train_times_)
assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.050 (s), length: 0.050 (s), n_time_windows: 15 x 15>",
"%s" % gat.test_times_)
# the y-check
gat.predict_mode = 'mean-prediction'
epochs2.events[:, 2] += 10
gat_ = copy.deepcopy(gat)
assert_raises(ValueError, gat_.score, epochs2)
gat.predict_mode = 'cross-validation'
# Test basics
# --- number of trials
assert_true(gat.y_train_.shape[0] ==
gat.y_true_.shape[0] ==
len(gat.y_pred_[0][0]) == 14)
# --- number of folds
assert_true(np.shape(gat.estimators_)[1] == gat.cv)
# --- length training size
assert_true(len(gat.train_times_['slices']) == 15 ==
np.shape(gat.estimators_)[0])
# --- length testing sizes
assert_true(len(gat.test_times_['slices']) == 15 ==
np.shape(gat.scores_)[0])
assert_true(len(gat.test_times_['slices'][0]) == 15 ==
np.shape(gat.scores_)[1])
# Test longer time window
gat = GeneralizationAcrossTime(train_times={'length': .100})
with warnings.catch_warnings(record=True):
gat2 = gat.fit(epochs)
assert_true(gat is gat2) # return self
assert_true(hasattr(gat2, 'cv_'))
assert_true(gat2.cv_ != gat.cv)
scores = gat.score(epochs)
assert_true(isinstance(scores, list)) # type check
assert_equal(len(scores[0]), len(scores)) # shape check
assert_equal(len(gat.test_times_['slices'][0][0]), 2)
# Decim training steps
gat = GeneralizationAcrossTime(train_times={'step': .100})
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_true(len(gat.scores_) == len(gat.estimators_) == 8) # training time
assert_equal(len(gat.scores_[0]), 15) # testing time
# Test start stop training & test cv without n_fold params
y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1))
gat = GeneralizationAcrossTime(cv=LeaveOneLabelOut(y_4classes),
train_times={'start': 0.090, 'stop': 0.250})
# predict without fit
assert_raises(RuntimeError, gat.predict, epochs)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=y_4classes)
gat.score(epochs)
assert_equal(len(gat.scores_), 4)
assert_equal(gat.train_times_['times'][0], epochs.times[6])
assert_equal(gat.train_times_['times'][-1], epochs.times[9])
# Test score without passing epochs & Test diagonal decoding
gat = GeneralizationAcrossTime(test_times='diagonal')
with warnings.catch_warnings(record=True):
gat.fit(epochs)
assert_raises(RuntimeError, gat.score)
gat.predict(epochs)
scores = gat.score()
assert_true(scores is gat.scores_)
assert_equal(np.shape(gat.scores_), (15, 1))
assert_array_equal([tim for ttime in gat.test_times_['times']
for tim in ttime], gat.train_times_['times'])
# Test generalization across conditions
gat = GeneralizationAcrossTime(predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs[0:6])
gat.predict(epochs[7:])
gat.score(epochs[7:])
# Test training time parameters
gat_ = copy.deepcopy(gat)
# --- start stop outside time range
gat_.train_times = dict(start=-999.)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(start=999.)
assert_raises(ValueError, gat_.fit, epochs)
# --- impossible slices
gat_.train_times = dict(step=.000001)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(length=.000001)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(length=999.)
assert_raises(ValueError, gat_.fit, epochs)
# Test testing time parameters
# --- outside time range
gat.test_times = dict(start=-999.)
assert_raises(ValueError, gat.predict, epochs)
gat.test_times = dict(start=999.)
assert_raises(ValueError, gat.predict, epochs)
# --- impossible slices
gat.test_times = dict(step=.000001)
assert_raises(ValueError, gat.predict, epochs)
gat_ = copy.deepcopy(gat)
gat_.train_times_['length'] = .000001
gat_.test_times = dict(length=.000001)
assert_raises(ValueError, gat_.predict, epochs)
# --- test time region of interest
gat.test_times = dict(step=.150)
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 5, 14, 1))
# --- silly value
gat.test_times = 'foo'
assert_raises(ValueError, gat.predict, epochs)
assert_raises(RuntimeError, gat.score)
# --- unmatched length between training and testing time
gat.test_times = dict(length=.150)
assert_raises(ValueError, gat.predict, epochs)
svc = SVC(C=1, kernel='linear', probability=True)
gat = GeneralizationAcrossTime(clf=svc, predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs)
# sklearn needs it: c.f.
# https://github.com/scikit-learn/scikit-learn/issues/2723
# and http://bit.ly/1u7t8UT
assert_raises(ValueError, gat.score, epochs2)
gat.score(epochs)
scores = sum(scores, []) # flatten
assert_true(0.0 <= np.min(scores) <= 1.0)
assert_true(0.0 <= np.max(scores) <= 1.0)
# Test that gets error if train on one dataset, test on another, and don't
# specify appropriate cv:
gat = GeneralizationAcrossTime()
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
assert_raises(IndexError, gat.predict, epochs[:10])
# TODO JRK: test GAT with non-exhaustive CV (eg. train on 80%, test on 10%)
# Check that still works with classifier that output y_pred with
# shape = (n_trials, 1) instead of (n_trials,)
gat = GeneralizationAcrossTime(clf=RANSACRegressor(LinearRegression()),
cv=2)
epochs.crop(None, epochs.times[2])
gat.fit(epochs)
gat.predict(epochs)
# Test combinations of complex scenarios
# 2 or more distinct classes
n_classes = [2, 4] # 4 tested
# nicely ordered labels or not
le = LabelEncoder()
y = le.fit_transform(epochs.events[:, 2])
y[len(y) // 2:] += 2
ys = (y, y + 1000)
# Univariate and multivariate prediction
svc = SVC(C=1, kernel='linear')
class SVC_proba(SVC):
def predict(self, x):
probas = super(SVC_proba, self).predict_proba(x)
return probas[:, 0]
svcp = SVC_proba(C=1, kernel='linear', probability=True)
clfs = [svc, svcp]
scorers = [None, mean_squared_error]
# Test all combinations
for clf, scorer in zip(clfs, scorers):
for y in ys:
for n_class in n_classes:
y_ = y % n_class
with warnings.catch_warnings(record=True):
gat = GeneralizationAcrossTime(cv=2, clf=clf,
scorer=scorer)
gat.fit(epochs, y=y_)
gat.score(epochs, y=y_)
def test_generalization_across_time():
"""Test time generalization decoding
"""
from sklearn.svm import SVC
from sklearn.linear_model import RANSACRegressor, LinearRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import mean_squared_error
from sklearn.cross_validation import LeaveOneLabelOut
epochs = make_epochs()
# Test default running
gat = GeneralizationAcrossTime(picks='foo')
assert_equal("<GAT | no fit, no prediction, no score>", "%s" % gat)
assert_raises(ValueError, gat.fit, epochs)
with warnings.catch_warnings(record=True):
# check classic fit + check manual picks
gat.picks = [0]
gat.fit(epochs)
# check optional y as array
gat.picks = None
gat.fit(epochs, y=epochs.events[:, 2])
# check optional y as list
gat.fit(epochs, y=epochs.events[:, 2].tolist())
assert_equal(len(gat.picks_), len(gat.ch_names), 1)
assert_equal(
"<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), no "
"prediction, no score>", '%s' % gat)
assert_equal(gat.ch_names, epochs.ch_names)
gat.predict(epochs)
assert_equal(
"<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs, no score>", "%s" % gat)
gat.score(epochs)
gat.score(epochs, y=epochs.events[:, 2])
gat.score(epochs, y=epochs.events[:, 2].tolist())
assert_equal(
"<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predicted 14 epochs,\n scored "
"(accuracy_score)>", "%s" % gat)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=epochs.events[:, 2])
old_mode = gat.predict_mode
gat.predict_mode = 'super-foo-mode'
assert_raises(ValueError, gat.predict, epochs)
gat.predict_mode = old_mode
gat.score(epochs, y=epochs.events[:, 2])
assert_true("accuracy_score" in '%s' % gat.scorer_)
epochs2 = epochs.copy()
# check _DecodingTime class
assert_equal(
"<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.047 (s), length: 0.047 (s), n_time_windows: 15>",
"%s" % gat.train_times_)
assert_equal(
"<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.047 (s), length: 0.047 (s), n_time_windows: 15 x 15>",
"%s" % gat.test_times_)
# the y-check
gat.predict_mode = 'mean-prediction'
epochs2.events[:, 2] += 10
gat_ = copy.deepcopy(gat)
assert_raises(ValueError, gat_.score, epochs2)
gat.predict_mode = 'cross-validation'
# Test basics
# --- number of trials
assert_true(gat.y_train_.shape[0] == gat.y_true_.shape[0] == len(
gat.y_pred_[0][0]) == 14)
# --- number of folds
assert_true(np.shape(gat.estimators_)[1] == gat.cv)
# --- length training size
assert_true(
len(gat.train_times_['slices']) == 15 == np.shape(gat.estimators_)[0])
# --- length testing sizes
assert_true(
len(gat.test_times_['slices']) == 15 == np.shape(gat.scores_)[0])
assert_true(
len(gat.test_times_['slices'][0]) == 15 == np.shape(gat.scores_)[1])
# Test longer time window
gat = GeneralizationAcrossTime(train_times={'length': .100})
with warnings.catch_warnings(record=True):
gat2 = gat.fit(epochs)
assert_true(gat is gat2) # return self
assert_true(hasattr(gat2, 'cv_'))
assert_true(gat2.cv_ != gat.cv)
scores = gat.score(epochs)
assert_true(isinstance(scores, list)) # type check
assert_equal(len(scores[0]), len(scores)) # shape check
assert_equal(len(gat.test_times_['slices'][0][0]), 2)
# Decim training steps
gat = GeneralizationAcrossTime(train_times={'step': .100})
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_true(len(gat.scores_) == len(gat.estimators_) == 8) # training time
assert_equal(len(gat.scores_[0]), 15) # testing time
# Test start stop training & test cv without n_fold params
y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1))
gat = GeneralizationAcrossTime(cv=LeaveOneLabelOut(y_4classes),
train_times={
'start': 0.090,
'stop': 0.250
})
# predict without fit
assert_raises(RuntimeError, gat.predict, epochs)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=y_4classes)
gat.score(epochs)
assert_equal(len(gat.scores_), 4)
assert_equal(gat.train_times_['times'][0], epochs.times[6])
assert_equal(gat.train_times_['times'][-1], epochs.times[9])
# Test score without passing epochs & Test diagonal decoding
gat = GeneralizationAcrossTime(test_times='diagonal')
with warnings.catch_warnings(record=True):
gat.fit(epochs)
assert_raises(RuntimeError, gat.score)
gat.predict(epochs)
scores = gat.score()
assert_true(scores is gat.scores_)
assert_equal(np.shape(gat.scores_), (15, 1))
assert_array_equal(
[tim for ttime in gat.test_times_['times'] for tim in ttime],
gat.train_times_['times'])
# Test generalization across conditions
gat = GeneralizationAcrossTime(predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs[0:6])
gat.predict(epochs[7:])
gat.score(epochs[7:])
# Test training time parameters
gat_ = copy.deepcopy(gat)
# --- start stop outside time range
gat_.train_times = dict(start=-999.)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(start=999.)
assert_raises(ValueError, gat_.fit, epochs)
# --- impossible slices
gat_.train_times = dict(step=.000001)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(length=.000001)
assert_raises(ValueError, gat_.fit, epochs)
gat_.train_times = dict(length=999.)
assert_raises(ValueError, gat_.fit, epochs)
# Test testing time parameters
# --- outside time range
gat.test_times = dict(start=-999.)
assert_raises(ValueError, gat.predict, epochs)
gat.test_times = dict(start=999.)
assert_raises(ValueError, gat.predict, epochs)
# --- impossible slices
gat.test_times = dict(step=.000001)
assert_raises(ValueError, gat.predict, epochs)
gat_ = copy.deepcopy(gat)
gat_.train_times_['length'] = .000001
gat_.test_times = dict(length=.000001)
assert_raises(ValueError, gat_.predict, epochs)
# --- test time region of interest
gat.test_times = dict(step=.150)
gat.predict(epochs)
assert_array_equal(np.shape(gat.y_pred_), (15, 5, 14, 1))
# --- silly value
gat.test_times = 'foo'
assert_raises(ValueError, gat.predict, epochs)
assert_raises(RuntimeError, gat.score)
# --- unmatched length between training and testing time
gat.test_times = dict(length=.150)
assert_raises(ValueError, gat.predict, epochs)
svc = SVC(C=1, kernel='linear', probability=True)
gat = GeneralizationAcrossTime(clf=svc, predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs)
# sklearn needs it: c.f.
# https://github.com/scikit-learn/scikit-learn/issues/2723
# and http://bit.ly/1u7t8UT
assert_raises(ValueError, gat.score, epochs2)
gat.score(epochs)
scores = sum(scores, []) # flatten
assert_true(0.0 <= np.min(scores) <= 1.0)
assert_true(0.0 <= np.max(scores) <= 1.0)
# Test that gets error if train on one dataset, test on another, and don't
# specify appropriate cv:
gat = GeneralizationAcrossTime()
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
assert_raises(ValueError, gat.predict, epochs[:10])
# Check that still works with classifier that output y_pred with
# shape = (n_trials, 1) instead of (n_trials,)
gat = GeneralizationAcrossTime(clf=RANSACRegressor(LinearRegression()),
cv=2)
epochs.crop(None, epochs.times[2])
gat.fit(epochs)
gat.predict(epochs)
# Test combinations of complex scenarios
# 2 or more distinct classes
n_classes = [2, 4] # 4 tested
# nicely ordered labels or not
le = LabelEncoder()
y = le.fit_transform(epochs.events[:, 2])
y[len(y) // 2:] += 2
ys = (y, y + 1000)
# Univariate and multivariate prediction
svc = SVC(C=1, kernel='linear')
class SVC_proba(SVC):
def predict(self, x):
probas = super(SVC_proba, self).predict_proba(x)
return probas[:, 0]
svcp = SVC_proba(C=1, kernel='linear', probability=True)
clfs = [svc, svcp]
scorers = [None, mean_squared_error]
# Test all combinations
for clf, scorer in zip(clfs, scorers):
for y in ys:
for n_class in n_classes:
y_ = y % n_class
with warnings.catch_warnings(record=True):
gat = GeneralizationAcrossTime(cv=2,
clf=clf,
scorer=scorer)
gat.fit(epochs, y=y_)
gat.score(epochs, y=y_)
def test_generalization_across_time():
"""Test time generalization decoding
"""
from sklearn.svm import SVC
raw = io.Raw(raw_fname, preload=False)
events = read_events(event_name)
picks = pick_types(raw.info, meg='mag', stim=False, ecg=False,
eog=False, exclude='bads')
picks = picks[0:2]
decim = 30
# Test on time generalization within one condition
with warnings.catch_warnings(record=True):
epochs = Epochs(raw, events, event_id, tmin, tmax, picks=picks,
baseline=(None, 0), preload=True, decim=decim)
# Test default running
gat = GeneralizationAcrossTime()
assert_equal("<GAT | no fit, no prediction, no score>", "%s" % gat)
with warnings.catch_warnings(record=True):
gat.fit(epochs)
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), no "
"prediction, no score>", '%s' % gat)
gat.predict(epochs)
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predict_type : 'predict' on 15 epochs, no score>",
"%s" % gat)
gat.score(epochs)
assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predict_type : 'predict' on 15 epochs,\n scored "
"(accuracy_score)>", "%s" % gat)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=epochs.events[:, 2])
old_type = gat.predict_type
gat.predict_type = 'foo'
assert_raises(ValueError, gat.predict, epochs)
gat.predict_type = old_type
old_mode = gat.predict_mode
gat.predict_mode = 'super-foo-mode'
assert_raises(ValueError, gat.predict, epochs)
gat.predict_mode = old_mode
gat.score(epochs, y=epochs.events[:, 2])
assert_true("accuracy_score" in '%s' % gat.scorer_)
epochs2 = epochs.copy()
# check _DecodingTime class
assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.047 (s), length: 0.047 (s), n_time_windows: 15>",
"%s" % gat.train_times)
assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.047 (s), length: 0.047 (s), n_time_windows: 15 x 15>",
"%s" % gat.test_times_)
# the y-check
gat.predict_mode = 'mean-prediction'
epochs2.events[:, 2] += 10
assert_raises(ValueError, gat.score, epochs2)
gat.predict_mode = 'cross-validation'
# Test basics
# --- number of trials
assert_true(gat.y_train_.shape[0] ==
gat.y_true_.shape[0] ==
gat.y_pred_.shape[2] == 14)
# --- number of folds
assert_true(np.shape(gat.estimators_)[1] == gat.cv)
# --- length training size
assert_true(len(gat.train_times['slices']) == 15 ==
np.shape(gat.estimators_)[0])
# --- length testing sizes
assert_true(len(gat.test_times_['slices']) == 15 ==
np.shape(gat.scores_)[0])
assert_true(len(gat.test_times_['slices'][0]) == 15 ==
np.shape(gat.scores_)[1])
# Test longer time window
gat = GeneralizationAcrossTime(train_times={'length': .100})
with warnings.catch_warnings(record=True):
gat2 = gat.fit(epochs)
assert_true(gat is gat2) # return self
assert_true(hasattr(gat2, 'cv_'))
assert_true(gat2.cv_ != gat.cv)
scores = gat.score(epochs)
assert_true(isinstance(scores, list)) # type check
assert_equal(len(scores[0]), len(scores)) # shape check
assert_equal(len(gat.test_times_['slices'][0][0]), 2)
# Decim training steps
gat = GeneralizationAcrossTime(train_times={'step': .100})
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_equal(len(gat.scores_), 8)
# Test start stop training
gat = GeneralizationAcrossTime(train_times={'start': 0.090,
'stop': 0.250})
# predict without fit
assert_raises(RuntimeError, gat.predict, epochs)
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_equal(len(gat.scores_), 4)
assert_equal(gat.train_times['times_'][0], epochs.times[6])
assert_equal(gat.train_times['times_'][-1], epochs.times[9])
# Test score without passing epochs
gat = GeneralizationAcrossTime()
with warnings.catch_warnings(record=True):
gat.fit(epochs)
assert_raises(RuntimeError, gat.score)
gat.predict(epochs, test_times='diagonal') # Test diagonal decoding
scores = gat.score()
assert_true(scores is gat.scores_)
assert_equal(np.shape(gat.scores_), (15, 1))
# Test generalization across conditions
gat = GeneralizationAcrossTime(predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs[0:6])
gat.predict(epochs[7:])
assert_raises(ValueError, gat.predict, epochs, test_times='hahahaha')
assert_raises(RuntimeError, gat.score)
gat.score(epochs[7:])
svc = SVC(C=1, kernel='linear', probability=True)
gat = GeneralizationAcrossTime(clf=svc, predict_type='predict_proba',
predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs)
# sklearn needs it: c.f.
# https://github.com/scikit-learn/scikit-learn/issues/2723
# and http://bit.ly/1u7t8UT
assert_raises(ValueError, gat.score, epochs2)
gat.score(epochs)
scores = sum(scores, []) # flatten
assert_true(0.0 <= np.min(scores) <= 1.0)
assert_true(0.0 <= np.max(scores) <= 1.0)
# test various predict_type
gat = GeneralizationAcrossTime(clf=svc, predict_type="predict_proba")
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
# check that 2 class probabilistic estimates are [p, 1-p]
assert_true(gat.y_pred_.shape[3] == 2)
gat.score(epochs)
# check that continuous prediction leads to AUC rather than accuracy
assert_true("roc_auc_score" in '%s' % gat.scorer_)
gat = GeneralizationAcrossTime(predict_type="decision_function")
# XXX Sklearn doesn't like non-binary inputs. We could binarize the data,
# or change Sklearn default behavior
epochs.events[:, 2][epochs.events[:, 2] == 3] = 0
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
# check that 2 class non-probabilistic continuous estimates are [distance]
assert_true(gat.y_pred_.shape[3] == 1)
gat.score(epochs)
# check that continuous prediction leads to AUC rather than accuracy
assert_true("roc_auc_score" in '%s' % gat.scorer_)
# Test that gets error if train on one dataset, test on another, and don't
# specify appropriate cv:
gat = GeneralizationAcrossTime()
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
assert_raises(ValueError, gat.predict, epochs[:10])
# Test combinations of complex scenarios
# 2 or more distinct classes
n_classes = [2] # 4 tested
# nicely ordered labels or not
y = epochs.events[:, 2]
y[len(y) // 2:] += 2
ys = (y, y + 1000)
# Classifier and regressor
svc = SVC(C=1, kernel='linear', probability=True)
clfs = [svc] # SVR tested
# Continuous, and probabilistic estimate
predict_types = ['predict_proba', 'decision_function']
# Test all combinations
for clf_n, clf in enumerate(clfs):
for y in ys:
for n_class in n_classes:
for pt in predict_types:
y_ = y % n_class
with warnings.catch_warnings(record=True):
gat = GeneralizationAcrossTime(
cv=2, clf=clf, predict_type=pt)
gat.fit(epochs, y=y_)
gat.score(epochs, y=y_)
stim_list_rsvp[itrl,istim] = 3 elif 'body' in rsvpfile: stim_list_rsvp[itrl,istim] = 4 # get lag indices lag = mat['output']['target'][0][0][0][0][0][0] R1 = mat['output']['response'][0][0][0][0][3][0][0][0][0] R2 = mat['output']['response'][0][0][0][0][4][0][0][0] ulag = np.unique(lag) n_lag = len(ulag) # offset = [-1, 0, 1] # decoding epochs = epochs.decimate(decim) n_te = len(epochs.times) gat.predict_mode = 'mean-prediction' # generalization across conditions so no cross validation here. # compute scores for each stimulus rsvp_score_all = np.zeros((n_tr, n_te, n_stim)) rsvp_score_lag = np.zeros((n_tr, n_te, n_stim, n_lag)) rsvp_score_offset0_lag = np.zeros((n_tr, n_te, n_stim, n_lag)) rsvp_score_offset_1_lag = np.zeros((n_tr, n_te, n_stim, n_lag)) rsvp_score_offset1_lag = np.zeros((n_tr, n_te, n_stim, n_lag)) for istim in range(n_stim): # compute scores across all trials s = gat.score(epochs, stim_list_rsvp[:,istim]) # compute predictions as well. s = np.array(s) rsvp_score_all[:,:,istim] = s # compute scores for lags for ilag in range(len(ulag)):
def test_generalization_across_time():
"""Test time generalization decoding
"""
from sklearn.svm import SVC
raw = io.Raw(raw_fname, preload=False)
events = read_events(event_name)
picks = pick_types(raw.info,
meg='mag',
stim=False,
ecg=False,
eog=False,
exclude='bads')
picks = picks[0:2]
decim = 30
# Test on time generalization within one condition
with warnings.catch_warnings(record=True):
epochs = Epochs(raw,
events,
event_id,
tmin,
tmax,
picks=picks,
baseline=(None, 0),
preload=True,
decim=decim)
# Test default running
gat = GeneralizationAcrossTime()
assert_equal("<GAT | no fit, no prediction, no score>", "%s" % gat)
with warnings.catch_warnings(record=True):
gat.fit(epochs)
assert_equal(
"<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), no "
"prediction, no score>", '%s' % gat)
gat.predict(epochs)
assert_equal(
"<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predict_type : 'predict' on 15 epochs, no score>", "%s" % gat)
gat.score(epochs)
assert_equal(
"<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), "
"predict_type : 'predict' on 15 epochs,\n scored "
"(accuracy_score)>", "%s" % gat)
with warnings.catch_warnings(record=True):
gat.fit(epochs, y=epochs.events[:, 2])
old_type = gat.predict_type
gat.predict_type = 'foo'
assert_raises(ValueError, gat.predict, epochs)
gat.predict_type = old_type
old_mode = gat.predict_mode
gat.predict_mode = 'super-foo-mode'
assert_raises(ValueError, gat.predict, epochs)
gat.predict_mode = old_mode
gat.score(epochs, y=epochs.events[:, 2])
assert_true("accuracy_score" in '%s' % gat.scorer_)
epochs2 = epochs.copy()
# check _DecodingTime class
assert_equal(
"<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.047 (s), length: 0.047 (s), n_time_windows: 15>",
"%s" % gat.train_times)
assert_equal(
"<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: "
"0.047 (s), length: 0.047 (s), n_time_windows: 15 x 15>",
"%s" % gat.test_times_)
# the y-check
gat.predict_mode = 'mean-prediction'
epochs2.events[:, 2] += 10
assert_raises(ValueError, gat.score, epochs2)
gat.predict_mode = 'cross-validation'
# Test basics
# --- number of trials
assert_true(gat.y_train_.shape[0] == gat.y_true_.shape[0] ==
gat.y_pred_.shape[2] == 14)
# --- number of folds
assert_true(np.shape(gat.estimators_)[1] == gat.cv)
# --- length training size
assert_true(
len(gat.train_times['slices']) == 15 == np.shape(gat.estimators_)[0])
# --- length testing sizes
assert_true(
len(gat.test_times_['slices']) == 15 == np.shape(gat.scores_)[0])
assert_true(
len(gat.test_times_['slices'][0]) == 15 == np.shape(gat.scores_)[1])
# Test longer time window
gat = GeneralizationAcrossTime(train_times={'length': .100})
with warnings.catch_warnings(record=True):
gat2 = gat.fit(epochs)
assert_true(gat is gat2) # return self
assert_true(hasattr(gat2, 'cv_'))
assert_true(gat2.cv_ != gat.cv)
scores = gat.score(epochs)
assert_true(isinstance(scores, list)) # type check
assert_equal(len(scores[0]), len(scores)) # shape check
assert_equal(len(gat.test_times_['slices'][0][0]), 2)
# Decim training steps
gat = GeneralizationAcrossTime(train_times={'step': .100})
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_equal(len(gat.scores_), 8)
# Test start stop training
gat = GeneralizationAcrossTime(train_times={'start': 0.090, 'stop': 0.250})
# predict without fit
assert_raises(RuntimeError, gat.predict, epochs)
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.score(epochs)
assert_equal(len(gat.scores_), 4)
assert_equal(gat.train_times['times_'][0], epochs.times[6])
assert_equal(gat.train_times['times_'][-1], epochs.times[9])
# Test diagonal decoding
gat = GeneralizationAcrossTime()
with warnings.catch_warnings(record=True):
gat.fit(epochs)
scores = gat.score(epochs, test_times='diagonal')
assert_true(scores is gat.scores_)
assert_equal(np.shape(gat.scores_), (15, 1))
# Test generalization across conditions
gat = GeneralizationAcrossTime(predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs[0:6])
gat.predict(epochs[7:])
assert_raises(ValueError, gat.predict, epochs, test_times='hahahaha')
gat.score(epochs[7:])
svc = SVC(C=1, kernel='linear', probability=True)
gat = GeneralizationAcrossTime(clf=svc,
predict_type='predict_proba',
predict_mode='mean-prediction')
with warnings.catch_warnings(record=True):
gat.fit(epochs)
# sklearn needs it: c.f.
# https://github.com/scikit-learn/scikit-learn/issues/2723
# and http://bit.ly/1u7t8UT
assert_raises(ValueError, gat.score, epochs2)
gat.score(epochs)
scores = sum(scores, []) # flatten
assert_true(0.0 <= np.min(scores) <= 1.0)
assert_true(0.0 <= np.max(scores) <= 1.0)
# test various predict_type
gat = GeneralizationAcrossTime(clf=svc, predict_type="predict_proba")
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
# check that 2 class probabilistic estimates are [p, 1-p]
assert_true(gat.y_pred_.shape[3] == 2)
gat.score(epochs)
# check that continuous prediction leads to AUC rather than accuracy
assert_true("roc_auc_score" in '%s' % gat.scorer_)
gat = GeneralizationAcrossTime(predict_type="decision_function")
# XXX Sklearn doesn't like non-binary inputs. We could binarize the data,
# or change Sklearn default behavior
epochs.events[:, 2][epochs.events[:, 2] == 3] = 0
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
# check that 2 class non-probabilistic continuous estimates are [distance]
assert_true(gat.y_pred_.shape[3] == 1)
gat.score(epochs)
# check that continuous prediction leads to AUC rather than accuracy
assert_true("roc_auc_score" in '%s' % gat.scorer_)
# Test that gets error if train on one dataset, test on another, and don't
# specify appropriate cv:
gat = GeneralizationAcrossTime()
with warnings.catch_warnings(record=True):
gat.fit(epochs)
gat.predict(epochs)
assert_raises(ValueError, gat.predict, epochs[:10])
# Test combinations of complex scenarios
# 2 or more distinct classes
n_classes = [2] # 4 tested
# nicely ordered labels or not
y = epochs.events[:, 2]
y[len(y) // 2:] += 2
ys = (y, y + 1000)
# Classifier and regressor
svc = SVC(C=1, kernel='linear', probability=True)
clfs = [svc] # SVR tested
# Continuous, and probabilistic estimate
predict_types = ['predict_proba', 'decision_function']
# Test all combinations
for clf_n, clf in enumerate(clfs):
for y in ys:
for n_class in n_classes:
for pt in predict_types:
y_ = y % n_class
with warnings.catch_warnings(record=True):
gat = GeneralizationAcrossTime(cv=2,
clf=clf,
predict_type=pt)
gat.fit(epochs, y=y_)
gat.score(epochs, y=y_)
