def test_cross_val_multiscore(): """Test cross_val_multiscore for computing scores on decoding over time.""" from sklearn.model_selection import KFold, StratifiedKFold, cross_val_score from sklearn.linear_model import LogisticRegression, LinearRegression # compare to cross-val-score X = np.random.rand(20, 3) y = np.arange(20) % 2 clf = LogisticRegression() cv = KFold(2, random_state=0) assert_array_equal(cross_val_score(clf, X, y, cv=cv), cross_val_multiscore(clf, X, y, cv=cv)) # Test with search light X = np.random.rand(20, 4, 3) y = np.arange(20) % 2 clf = SlidingEstimator(LogisticRegression(), scoring='accuracy') scores_acc = cross_val_multiscore(clf, X, y, cv=cv) assert_array_equal(np.shape(scores_acc), [2, 3]) # check values scores_acc_manual = list() for train, test in cv.split(X, y): clf.fit(X[train], y[train]) scores_acc_manual.append(clf.score(X[test], y[test])) assert_array_equal(scores_acc, scores_acc_manual) # check scoring metric # raise an error if scoring is defined at cross-val-score level and # search light, because search light does not return a 1-dimensional # prediction. assert_raises(ValueError, cross_val_multiscore, clf, X, y, cv=cv, scoring='roc_auc') clf = SlidingEstimator(LogisticRegression(), scoring='roc_auc') scores_auc = cross_val_multiscore(clf, X, y, cv=cv, n_jobs=1) scores_auc_manual = list() for train, test in cv.split(X, y): clf.fit(X[train], y[train]) scores_auc_manual.append(clf.score(X[test], y[test])) assert_array_equal(scores_auc, scores_auc_manual) # indirectly test that cross_val_multiscore rightly detects the type of # estimator and generates a StratifiedKFold for classiers and a KFold # otherwise X = np.random.randn(1000, 3) y = np.r_[np.zeros(500), np.ones(500)] clf = LogisticRegression(random_state=0) reg = LinearRegression() for cross_val in (cross_val_score, cross_val_multiscore): manual = cross_val(clf, X, y, cv=StratifiedKFold(2)) auto = cross_val(clf, X, y, cv=2) assert_array_equal(manual, auto) assert_raises(ValueError, cross_val, clf, X, y, cv=KFold(2)) manual = cross_val(reg, X, y, cv=KFold(2)) auto = cross_val(reg, X, y, cv=2) assert_array_equal(manual, auto)
def test_get_coef_multiclass_full(n_classes, n_channels, n_times): """Test a full example with pattern extraction.""" from sklearn.pipeline import make_pipeline from sklearn.linear_model import LogisticRegression from sklearn.model_selection import StratifiedKFold data = np.zeros((10 * n_classes, n_channels, n_times)) # Make only the first channel informative for ii in range(n_classes): data[ii * 10:(ii + 1) * 10, 0] = ii events = np.zeros((len(data), 3), int) events[:, 0] = np.arange(len(events)) events[:, 2] = data[:, 0, 0] info = create_info(n_channels, 1000., 'eeg') epochs = EpochsArray(data, info, events, tmin=0) clf = make_pipeline( Scaler(epochs.info), Vectorizer(), LinearModel(LogisticRegression(random_state=0, multi_class='ovr')), ) scorer = 'roc_auc_ovr_weighted' time_gen = GeneralizingEstimator(clf, scorer, verbose=True) X = epochs.get_data() y = epochs.events[:, 2] n_splits = 3 cv = StratifiedKFold(n_splits=n_splits) scores = cross_val_multiscore(time_gen, X, y, cv=cv, verbose=True) want = (n_splits, ) if n_times > 1: want += (n_times, n_times) assert scores.shape == want assert_array_less(0.8, scores) clf.fit(X, y) patterns = get_coef(clf, 'patterns_', inverse_transform=True) assert patterns.shape == (n_classes, n_channels, n_times) assert_allclose(patterns[:, 1:], 0., atol=1e-7) # no other channels useful
def test_cross_val_multiscore(): """Test cross_val_multiscore for computing scores on decoding over time. """ from sklearn.model_selection import KFold, cross_val_score from sklearn.linear_model import LogisticRegression # compare to cross-val-score X = np.random.rand(20, 3) y = np.arange(20) % 2 clf = LogisticRegression() cv = KFold(2, random_state=0) assert_array_equal(cross_val_score(clf, X, y, cv=cv), cross_val_multiscore(clf, X, y, cv=cv)) # Test with search light X = np.random.rand(20, 4, 3) y = np.arange(20) % 2 clf = SlidingEstimator(LogisticRegression(), scoring='accuracy') scores_acc = cross_val_multiscore(clf, X, y, cv=cv) assert_array_equal(np.shape(scores_acc), [2, 3]) # check values scores_acc_manual = list() for train, test in cv.split(X, y): clf.fit(X[train], y[train]) scores_acc_manual.append(clf.score(X[test], y[test])) assert_array_equal(scores_acc, scores_acc_manual) # check scoring metric # raise an error if scoring is defined at cross-val-score level and # search light, because search light does not return a 1-dimensional # prediction. assert_raises(ValueError, cross_val_multiscore, clf, X, y, cv=cv, scoring='roc_auc') clf = SlidingEstimator(LogisticRegression(), scoring='roc_auc') scores_auc = cross_val_multiscore(clf, X, y, cv=cv, n_jobs=1) scores_auc_manual = list() for train, test in cv.split(X, y): clf.fit(X[train], y[train]) scores_auc_manual.append(clf.score(X[test], y[test])) assert_array_equal(scores_auc, scores_auc_manual)
def test_cross_val_multiscore(): """Test cross_val_multiscore for computing scores on decoding over time.""" from sklearn.model_selection import KFold, StratifiedKFold, cross_val_score from sklearn.linear_model import LogisticRegression, LinearRegression if check_version('sklearn', '0.20'): logreg = LogisticRegression(solver='liblinear', random_state=0) else: logreg = LogisticRegression(random_state=0) # compare to cross-val-score X = np.random.rand(20, 3) y = np.arange(20) % 2 cv = KFold(2, random_state=0, shuffle=True) clf = logreg assert_array_equal(cross_val_score(clf, X, y, cv=cv), cross_val_multiscore(clf, X, y, cv=cv)) # Test with search light X = np.random.rand(20, 4, 3) y = np.arange(20) % 2 clf = SlidingEstimator(logreg, scoring='accuracy') scores_acc = cross_val_multiscore(clf, X, y, cv=cv) assert_array_equal(np.shape(scores_acc), [2, 3]) # check values scores_acc_manual = list() for train, test in cv.split(X, y): clf.fit(X[train], y[train]) scores_acc_manual.append(clf.score(X[test], y[test])) assert_array_equal(scores_acc, scores_acc_manual) # check scoring metric # raise an error if scoring is defined at cross-val-score level and # search light, because search light does not return a 1-dimensional # prediction. pytest.raises(ValueError, cross_val_multiscore, clf, X, y, cv=cv, scoring='roc_auc') clf = SlidingEstimator(logreg, scoring='roc_auc') scores_auc = cross_val_multiscore(clf, X, y, cv=cv, n_jobs=1) scores_auc_manual = list() for train, test in cv.split(X, y): clf.fit(X[train], y[train]) scores_auc_manual.append(clf.score(X[test], y[test])) assert_array_equal(scores_auc, scores_auc_manual) # indirectly test that cross_val_multiscore rightly detects the type of # estimator and generates a StratifiedKFold for classiers and a KFold # otherwise X = np.random.randn(1000, 3) y = np.ones(1000, dtype=int) y[::2] = 0 clf = logreg reg = LinearRegression() for cross_val in (cross_val_score, cross_val_multiscore): manual = cross_val(clf, X, y, cv=StratifiedKFold(2)) auto = cross_val(clf, X, y, cv=2) assert_array_equal(manual, auto) manual = cross_val(reg, X, y, cv=KFold(2)) auto = cross_val(reg, X, y, cv=2) assert_array_equal(manual, auto)
path, subjects[:1], #os.path.join(path, 'subjects.csv'), 'meditation_permut1.conf', 'fmri', prepro=MonksPreprocessingPipeline(), roi_labels=atlas_dict) clf = make_pipeline(StandardScaler(), LinearSVC(C=1)) time_gen = GeneralizingEstimator(clf, scoring='accuracy', n_jobs=20) ds = SampleSlicer({'group': ['E']}).transform(ds) scores_dict = {} # Generalization of time for network in os.listdir(path_templates): network = network[:-21] ds_network = FeatureSlicer({network: ['!0']}).transform(ds) n_samples, n_voxels = ds_network.shape data = ds_network.samples.reshape(-1, 135, n_voxels) X = np.rollaxis(data, 1, 3) y = np.arange(data.shape[0]) % 2 scores = cross_val_multiscore(time_gen, X, y, cv=12, n_jobs=20) scores_dict[network] = scores ######## Pattern connectivity ################ results = trajectory_connectivity(ds)