def cross_val_predict_proba(estimator, X, y, groups = None, cv = None, n_jobs = 1, verbose = 0, fit_params = None, pre_dispatch = '2*n_jobs'): ''' Gets class probability predictions for test examples over cross validations runs. Adapted from mne.decoding.base.cross_val_multiscore(). See that func's documentation for details on inputs. ''' import time import numbers from mne.parallel import parallel_func from mne.fixes import is_classifier from sklearn.base import clone from sklearn.utils import indexable from sklearn.model_selection._split import check_cv # check arguments X, y, groups = indexable(X, y, groups) cv = check_cv(cv, y, classifier = is_classifier(estimator)) cv_iter = list(cv.split(X, y, groups)) # We clone the estimator to make sure that all the folds are # independent, and that it is pickle-able. # Note: this parallelization is implemented using MNE Parallel parallel, p_func, n_jobs = parallel_func(_predict_proba, n_jobs, pre_dispatch = pre_dispatch) preds = parallel(p_func(clone(estimator), X, y, train, test, 0, None, fit_params) for train, test in cv_iter) # flatten over parallel output y_hat = np.concatenate([p[0] for p in preds], axis = 0) is_y_true = True try: y_true = np.concatenate([p[1] for p in preds], axis = 0) except: # learner was unsupervised is_y_true = False # return results if is_y_true: return y_hat, y_true else: return y_hat
def test_get_coef(): """Test getting linear coefficients (filters/patterns) from estimators.""" from sklearn.base import TransformerMixin, BaseEstimator from sklearn.pipeline import make_pipeline from sklearn.preprocessing import StandardScaler from sklearn.linear_model import Ridge, LinearRegression lm = LinearModel() assert (is_classifier(lm)) lm = LinearModel(Ridge()) assert (is_regressor(lm)) # Define a classifier, an invertible transformer and an non-invertible one. class Clf(BaseEstimator): def fit(self, X, y): return self class NoInv(TransformerMixin): def fit(self, X, y): return self def transform(self, X): return X class Inv(NoInv): def inverse_transform(self, X): return X X, y, A = _make_data(n_samples=2000, n_features=3, n_targets=1) # I. Test inverse function # Check that we retrieve the right number of inverse functions even if # there are nested pipelines good_estimators = [ (1, make_pipeline(Inv(), Clf())), (2, make_pipeline(Inv(), Inv(), Clf())), (3, make_pipeline(Inv(), make_pipeline(Inv(), Inv()), Clf())), ] for expected_n, est in good_estimators: est.fit(X, y) assert (expected_n == len(_get_inverse_funcs(est))) bad_estimators = [ Clf(), # no preprocessing Inv(), # final estimator isn't classifier make_pipeline(NoInv(), Clf()), # first step isn't invertible make_pipeline(Inv(), make_pipeline(Inv(), NoInv()), Clf()), # nested step isn't invertible ] for est in bad_estimators: est.fit(X, y) invs = _get_inverse_funcs(est) assert_equal(invs, list()) # II. Test get coef for simple estimator and pipelines for clf in (lm, make_pipeline(StandardScaler(), lm)): clf.fit(X, y) # Retrieve final linear model filters = get_coef(clf, 'filters_', False) if hasattr(clf, 'steps'): coefs = clf.steps[-1][-1].model.coef_ else: coefs = clf.model.coef_ assert_array_equal(filters, coefs[0]) patterns = get_coef(clf, 'patterns_', False) assert (filters[0] != patterns[0]) n_chans = X.shape[1] assert_array_equal(filters.shape, patterns.shape, [n_chans, n_chans]) # Inverse transform linear model filters_inv = get_coef(clf, 'filters_', True) assert (filters[0] != filters_inv[0]) patterns_inv = get_coef(clf, 'patterns_', True) assert (patterns[0] != patterns_inv[0]) # Check with search_light and combination of preprocessing ending with sl: slider = SlidingEstimator(make_pipeline(StandardScaler(), lm)) X = np.transpose([X, -X], [1, 2, 0]) # invert X across 2 time samples clfs = (make_pipeline(Scaler(None, scalings='mean'), slider), slider) for clf in clfs: clf.fit(X, y) for inverse in (True, False): patterns = get_coef(clf, 'patterns_', inverse) filters = get_coef(clf, 'filters_', inverse) assert_array_equal(filters.shape, patterns.shape, X.shape[1:]) # the two time samples get inverted patterns assert_equal(patterns[0, 0], -patterns[0, 1]) for t in [0, 1]: assert_array_equal(get_coef(clf.estimators_[t], 'filters_', False), filters[:, t]) # Check patterns with more than 1 regressor for n_features in [1, 5]: for n_targets in [1, 3]: X, Y, A = _make_data(n_samples=5000, n_features=5, n_targets=3) lm = LinearModel(LinearRegression()).fit(X, Y) assert_array_equal(lm.filters_.shape, lm.patterns_.shape) assert_array_equal(lm.filters_.shape, [3, 5]) assert_array_almost_equal(A, lm.patterns_.T, decimal=2) lm = LinearModel(Ridge(alpha=1)).fit(X, Y) assert_array_almost_equal(A, lm.patterns_.T, decimal=2) # Check can pass fitting parameters lm.fit(X, Y, sample_weight=np.ones(len(Y)))
def test_get_coef(): """Test getting linear coefficients (filters/patterns) from estimators.""" from sklearn.base import TransformerMixin, BaseEstimator from sklearn.pipeline import make_pipeline from sklearn.preprocessing import StandardScaler from sklearn import svm from sklearn.linear_model import Ridge from sklearn.model_selection import GridSearchCV lm_classification = LinearModel() assert (is_classifier(lm_classification)) lm_regression = LinearModel(Ridge()) assert (is_regressor(lm_regression)) parameters = {'kernel': ['linear'], 'C': [1, 10]} lm_gs_classification = LinearModel( GridSearchCV(svm.SVC(), parameters, cv=2, refit=True, n_jobs=1)) assert (is_classifier(lm_gs_classification)) lm_gs_regression = LinearModel( GridSearchCV(svm.SVR(), parameters, cv=2, refit=True, n_jobs=1)) assert (is_regressor(lm_gs_regression)) # Define a classifier, an invertible transformer and an non-invertible one. class Clf(BaseEstimator): def fit(self, X, y): return self class NoInv(TransformerMixin): def fit(self, X, y): return self def transform(self, X): return X class Inv(NoInv): def inverse_transform(self, X): return X X, y, A = _make_data(n_samples=1000, n_features=3, n_targets=1) # I. Test inverse function # Check that we retrieve the right number of inverse functions even if # there are nested pipelines good_estimators = [ (1, make_pipeline(Inv(), Clf())), (2, make_pipeline(Inv(), Inv(), Clf())), (3, make_pipeline(Inv(), make_pipeline(Inv(), Inv()), Clf())), ] for expected_n, est in good_estimators: est.fit(X, y) assert (expected_n == len(_get_inverse_funcs(est))) bad_estimators = [ Clf(), # no preprocessing Inv(), # final estimator isn't classifier make_pipeline(NoInv(), Clf()), # first step isn't invertible make_pipeline(Inv(), make_pipeline(Inv(), NoInv()), Clf()), # nested step isn't invertible ] for est in bad_estimators: est.fit(X, y) invs = _get_inverse_funcs(est) assert_equal(invs, list()) # II. Test get coef for classification/regression estimators and pipelines rng = np.random.RandomState(0) for clf in (lm_regression, lm_gs_classification, make_pipeline(StandardScaler(), lm_classification), make_pipeline(StandardScaler(), lm_gs_regression)): # generate some categorical/continuous data # according to the type of estimator. if is_classifier(clf): n, n_features = 1000, 3 X = rng.rand(n, n_features) y = np.arange(n) % 2 else: X, y, A = _make_data(n_samples=1000, n_features=3, n_targets=1) y = np.ravel(y) clf.fit(X, y) # Retrieve final linear model filters = get_coef(clf, 'filters_', False) if hasattr(clf, 'steps'): if hasattr(clf.steps[-1][-1].model, 'best_estimator_'): # Linear Model with GridSearchCV coefs = clf.steps[-1][-1].model.best_estimator_.coef_ else: # Standard Linear Model coefs = clf.steps[-1][-1].model.coef_ else: if hasattr(clf.model, 'best_estimator_'): # Linear Model with GridSearchCV coefs = clf.model.best_estimator_.coef_ else: # Standard Linear Model coefs = clf.model.coef_ if coefs.ndim == 2 and coefs.shape[0] == 1: coefs = coefs[0] assert_array_equal(filters, coefs) patterns = get_coef(clf, 'patterns_', False) assert (filters[0] != patterns[0]) n_chans = X.shape[1] assert_array_equal(filters.shape, patterns.shape, [n_chans, n_chans]) # Inverse transform linear model filters_inv = get_coef(clf, 'filters_', True) assert (filters[0] != filters_inv[0]) patterns_inv = get_coef(clf, 'patterns_', True) assert (patterns[0] != patterns_inv[0])
def test_generalization_across_time(): """Test time generalization decoding.""" from sklearn.svm import SVC # KernelRidge is used for testing 1) regression analyses 2) n-dimensional # predictions. from sklearn.kernel_ridge import KernelRidge from sklearn.preprocessing import LabelEncoder from sklearn.metrics import roc_auc_score, mean_squared_error epochs = make_epochs() y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1)) if check_version('sklearn', '0.18'): from sklearn.model_selection import (KFold, StratifiedKFold, ShuffleSplit, LeaveOneGroupOut) cv = LeaveOneGroupOut() cv_shuffle = ShuffleSplit() # XXX we cannot pass any other parameters than X and y to cv.split # so we have to build it before hand cv_lolo = [(train, test) for train, test in cv.split( y_4classes, y_4classes, y_4classes)] # With sklearn >= 0.17, `clf` can be identified as a regressor, and # the scoring metrics can therefore be automatically assigned. scorer_regress = None else: from sklearn.cross_validation import (KFold, StratifiedKFold, ShuffleSplit, LeaveOneLabelOut) cv_shuffle = ShuffleSplit(len(epochs)) cv_lolo = LeaveOneLabelOut(y_4classes) # With sklearn < 0.17, `clf` cannot be identified as a regressor, and # therefore the scoring metrics cannot be automatically assigned. scorer_regress = mean_squared_error # Test default running with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(picks='foo') assert_equal("<GAT | no fit, no prediction, no score>", "%s" % gat) assert_raises(ValueError, gat.fit, epochs) with warnings.catch_warnings(record=True): # check classic fit + check manual picks gat.picks = [0] gat.fit(epochs) # check optional y as array gat.picks = None gat.fit(epochs, y=epochs.events[:, 2]) # check optional y as list gat.fit(epochs, y=epochs.events[:, 2].tolist()) assert_equal(len(gat.picks_), len(gat.ch_names), 1) assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), no " "prediction, no score>", '%s' % gat) assert_equal(gat.ch_names, epochs.ch_names) # test different predict function: with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(predict_method='decision_function') gat.fit(epochs) # With classifier, the default cv is StratifiedKFold assert_true(gat.cv_.__class__ == StratifiedKFold) gat.predict(epochs) assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 1)) gat.predict_method = 'predict_proba' gat.predict(epochs) assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 2)) gat.predict_method = 'foo' assert_raises(NotImplementedError, gat.predict, epochs) gat.predict_method = 'predict' gat.predict(epochs) assert_array_equal(np.shape(gat.y_pred_), (15, 15, 14, 1)) assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), " "predicted 14 epochs, no score>", "%s" % gat) gat.score(epochs) assert_true(gat.scorer_.__name__ == 'accuracy_score') # check clf / predict_method combinations for which the scoring metrics # cannot be inferred. gat.scorer = None gat.predict_method = 'decision_function' assert_raises(ValueError, gat.score, epochs) # Check specifying y manually gat.predict_method = 'predict' gat.score(epochs, y=epochs.events[:, 2]) gat.score(epochs, y=epochs.events[:, 2].tolist()) assert_equal("<GAT | fitted, start : -0.200 (s), stop : 0.499 (s), " "predicted 14 epochs,\n scored " "(accuracy_score)>", "%s" % gat) with warnings.catch_warnings(record=True): gat.fit(epochs, y=epochs.events[:, 2]) old_mode = gat.predict_mode gat.predict_mode = 'super-foo-mode' assert_raises(ValueError, gat.predict, epochs) gat.predict_mode = old_mode gat.score(epochs, y=epochs.events[:, 2]) assert_true("accuracy_score" in '%s' % gat.scorer_) epochs2 = epochs.copy() # check _DecodingTime class assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: " "0.050 (s), length: 0.050 (s), n_time_windows: 15>", "%s" % gat.train_times_) assert_equal("<DecodingTime | start: -0.200 (s), stop: 0.499 (s), step: " "0.050 (s), length: 0.050 (s), n_time_windows: 15 x 15>", "%s" % gat.test_times_) # the y-check gat.predict_mode = 'mean-prediction' epochs2.events[:, 2] += 10 gat_ = copy.deepcopy(gat) with use_log_level('error'): assert_raises(ValueError, gat_.score, epochs2) gat.predict_mode = 'cross-validation' # Test basics # --- number of trials assert_true(gat.y_train_.shape[0] == gat.y_true_.shape[0] == len(gat.y_pred_[0][0]) == 14) # --- number of folds assert_true(np.shape(gat.estimators_)[1] == gat.cv) # --- length training size assert_true(len(gat.train_times_['slices']) == 15 == np.shape(gat.estimators_)[0]) # --- length testing sizes assert_true(len(gat.test_times_['slices']) == 15 == np.shape(gat.scores_)[0]) assert_true(len(gat.test_times_['slices'][0]) == 15 == np.shape(gat.scores_)[1]) # Test score_mode gat.score_mode = 'foo' assert_raises(ValueError, gat.score, epochs) gat.score_mode = 'fold-wise' scores = gat.score(epochs) assert_array_equal(np.shape(scores), [15, 15, 5]) gat.score_mode = 'mean-sample-wise' scores = gat.score(epochs) assert_array_equal(np.shape(scores), [15, 15]) gat.score_mode = 'mean-fold-wise' scores = gat.score(epochs) assert_array_equal(np.shape(scores), [15, 15]) gat.predict_mode = 'mean-prediction' with warnings.catch_warnings(record=True) as w: gat.score(epochs) assert_true(any("score_mode changed from " in str(ww.message) for ww in w)) # Test longer time window with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(train_times={'length': .100}) with warnings.catch_warnings(record=True): gat2 = gat.fit(epochs) assert_true(gat is gat2) # return self assert_true(hasattr(gat2, 'cv_')) assert_true(gat2.cv_ != gat.cv) with warnings.catch_warnings(record=True): # not vectorizing scores = gat.score(epochs) assert_true(isinstance(scores, np.ndarray)) # type check assert_equal(len(scores[0]), len(scores)) # shape check assert_equal(len(gat.test_times_['slices'][0][0]), 2) # Decim training steps with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(train_times={'step': .100}) with warnings.catch_warnings(record=True): gat.fit(epochs) gat.score(epochs) assert_true(len(gat.scores_) == len(gat.estimators_) == 8) # training time assert_equal(len(gat.scores_[0]), 15) # testing time # Test start stop training & test cv without n_fold params y_4classes = np.hstack((epochs.events[:7, 2], epochs.events[7:, 2] + 1)) train_times = dict(start=0.090, stop=0.250) with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(cv=cv_lolo, train_times=train_times) # predict without fit assert_raises(RuntimeError, gat.predict, epochs) with warnings.catch_warnings(record=True): gat.fit(epochs, y=y_4classes) gat.score(epochs) assert_equal(len(gat.scores_), 4) assert_equal(gat.train_times_['times'][0], epochs.times[6]) assert_equal(gat.train_times_['times'][-1], epochs.times[9]) # Test score without passing epochs & Test diagonal decoding with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(test_times='diagonal') with warnings.catch_warnings(record=True): # not vectorizing gat.fit(epochs) assert_raises(RuntimeError, gat.score) with warnings.catch_warnings(record=True): # not vectorizing gat.predict(epochs) scores = gat.score() assert_true(scores is gat.scores_) assert_equal(np.shape(gat.scores_), (15, 1)) assert_array_equal([tim for ttime in gat.test_times_['times'] for tim in ttime], gat.train_times_['times']) # Test generalization across conditions with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(predict_mode='mean-prediction', cv=2) with warnings.catch_warnings(record=True): gat.fit(epochs[0:6]) with warnings.catch_warnings(record=True): # There are some empty test folds because of n_trials gat.predict(epochs[7:]) gat.score(epochs[7:]) # Test training time parameters gat_ = copy.deepcopy(gat) # --- start stop outside time range gat_.train_times = dict(start=-999.) with use_log_level('error'): assert_raises(ValueError, gat_.fit, epochs) gat_.train_times = dict(start=999.) assert_raises(ValueError, gat_.fit, epochs) # --- impossible slices gat_.train_times = dict(step=.000001) assert_raises(ValueError, gat_.fit, epochs) gat_.train_times = dict(length=.000001) assert_raises(ValueError, gat_.fit, epochs) gat_.train_times = dict(length=999.) assert_raises(ValueError, gat_.fit, epochs) # Test testing time parameters # --- outside time range gat.test_times = dict(start=-999.) with warnings.catch_warnings(record=True): # no epochs in fold assert_raises(ValueError, gat.predict, epochs) gat.test_times = dict(start=999.) with warnings.catch_warnings(record=True): # no test epochs assert_raises(ValueError, gat.predict, epochs) # --- impossible slices gat.test_times = dict(step=.000001) with warnings.catch_warnings(record=True): # no test epochs assert_raises(ValueError, gat.predict, epochs) gat_ = copy.deepcopy(gat) gat_.train_times_['length'] = .000001 gat_.test_times = dict(length=.000001) with warnings.catch_warnings(record=True): # no test epochs assert_raises(ValueError, gat_.predict, epochs) # --- test time region of interest gat.test_times = dict(step=.150) with warnings.catch_warnings(record=True): # not vectorizing gat.predict(epochs) assert_array_equal(np.shape(gat.y_pred_), (15, 5, 14, 1)) # --- silly value gat.test_times = 'foo' with warnings.catch_warnings(record=True): # no test epochs assert_raises(ValueError, gat.predict, epochs) assert_raises(RuntimeError, gat.score) # --- unmatched length between training and testing time gat.test_times = dict(length=.150) assert_raises(ValueError, gat.predict, epochs) # --- irregular length training and testing times # 2 estimators, the first one is trained on two successive time samples # whereas the second one is trained on a single time sample. train_times = dict(slices=[[0, 1], [1]]) # The first estimator is tested once, the second estimator is tested on # two successive time samples. test_times = dict(slices=[[[0, 1]], [[0], [1]]]) with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(train_times=train_times, test_times=test_times) gat.fit(epochs) with warnings.catch_warnings(record=True): # not vectorizing gat.score(epochs) assert_array_equal(np.shape(gat.y_pred_[0]), [1, len(epochs), 1]) assert_array_equal(np.shape(gat.y_pred_[1]), [2, len(epochs), 1]) # check cannot Automatically infer testing times for adhoc training times gat.test_times = None assert_raises(ValueError, gat.predict, epochs) svc = SVC(C=1, kernel='linear', probability=True) with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(clf=svc, predict_mode='mean-prediction') with warnings.catch_warnings(record=True): gat.fit(epochs) # sklearn needs it: c.f. # https://github.com/scikit-learn/scikit-learn/issues/2723 # and http://bit.ly/1u7t8UT with use_log_level('error'): assert_raises(ValueError, gat.score, epochs2) gat.score(epochs) assert_true(0.0 <= np.min(scores) <= 1.0) assert_true(0.0 <= np.max(scores) <= 1.0) # Test that error if cv is not partition with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(cv=cv_shuffle, predict_mode='cross-validation') gat.fit(epochs) assert_raises(ValueError, gat.predict, epochs) with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(cv=cv_shuffle, predict_mode='mean-prediction') gat.fit(epochs) gat.predict(epochs) # Test that gets error if train on one dataset, test on another, and don't # specify appropriate cv: with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime() gat.fit(epochs) with warnings.catch_warnings(record=True): gat.fit(epochs) gat.predict(epochs) assert_raises(ValueError, gat.predict, epochs[:10]) # Make CV with some empty train and test folds: # --- empty test fold(s) should warn when gat.predict() gat._cv_splits[0] = [gat._cv_splits[0][0], np.empty(0)] with warnings.catch_warnings(record=True) as w: gat.predict(epochs) assert_true(len(w) > 0) assert_true(any('do not have any test epochs' in str(ww.message) for ww in w)) # --- empty train fold(s) should raise when gat.fit() with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(cv=[([0], [1]), ([], [0])]) assert_raises(ValueError, gat.fit, epochs[:2]) # Check that still works with classifier that output y_pred with # shape = (n_trials, 1) instead of (n_trials,) if check_version('sklearn', '0.17'): # no is_regressor before v0.17 with warnings.catch_warnings(record=True): # dep gat = GeneralizationAcrossTime(clf=KernelRidge(), cv=2) epochs.crop(None, epochs.times[2]) gat.fit(epochs) # With regression the default cv is KFold and not StratifiedKFold assert_true(gat.cv_.__class__ == KFold) gat.score(epochs) # with regression the default scoring metrics is mean squared error assert_true(gat.scorer_.__name__ == 'mean_squared_error') # Test combinations of complex scenarios # 2 or more distinct classes n_classes = [2, 4] # 4 tested # nicely ordered labels or not le = LabelEncoder() y = le.fit_transform(epochs.events[:, 2]) y[len(y) // 2:] += 2 ys = (y, y + 1000) # Univariate and multivariate prediction svc = SVC(C=1, kernel='linear', probability=True) reg = KernelRidge() def scorer_proba(y_true, y_pred): return roc_auc_score(y_true, y_pred[:, 0]) # We re testing 3 scenario: default, classifier + predict_proba, regressor scorers = [None, scorer_proba, scorer_regress] predict_methods = [None, 'predict_proba', None] clfs = [svc, svc, reg] # Test all combinations for clf, predict_method, scorer in zip(clfs, predict_methods, scorers): for y in ys: for n_class in n_classes: for predict_mode in ['cross-validation', 'mean-prediction']: # Cannot use AUC for n_class > 2 if (predict_method == 'predict_proba' and n_class != 2): continue y_ = y % n_class with warnings.catch_warnings(record=True): gat = GeneralizationAcrossTime( cv=2, clf=clf, scorer=scorer, predict_mode=predict_mode) gat.fit(epochs, y=y_) gat.score(epochs, y=y_) # Check that scorer is correctly defined manually and # automatically. scorer_name = gat.scorer_.__name__ if scorer is None: if is_classifier(clf): assert_equal(scorer_name, 'accuracy_score') else: assert_equal(scorer_name, 'mean_squared_error') else: assert_equal(scorer_name, scorer.__name__)
def test_get_coef(): """Test getting linear coefficients (filters/patterns) from estimators.""" from sklearn.base import TransformerMixin, BaseEstimator from sklearn.pipeline import make_pipeline from sklearn.preprocessing import StandardScaler from sklearn.linear_model import Ridge, LinearRegression lm = LinearModel() assert_true(is_classifier(lm)) lm = LinearModel(Ridge()) assert_true(is_regressor(lm)) # Define a classifier, an invertible transformer and an non-invertible one. class Clf(BaseEstimator): def fit(self, X, y): return self class NoInv(TransformerMixin): def fit(self, X, y): return self def transform(self, X): return X class Inv(NoInv): def inverse_transform(self, X): return X X, y, A = _make_data(n_samples=2000, n_features=3, n_targets=1) # I. Test inverse function # Check that we retrieve the right number of inverse functions even if # there are nested pipelines good_estimators = [ (1, make_pipeline(Inv(), Clf())), (2, make_pipeline(Inv(), Inv(), Clf())), (3, make_pipeline(Inv(), make_pipeline(Inv(), Inv()), Clf())), ] for expected_n, est in good_estimators: est.fit(X, y) assert_true(expected_n == len(_get_inverse_funcs(est))) bad_estimators = [ Clf(), # no preprocessing Inv(), # final estimator isn't classifier make_pipeline(NoInv(), Clf()), # first step isn't invertible make_pipeline(Inv(), make_pipeline( Inv(), NoInv()), Clf()), # nested step isn't invertible ] for est in bad_estimators: est.fit(X, y) invs = _get_inverse_funcs(est) assert_equal(invs, list()) # II. Test get coef for simple estimator and pipelines for clf in (lm, make_pipeline(StandardScaler(), lm)): clf.fit(X, y) # Retrieve final linear model filters = get_coef(clf, 'filters_', False) if hasattr(clf, 'steps'): coefs = clf.steps[-1][-1].model.coef_ else: coefs = clf.model.coef_ assert_array_equal(filters, coefs[0]) patterns = get_coef(clf, 'patterns_', False) assert_true(filters[0] != patterns[0]) n_chans = X.shape[1] assert_array_equal(filters.shape, patterns.shape, [n_chans, n_chans]) # Inverse transform linear model filters_inv = get_coef(clf, 'filters_', True) assert_true(filters[0] != filters_inv[0]) patterns_inv = get_coef(clf, 'patterns_', True) assert_true(patterns[0] != patterns_inv[0]) # Check with search_light and combination of preprocessing ending with sl: slider = SlidingEstimator(make_pipeline(StandardScaler(), lm)) X = np.transpose([X, -X], [1, 2, 0]) # invert X across 2 time samples clfs = (make_pipeline(Scaler(None, scalings='mean'), slider), slider) for clf in clfs: clf.fit(X, y) for inverse in (True, False): patterns = get_coef(clf, 'patterns_', inverse) filters = get_coef(clf, 'filters_', inverse) assert_array_equal(filters.shape, patterns.shape, X.shape[1:]) # the two time samples get inverted patterns assert_equal(patterns[0, 0], -patterns[0, 1]) for t in [0, 1]: assert_array_equal(get_coef(clf.estimators_[t], 'filters_', False), filters[:, t]) # Check patterns with more than 1 regressor for n_features in [1, 5]: for n_targets in [1, 3]: X, Y, A = _make_data(n_samples=5000, n_features=5, n_targets=3) lm = LinearModel(LinearRegression()).fit(X, Y) assert_array_equal(lm.filters_.shape, lm.patterns_.shape) assert_array_equal(lm.filters_.shape, [3, 5]) assert_array_almost_equal(A, lm.patterns_.T, decimal=2) lm = LinearModel(Ridge(alpha=1)).fit(X, Y) assert_array_almost_equal(A, lm.patterns_.T, decimal=2) # Check can pass fitting parameters lm.fit(X, Y, sample_weight=np.ones(len(Y)))
def test_get_coef(): """Test getting linear coefficients (filters/patterns) from estimators.""" from sklearn.base import TransformerMixin, BaseEstimator from sklearn.pipeline import make_pipeline from sklearn.preprocessing import StandardScaler from sklearn import svm from sklearn.linear_model import Ridge, LinearRegression from sklearn.model_selection import GridSearchCV lm_classification = LinearModel() assert (is_classifier(lm_classification)) lm_regression = LinearModel(Ridge()) assert (is_regressor(lm_regression)) parameters = {'kernel': ['linear'], 'C': [1, 10]} lm_gs_classification = LinearModel( GridSearchCV(svm.SVC(), parameters, cv=2, refit=True, iid=False, n_jobs=1)) assert (is_classifier(lm_gs_classification)) lm_gs_regression = LinearModel( GridSearchCV(svm.SVR(), parameters, cv=2, refit=True, iid=False, n_jobs=1)) assert (is_regressor(lm_gs_regression)) # Define a classifier, an invertible transformer and an non-invertible one. class Clf(BaseEstimator): def fit(self, X, y): return self class NoInv(TransformerMixin): def fit(self, X, y): return self def transform(self, X): return X class Inv(NoInv): def inverse_transform(self, X): return X X, y, A = _make_data(n_samples=1000, n_features=3, n_targets=1) # I. Test inverse function # Check that we retrieve the right number of inverse functions even if # there are nested pipelines good_estimators = [ (1, make_pipeline(Inv(), Clf())), (2, make_pipeline(Inv(), Inv(), Clf())), (3, make_pipeline(Inv(), make_pipeline(Inv(), Inv()), Clf())), ] for expected_n, est in good_estimators: est.fit(X, y) assert (expected_n == len(_get_inverse_funcs(est))) bad_estimators = [ Clf(), # no preprocessing Inv(), # final estimator isn't classifier make_pipeline(NoInv(), Clf()), # first step isn't invertible make_pipeline(Inv(), make_pipeline(Inv(), NoInv()), Clf()), # nested step isn't invertible ] for est in bad_estimators: est.fit(X, y) invs = _get_inverse_funcs(est) assert_equal(invs, list()) # II. Test get coef for classification/regression estimators and pipelines rng = np.random.RandomState(0) for clf in (lm_regression, lm_gs_classification, make_pipeline(StandardScaler(), lm_classification), make_pipeline(StandardScaler(), lm_gs_regression)): # generate some categorical/continuous data # according to the type of estimator. if is_classifier(clf): n, n_features = 1000, 3 X = rng.rand(n, n_features) y = np.arange(n) % 2 else: X, y, A = _make_data(n_samples=1000, n_features=3, n_targets=1) y = np.ravel(y) clf.fit(X, y) # Retrieve final linear model filters = get_coef(clf, 'filters_', False) if hasattr(clf, 'steps'): if hasattr(clf.steps[-1][-1].model, 'best_estimator_'): # Linear Model with GridSearchCV coefs = clf.steps[-1][-1].model.best_estimator_.coef_ else: # Standard Linear Model coefs = clf.steps[-1][-1].model.coef_ else: if hasattr(clf.model, 'best_estimator_'): # Linear Model with GridSearchCV coefs = clf.model.best_estimator_.coef_ else: # Standard Linear Model coefs = clf.model.coef_ if coefs.ndim == 2 and coefs.shape[0] == 1: coefs = coefs[0] assert_array_equal(filters, coefs) patterns = get_coef(clf, 'patterns_', False) assert (filters[0] != patterns[0]) n_chans = X.shape[1] assert_array_equal(filters.shape, patterns.shape, [n_chans, n_chans]) # Inverse transform linear model filters_inv = get_coef(clf, 'filters_', True) assert (filters[0] != filters_inv[0]) patterns_inv = get_coef(clf, 'patterns_', True) assert (patterns[0] != patterns_inv[0]) # Check with search_light and combination of preprocessing ending with sl: slider = SlidingEstimator(make_pipeline(StandardScaler(), lm_regression)) X = np.transpose([X, -X], [1, 2, 0]) # invert X across 2 time samples clfs = (make_pipeline(Scaler(None, scalings='mean'), slider), slider) for clf in clfs: clf.fit(X, y) for inverse in (True, False): patterns = get_coef(clf, 'patterns_', inverse) filters = get_coef(clf, 'filters_', inverse) assert_array_equal(filters.shape, patterns.shape, X.shape[1:]) # the two time samples get inverted patterns assert_equal(patterns[0, 0], -patterns[0, 1]) for t in [0, 1]: assert_array_equal(get_coef(clf.estimators_[t], 'filters_', False), filters[:, t]) # Check patterns with more than 1 regressor for n_features in [1, 5]: for n_targets in [1, 3]: X, Y, A = _make_data(n_samples=3000, n_features=5, n_targets=3) lm = LinearModel(LinearRegression()).fit(X, Y) assert_array_equal(lm.filters_.shape, lm.patterns_.shape) assert_array_equal(lm.filters_.shape, [3, 5]) assert_array_almost_equal(A, lm.patterns_.T, decimal=2) lm = LinearModel(Ridge(alpha=1)).fit(X, Y) assert_array_almost_equal(A, lm.patterns_.T, decimal=2) # Check can pass fitting parameters lm.fit(X, Y, sample_weight=np.ones(len(Y)))