def test_cluster_permutation_test(): """Test cluster level permutations tests.""" condition1_1d, condition2_1d, condition1_2d, condition2_2d = \ _get_conditions() for condition1, condition2 in zip((condition1_1d, condition1_2d), (condition2_1d, condition2_2d)): T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=1, seed=1, buffer_size=None) assert_equal(np.sum(cluster_p_values < 0.05), 1) T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=0, seed=1, buffer_size=None) assert_equal(np.sum(cluster_p_values < 0.05), 1) # test with 2 jobs and buffer_size enabled buffer_size = condition1.shape[1] // 10 T_obs, clusters, cluster_p_values_buff, hist =\ permutation_cluster_test([condition1, condition2], n_permutations=100, tail=0, seed=1, n_jobs=2, buffer_size=buffer_size) assert_array_equal(cluster_p_values, cluster_p_values_buff)
def test_cluster_permutation_test(numba_conditional): """Test cluster level permutations tests.""" condition1_1d, condition2_1d, condition1_2d, condition2_2d = \ _get_conditions() for condition1, condition2 in zip((condition1_1d, condition1_2d), (condition2_1d, condition2_2d)): T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=1, seed=1, buffer_size=None, out_type='mask') p_min = np.min(cluster_p_values) assert_equal(np.sum(cluster_p_values < 0.05), 1) assert_allclose(p_min, 0.01, atol=1e-6) # test with 2 jobs and buffer_size enabled buffer_size = condition1.shape[1] // 10 T_obs, clusters, cluster_p_values_buff, hist =\ permutation_cluster_test([condition1, condition2], n_permutations=100, tail=1, seed=1, n_jobs=2, buffer_size=buffer_size, out_type='mask') assert_array_equal(cluster_p_values, cluster_p_values_buff) def stat_fun(X, Y): return stats.f_oneway(X, Y)[0] with pytest.warns(RuntimeWarning, match='is only valid'): permutation_cluster_test([condition1, condition2], n_permutations=1, stat_fun=stat_fun, out_type='mask')
def test_thresholds(numba_conditional): """Test automatic threshold calculations.""" # within subjects rng = np.random.RandomState(0) X = rng.randn(10, 1, 1) + 0.08 want_thresh = -stats.t.ppf(0.025, len(X) - 1) assert 0.03 < stats.ttest_1samp(X[:, 0, 0], 0)[1] < 0.05 my_fun = partial(ttest_1samp_no_p) with catch_logging() as log: with pytest.warns(RuntimeWarning, match='threshold is only valid'): out = permutation_cluster_1samp_test(X, stat_fun=my_fun, seed=0, verbose=True, out_type='mask') log = log.getvalue() assert str(want_thresh)[:6] in log assert len(out[1]) == 1 # 1 cluster assert_allclose(out[2], 0.033203, atol=1e-6) # between subjects Y = rng.randn(10, 1, 1) Z = rng.randn(10, 1, 1) - 0.7 X = [X, Y, Z] want_thresh = stats.f.ppf(1. - 0.05, 2, sum(len(a) for a in X) - len(X)) p = stats.f_oneway(*X)[1] assert 0.03 < p < 0.05 my_fun = partial(f_oneway) # just to make the check fail with catch_logging() as log: with pytest.warns(RuntimeWarning, match='threshold is only valid'): out = permutation_cluster_test(X, tail=1, stat_fun=my_fun, seed=0, verbose=True, out_type='mask') log = log.getvalue() assert str(want_thresh)[:6] in log assert len(out[1]) == 1 # 1 cluster assert_allclose(out[2], 0.041992, atol=1e-6) with pytest.warns(RuntimeWarning, match='Ignoring argument "tail"'): permutation_cluster_test(X, tail=0, out_type='mask') # nan handling in TFCE X = np.repeat(X[0], 2, axis=1) X[:, 1] = 0 with pytest.warns(RuntimeWarning, match='invalid value'): # NumPy out = permutation_cluster_1samp_test(X, seed=0, threshold=dict(start=0, step=0.1), out_type='mask') assert (out[2] < 0.05).any() assert not (out[2] < 0.05).all() X[:, 0] = 0 with pytest.raises(RuntimeError, match='finite'): with np.errstate(invalid='ignore'): permutation_cluster_1samp_test(X, seed=0, threshold=dict(start=0, step=0.1), buffer_size=None, out_type='mask')
def test_thresholds(): """Test automatic threshold calculations.""" # within subjects rng = np.random.RandomState(0) X = rng.randn(10, 1, 1) + 0.08 want_thresh = -stats.t.ppf(0.025, len(X) - 1) assert 0.03 < stats.ttest_1samp(X[:, 0, 0], 0)[1] < 0.05 my_fun = partial(ttest_1samp_no_p) with catch_logging() as log: with pytest.warns(RuntimeWarning, match='threshold is only valid'): out = permutation_cluster_1samp_test(X, stat_fun=my_fun, verbose=True) log = log.getvalue() assert str(want_thresh)[:6] in log assert len(out[1]) == 1 # 1 cluster assert 0.03 < out[2] < 0.05 # between subjects Y = rng.randn(10, 1, 1) Z = rng.randn(10, 1, 1) - 0.7 X = [X, Y, Z] want_thresh = stats.f.ppf(1. - 0.05, 2, sum(len(a) for a in X) - len(X)) p = stats.f_oneway(*X)[1] assert 0.03 < p < 0.05 my_fun = partial(f_oneway) # just to make the check fail with catch_logging() as log: with pytest.warns(RuntimeWarning, match='threshold is only valid'): out = permutation_cluster_test(X, tail=1, stat_fun=my_fun, verbose=True) log = log.getvalue() assert str(want_thresh)[:6] in log assert len(out[1]) == 1 # 1 cluster assert 0.03 < out[2] < 0.05 with pytest.warns(RuntimeWarning, match='Ignoring argument "tail"'): permutation_cluster_test(X, tail=0)
def test_cluster_permutation_test(): """Test cluster level permutations tests """ condition1_1d, condition2_1d, condition1_2d, condition2_2d = \ _get_conditions() for condition1, condition2 in zip((condition1_1d, condition1_2d), (condition2_1d, condition2_2d)): T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=1, seed=1, buffer_size=None) assert_equal(np.sum(cluster_p_values < 0.05), 1) T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=0, seed=1, buffer_size=None) assert_equal(np.sum(cluster_p_values < 0.05), 1) # test with 2 jobs and buffer_size enabled buffer_size = condition1.shape[1] // 10 T_obs, clusters, cluster_p_values_buff, hist =\ permutation_cluster_test([condition1, condition2], n_permutations=100, tail=0, seed=1, n_jobs=2, buffer_size=buffer_size) assert_array_equal(cluster_p_values, cluster_p_values_buff)
def test_cluster_permutation_test(): """Test cluster level permutations tests.""" condition1_1d, condition2_1d, condition1_2d, condition2_2d = \ _get_conditions() for condition1, condition2 in zip((condition1_1d, condition1_2d), (condition2_1d, condition2_2d)): T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=1, seed=1, buffer_size=None) assert_equal(np.sum(cluster_p_values < 0.05), 1) T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=0, seed=1, buffer_size=None) assert_equal(np.sum(cluster_p_values < 0.05), 1) # test with 2 jobs and buffer_size enabled buffer_size = condition1.shape[1] // 10 T_obs, clusters, cluster_p_values_buff, hist =\ permutation_cluster_test([condition1, condition2], n_permutations=100, tail=0, seed=1, n_jobs=2, buffer_size=buffer_size) assert_array_equal(cluster_p_values, cluster_p_values_buff) def stat_fun(X, Y): return stats.f_oneway(X, Y)[0] with warnings.catch_warnings(record=True) as w: permutation_cluster_test([condition1, condition2], n_permutations=1, stat_fun=stat_fun) assert_equal(len(w), 1) assert 'is only valid' in str(w[0].message)
def test_cluster_permutation_test(): """Test cluster level permutations tests.""" condition1_1d, condition2_1d, condition1_2d, condition2_2d = \ _get_conditions() for condition1, condition2 in zip((condition1_1d, condition1_2d), (condition2_1d, condition2_2d)): T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=1, seed=1, buffer_size=None) assert_equal(np.sum(cluster_p_values < 0.05), 1) T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=1, seed=1, buffer_size=None) assert_equal(np.sum(cluster_p_values < 0.05), 1) # test with 2 jobs and buffer_size enabled buffer_size = condition1.shape[1] // 10 T_obs, clusters, cluster_p_values_buff, hist =\ permutation_cluster_test([condition1, condition2], n_permutations=100, tail=1, seed=1, n_jobs=2, buffer_size=buffer_size) assert_array_equal(cluster_p_values, cluster_p_values_buff) def stat_fun(X, Y): return stats.f_oneway(X, Y)[0] with pytest.warns(RuntimeWarning, match='is only valid'): permutation_cluster_test([condition1, condition2], n_permutations=1, stat_fun=stat_fun)
def test_thresholds(numba_conditional): """Test automatic threshold calculations.""" # within subjects rng = np.random.RandomState(0) X = rng.randn(10, 1, 1) + 0.08 want_thresh = -stats.t.ppf(0.025, len(X) - 1) assert 0.03 < stats.ttest_1samp(X[:, 0, 0], 0)[1] < 0.05 my_fun = partial(ttest_1samp_no_p) with catch_logging() as log: with pytest.warns(RuntimeWarning, match='threshold is only valid'): out = permutation_cluster_1samp_test(X, stat_fun=my_fun, seed=0, verbose=True) log = log.getvalue() assert str(want_thresh)[:6] in log assert len(out[1]) == 1 # 1 cluster assert_allclose(out[2], 0.033203, atol=1e-6) # between subjects Y = rng.randn(10, 1, 1) Z = rng.randn(10, 1, 1) - 0.7 X = [X, Y, Z] want_thresh = stats.f.ppf(1. - 0.05, 2, sum(len(a) for a in X) - len(X)) p = stats.f_oneway(*X)[1] assert 0.03 < p < 0.05 my_fun = partial(f_oneway) # just to make the check fail with catch_logging() as log: with pytest.warns(RuntimeWarning, match='threshold is only valid'): out = permutation_cluster_test(X, tail=1, stat_fun=my_fun, seed=0, verbose=True) log = log.getvalue() assert str(want_thresh)[:6] in log assert len(out[1]) == 1 # 1 cluster assert_allclose(out[2], 0.041992, atol=1e-6) with pytest.warns(RuntimeWarning, match='Ignoring argument "tail"'): permutation_cluster_test(X, tail=0)
def test_cluster_permutation_test(): """Test cluster level permutations tests.""" T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=500, tail=1) assert_equal(np.sum(cluster_p_values < 0.05), 1) T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=500, tail=0) assert_equal(np.sum(cluster_p_values < 0.05), 1)
def test_cluster_permutation_test(): """Test cluster level permutations tests.""" for condition1, condition2 in zip((condition1_1d, condition1_2d), (condition2_1d, condition2_2d)): T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=500, tail=1, seed=1) assert_equal(np.sum(cluster_p_values < 0.05), 1) T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=500, tail=0, seed=1) assert_equal(np.sum(cluster_p_values < 0.05), 1) # test with 2 jobs T_obs, clusters, cluster_p_values_buff, hist =\ permutation_cluster_test([condition1, condition2], n_permutations=500, tail=0, seed=1, n_jobs=2) assert_array_equal(cluster_p_values, cluster_p_values_buff)
def test_cluster_permutation_test(): """Test cluster level permutations tests.""" for condition1, condition2 in zip((condition1_1d, condition1_2d), (condition2_1d, condition2_2d)): T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=1, seed=1) assert_equal(np.sum(cluster_p_values < 0.05), 1) T_obs, clusters, cluster_p_values, hist = permutation_cluster_test( [condition1, condition2], n_permutations=100, tail=0, seed=1) assert_equal(np.sum(cluster_p_values < 0.05), 1) # test with 2 jobs T_obs, clusters, cluster_p_values_buff, hist =\ permutation_cluster_test([condition1, condition2], n_permutations=100, tail=0, seed=1, n_jobs=2) assert_array_equal(cluster_p_values, cluster_p_values_buff)
def _conduct_analysis(data, contrast, space, avg_freq, selection, n_permutations, cluster_p_threshold, confounds, verbose): '''Conduct analysis.''' stat_info, adjacency = data['stat_info'], data['adjacency'] if 'vs' in contrast and not confounds: hi, lo = data['hi'], data['lo'] psd_ndim = hi.ndim else: hilo = data['hilo'] bdi = data['bdi'] psd_ndim = hilo.ndim # put spatial dimension last for cluster-based test if not avg_freq or psd_ndim == 3: if 'hi' in data: hi = hi.transpose((0, 2, 1)) lo = lo.transpose((0, 2, 1)) else: hilo = hilo.transpose((0, 2, 1)) # statistical analysis # -------------------- if 'pairs' not in selection: # cluster-based permutation tests for multiple comparisons stat_info.update(dict(cluster_p_threshold=cluster_p_threshold)) if 'vs' in contrast and not confounds: # t test in cluster-based permutation test from scipy.stats import t from sarna.stats import ttest_ind_welch_no_p from mne.stats.cluster_level import permutation_cluster_test # calculate t test threshold df = hi.shape[0] + lo.shape[0] - 2 threshold = t.ppf(1 - cluster_p_threshold / 2, df) # run cluster-based permutation test args = dict(threshold=threshold, n_permutations=n_permutations, stat_fun=ttest_ind_welch_no_p, verbose=verbose, out_type='mask') try: stat, clusters, pval, _ = permutation_cluster_test( [hi, lo], **args, connectivity=adjacency) except TypeError: stat, clusters, pval, _ = permutation_cluster_test( [hi, lo], **args, adjacency=adjacency) else: # regression in cluster-based permutation test from borsar.cluster import cluster_based_regression args = dict(n_permutations=n_permutations, adjacency=adjacency, alpha_threshold=cluster_p_threshold, cluster_pred=-1) stat, clusters, pval = cluster_based_regression(hilo, bdi, **args) # construct Clusters with stat_info in description return _construct_clusters(clusters, pval, stat, space, stat_info, data['info'], data['src'], data['subjects_dir'], data['subject'], data['ch_names'], data['freq']) else: # for selected pairs (two channel pairs) we don't correct for # multiple comparisons: if 'vs' in contrast and not confounds: from scipy.stats import t, ttest_ind stat, pval = ttest_ind(hi, lo, equal_var=False) else: from borsar.stats import compute_regression_t # compute regression and ignore intercept: stat, pval = compute_regression_t(hilo, bdi, return_p=True) stat, pval = stat[-1], pval[-1] stat_info.update(dict(stat=stat, pval=pval, ch_names=ch_names)) return stat_info