def test_permutation_step_down_p():
"""Test cluster level permutations with step_down_p
"""
try:
try:
from sklearn.feature_extraction.image import grid_to_graph
except ImportError:
from scikits.learn.feature_extraction.image import grid_to_graph
except ImportError:
return
rng = np.random.RandomState(0)
# subjects, time points, spatial points
X = rng.randn(9, 2, 10)
# add some significant points
X[:, 0:2, 0:2] += 2 # span two time points and two spatial points
X[:, 1, 5:9] += 0.5 # span four time points with 4x smaller amplitude
thresh = 2
# make sure it works when we use ALL points in step-down
t, clusters, p, H0 = \
permutation_cluster_1samp_test(X, threshold=thresh,
step_down_p=1.0)
# make sure using step-down will actually yield improvements sometimes
t, clusters, p_old, H0 = \
permutation_cluster_1samp_test(X, threshold=thresh,
step_down_p=0.0)
assert_equal(np.sum(p_old < 0.05), 1) # just spatial cluster
t, clusters, p_new, H0 = \
permutation_cluster_1samp_test(X, threshold=thresh,
step_down_p=0.05)
assert_equal(np.sum(p_new < 0.05), 2) # time one rescued
assert_true(np.all(p_old >= p_new))
def test_permutation_step_down_p():
"""Test cluster level permutations with step_down_p."""
try:
try:
from sklearn.feature_extraction.image import grid_to_graph
except ImportError:
from scikits.learn.feature_extraction.image import grid_to_graph # noqa: F401,E501
except ImportError:
return
rng = np.random.RandomState(0)
# subjects, time points, spatial points
X = rng.randn(9, 2, 10)
# add some significant points
X[:, 0:2, 0:2] += 2 # span two time points and two spatial points
X[:, 1, 5:9] += 0.5 # span four time points with 4x smaller amplitude
thresh = 2
# make sure it works when we use ALL points in step-down
t, clusters, p, H0 = \
permutation_cluster_1samp_test(X, threshold=thresh,
step_down_p=1.0)
# make sure using step-down will actually yield improvements sometimes
t, clusters, p_old, H0 = \
permutation_cluster_1samp_test(X, threshold=thresh,
step_down_p=0.0)
assert_equal(np.sum(p_old < 0.05), 1) # just spatial cluster
t, clusters, p_new, H0 = \
permutation_cluster_1samp_test(X, threshold=thresh,
step_down_p=0.05)
assert_equal(np.sum(p_new < 0.05), 2) # time one rescued
assert_true(np.all(p_old >= p_new))
def test_cache_dir():
"""Test use of cache dir
"""
tempdir = _TempDir()
orig_dir = os.getenv('MNE_CACHE_DIR', None)
orig_size = os.getenv('MNE_MEMMAP_MIN_SIZE', None)
rng = np.random.RandomState(0)
X = rng.randn(9, 2, 10)
try:
os.environ['MNE_MEMMAP_MIN_SIZE'] = '1K'
os.environ['MNE_CACHE_DIR'] = tempdir
# Fix error for #1507: in-place when memmapping
with catch_logging() as log_file:
permutation_cluster_1samp_test(
X, buffer_size=None, n_jobs=2, n_permutations=1,
seed=0, stat_fun=ttest_1samp_no_p, verbose=False)
# ensure that non-independence yields warning
stat_fun = partial(ttest_1samp_no_p, sigma=1e-3)
assert_true('independently' not in log_file.getvalue())
with warnings.catch_warnings(record=True): # independently
permutation_cluster_1samp_test(
X, buffer_size=10, n_jobs=2, n_permutations=1,
seed=0, stat_fun=stat_fun, verbose=False)
assert_true('independently' in log_file.getvalue())
finally:
if orig_dir is not None:
os.environ['MNE_CACHE_DIR'] = orig_dir
else:
del os.environ['MNE_CACHE_DIR']
if orig_size is not None:
os.environ['MNE_MEMMAP_MIN_SIZE'] = orig_size
else:
del os.environ['MNE_MEMMAP_MIN_SIZE']
def test_cache_dir():
"""Test use of cache dir
"""
tempdir = _TempDir()
orig_dir = os.getenv("MNE_CACHE_DIR", None)
orig_size = os.getenv("MNE_MEMMAP_MIN_SIZE", None)
rng = np.random.RandomState(0)
X = rng.randn(9, 2, 10)
log_file = op.join(tempdir, "log.txt")
try:
os.environ["MNE_MEMMAP_MIN_SIZE"] = "1K"
os.environ["MNE_CACHE_DIR"] = tempdir
# Fix error for #1507: in-place when memmapping
permutation_cluster_1samp_test(
X, buffer_size=None, n_jobs=2, n_permutations=1, seed=0, stat_fun=ttest_1samp_no_p, verbose=False
)
# ensure that non-independence yields warning
stat_fun = partial(ttest_1samp_no_p, sigma=1e-3)
set_log_file(log_file)
permutation_cluster_1samp_test(
X, buffer_size=10, n_jobs=2, n_permutations=1, seed=0, stat_fun=stat_fun, verbose=False
)
with open(log_file, "r") as fid:
assert_true("independently" in "".join(fid.readlines()))
finally:
if orig_dir is not None:
os.environ["MNE_CACHE_DIR"] = orig_dir
else:
del os.environ["MNE_CACHE_DIR"]
if orig_size is not None:
os.environ["MNE_MEMMAP_MIN_SIZE"] = orig_size
else:
del os.environ["MNE_MEMMAP_MIN_SIZE"]
set_log_file(None)
def test_permutation_step_down_p(numba_conditional):
"""Test cluster level permutations with step_down_p."""
rng = np.random.RandomState(0)
# subjects, time points, spatial points
X = rng.randn(9, 2, 10)
# add some significant points
X[:, 0:2, 0:2] += 2 # span two time points and two spatial points
X[:, 1, 5:9] += 0.5 # span four time points with 4x smaller amplitude
thresh = 2
# make sure it works when we use ALL points in step-down
t, clusters, p, H0 = \
permutation_cluster_1samp_test(X, threshold=thresh,
step_down_p=1.0, out_type='mask')
# make sure using step-down will actually yield improvements sometimes
t, clusters, p_old, H0 = \
permutation_cluster_1samp_test(X, threshold=thresh,
step_down_p=0.0, out_type='mask')
assert_equal(np.sum(p_old < 0.05), 1) # just spatial cluster
p_min = np.min(p_old)
assert_allclose(p_min, 0.003906, atol=1e-6)
t, clusters, p_new, H0 = \
permutation_cluster_1samp_test(X, threshold=thresh,
step_down_p=0.05, out_type='mask')
assert_equal(np.sum(p_new < 0.05), 2) # time one rescued
assert np.all(p_old >= p_new)
p_next = p_new[(p_new > 0.004) & (p_new < 0.05)][0]
assert_allclose(p_next, 0.015625, atol=1e-6)
def test_cluster_permutation_t_test():
"""Test cluster level permutations T-test."""
for condition1 in (condition1_1d, condition1_2d):
# these are so significant we can get away with fewer perms
T_obs, clusters, cluster_p_values, hist =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=0, seed=1)
assert_equal(np.sum(cluster_p_values < 0.05), 1)
T_obs_pos, c_1, cluster_p_values_pos, _ =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=1, threshold=1.67, seed=1)
T_obs_neg, _, cluster_p_values_neg, _ =\
permutation_cluster_1samp_test(-condition1, n_permutations=100,
tail=-1, threshold=-1.67, seed=1)
assert_array_equal(T_obs_pos, -T_obs_neg)
assert_array_equal(cluster_p_values_pos < 0.05,
cluster_p_values_neg < 0.05)
# test with 2 jobs
T_obs_neg, _, cluster_p_values_neg_buff, _ = \
permutation_cluster_1samp_test(-condition1, n_permutations=100,
tail=-1, threshold=-1.67, seed=1,
n_jobs=2)
assert_array_equal(cluster_p_values_neg, cluster_p_values_neg_buff)
def test_thresholds(numba_conditional):
"""Test automatic threshold calculations."""
# within subjects
rng = np.random.RandomState(0)
X = rng.randn(10, 1, 1) + 0.08
want_thresh = -stats.t.ppf(0.025, len(X) - 1)
assert 0.03 < stats.ttest_1samp(X[:, 0, 0], 0)[1] < 0.05
my_fun = partial(ttest_1samp_no_p)
with catch_logging() as log:
with pytest.warns(RuntimeWarning, match='threshold is only valid'):
out = permutation_cluster_1samp_test(X,
stat_fun=my_fun,
seed=0,
verbose=True,
out_type='mask')
log = log.getvalue()
assert str(want_thresh)[:6] in log
assert len(out[1]) == 1 # 1 cluster
assert_allclose(out[2], 0.033203, atol=1e-6)
# between subjects
Y = rng.randn(10, 1, 1)
Z = rng.randn(10, 1, 1) - 0.7
X = [X, Y, Z]
want_thresh = stats.f.ppf(1. - 0.05, 2, sum(len(a) for a in X) - len(X))
p = stats.f_oneway(*X)[1]
assert 0.03 < p < 0.05
my_fun = partial(f_oneway) # just to make the check fail
with catch_logging() as log:
with pytest.warns(RuntimeWarning, match='threshold is only valid'):
out = permutation_cluster_test(X,
tail=1,
stat_fun=my_fun,
seed=0,
verbose=True,
out_type='mask')
log = log.getvalue()
assert str(want_thresh)[:6] in log
assert len(out[1]) == 1 # 1 cluster
assert_allclose(out[2], 0.041992, atol=1e-6)
with pytest.warns(RuntimeWarning, match='Ignoring argument "tail"'):
permutation_cluster_test(X, tail=0, out_type='mask')
# nan handling in TFCE
X = np.repeat(X[0], 2, axis=1)
X[:, 1] = 0
with pytest.warns(RuntimeWarning, match='invalid value'): # NumPy
out = permutation_cluster_1samp_test(X,
seed=0,
threshold=dict(start=0, step=0.1),
out_type='mask')
assert (out[2] < 0.05).any()
assert not (out[2] < 0.05).all()
X[:, 0] = 0
with pytest.raises(RuntimeError, match='finite'):
with np.errstate(invalid='ignore'):
permutation_cluster_1samp_test(X,
seed=0,
threshold=dict(start=0, step=0.1),
buffer_size=None,
out_type='mask')
def test_cluster_permutation_t_test():
"""Test cluster level permutations T-test."""
for condition1 in (condition1_1d, condition1_2d):
T_obs, clusters, cluster_p_values, hist =\
permutation_cluster_1samp_test(condition1, n_permutations=500,
tail=0, seed=1)
assert_equal(np.sum(cluster_p_values < 0.05), 1)
T_obs_pos, c_1, cluster_p_values_pos, _ =\
permutation_cluster_1samp_test(condition1, n_permutations=500,
tail=1, threshold=1.67, seed=1)
T_obs_neg, _, cluster_p_values_neg, _ =\
permutation_cluster_1samp_test(-condition1, n_permutations=500,
tail=-1, threshold=-1.67, seed=1)
assert_array_equal(T_obs_pos, -T_obs_neg)
assert_array_equal(cluster_p_values_pos < 0.05,
cluster_p_values_neg < 0.05)
print c_1
# test with 2 jobs
T_obs_neg, _, cluster_p_values_neg_buff, _ = \
permutation_cluster_1samp_test(-condition1, n_permutations=500,
tail=-1, threshold=-1.67, seed=1,
n_jobs=2)
assert_array_equal(cluster_p_values_neg, cluster_p_values_neg_buff)
def test_cache_dir(tmpdir, numba_conditional):
"""Test use of cache dir."""
tempdir = str(tmpdir)
orig_dir = os.getenv('MNE_CACHE_DIR', None)
orig_size = os.getenv('MNE_MEMMAP_MIN_SIZE', None)
rng = np.random.RandomState(0)
X = rng.randn(9, 2, 10)
try:
os.environ['MNE_MEMMAP_MIN_SIZE'] = '1K'
os.environ['MNE_CACHE_DIR'] = tempdir
# Fix error for #1507: in-place when memmapping
with catch_logging() as log_file:
permutation_cluster_1samp_test(
X, buffer_size=None, n_jobs=2, n_permutations=1,
seed=0, stat_fun=ttest_1samp_no_p, verbose=False)
assert 'independently' not in log_file.getvalue()
# ensure that non-independence yields warning
stat_fun = partial(ttest_1samp_no_p, sigma=1e-3)
if check_version('numpy', '1.17'):
random_state = np.random.default_rng(0)
else:
random_state = 0
with pytest.warns(RuntimeWarning, match='independently'):
permutation_cluster_1samp_test(
X, buffer_size=10, n_jobs=2, n_permutations=1,
seed=random_state, stat_fun=stat_fun, verbose=False)
finally:
if orig_dir is not None:
os.environ['MNE_CACHE_DIR'] = orig_dir
else:
del os.environ['MNE_CACHE_DIR']
if orig_size is not None:
os.environ['MNE_MEMMAP_MIN_SIZE'] = orig_size
else:
del os.environ['MNE_MEMMAP_MIN_SIZE']
def test_cache_dir():
"""Test use of cache dir."""
tempdir = _TempDir()
orig_dir = os.getenv('MNE_CACHE_DIR', None)
orig_size = os.getenv('MNE_MEMMAP_MIN_SIZE', None)
rng = np.random.RandomState(0)
X = rng.randn(9, 2, 10)
try:
os.environ['MNE_MEMMAP_MIN_SIZE'] = '1K'
os.environ['MNE_CACHE_DIR'] = tempdir
# Fix error for #1507: in-place when memmapping
with catch_logging() as log_file:
permutation_cluster_1samp_test(
X, buffer_size=None, n_jobs=2, n_permutations=1,
seed=0, stat_fun=ttest_1samp_no_p, verbose=False)
# ensure that non-independence yields warning
stat_fun = partial(ttest_1samp_no_p, sigma=1e-3)
assert_true('independently' not in log_file.getvalue())
with warnings.catch_warnings(record=True): # independently
permutation_cluster_1samp_test(
X, buffer_size=10, n_jobs=2, n_permutations=1,
seed=0, stat_fun=stat_fun, verbose=False)
assert_true('independently' in log_file.getvalue())
finally:
if orig_dir is not None:
os.environ['MNE_CACHE_DIR'] = orig_dir
else:
del os.environ['MNE_CACHE_DIR']
if orig_size is not None:
os.environ['MNE_MEMMAP_MIN_SIZE'] = orig_size
else:
del os.environ['MNE_MEMMAP_MIN_SIZE']
def test_tfce_thresholds():
rng = np.random.RandomState(0)
data = rng.randn(7, 10, 1) - 0.5
# if tail==-1, step must also be negative
assert_raises(ValueError, permutation_cluster_1samp_test, data, tail=-1,
threshold=dict(start=0, step=0.1))
# this works (smoke test)
permutation_cluster_1samp_test(data, tail=-1,
threshold=dict(start=0, step=-0.1))
# thresholds must be monotonically increasing
assert_raises(ValueError, permutation_cluster_1samp_test, data, tail=1,
threshold=dict(start=1, step=-0.5))
def test_tfce_thresholds(numba_conditional):
"""Test TFCE thresholds."""
rng = np.random.RandomState(0)
data = rng.randn(7, 10, 1) - 0.5
# if tail==-1, step must also be negative
with pytest.raises(ValueError, match='must be < 0 for tail == -1'):
permutation_cluster_1samp_test(data,
tail=-1,
out_type='mask',
threshold=dict(start=0, step=0.1))
# this works (smoke test)
permutation_cluster_1samp_test(data,
tail=-1,
out_type='mask',
threshold=dict(start=0, step=-0.1))
# thresholds must be monotonically increasing
with pytest.raises(ValueError, match='must be monotonically increasing'):
permutation_cluster_1samp_test(data,
tail=1,
out_type='mask',
threshold=dict(start=1, step=-0.5))
# Should work with 2D data too
permutation_cluster_1samp_test(X=data[..., 0],
threshold=dict(start=0, step=0.2))
def test_thresholds(numba_conditional):
"""Test automatic threshold calculations."""
# within subjects
rng = np.random.RandomState(0)
X = rng.randn(10, 1, 1) + 0.08
want_thresh = -stats.t.ppf(0.025, len(X) - 1)
assert 0.03 < stats.ttest_1samp(X[:, 0, 0], 0)[1] < 0.05
my_fun = partial(ttest_1samp_no_p)
with catch_logging() as log:
with pytest.warns(RuntimeWarning, match='threshold is only valid'):
out = permutation_cluster_1samp_test(X, stat_fun=my_fun,
seed=0, verbose=True)
log = log.getvalue()
assert str(want_thresh)[:6] in log
assert len(out[1]) == 1 # 1 cluster
assert_allclose(out[2], 0.033203, atol=1e-6)
# between subjects
Y = rng.randn(10, 1, 1)
Z = rng.randn(10, 1, 1) - 0.7
X = [X, Y, Z]
want_thresh = stats.f.ppf(1. - 0.05, 2, sum(len(a) for a in X) - len(X))
p = stats.f_oneway(*X)[1]
assert 0.03 < p < 0.05
my_fun = partial(f_oneway) # just to make the check fail
with catch_logging() as log:
with pytest.warns(RuntimeWarning, match='threshold is only valid'):
out = permutation_cluster_test(X, tail=1, stat_fun=my_fun,
seed=0, verbose=True)
log = log.getvalue()
assert str(want_thresh)[:6] in log
assert len(out[1]) == 1 # 1 cluster
assert_allclose(out[2], 0.041992, atol=1e-6)
with pytest.warns(RuntimeWarning, match='Ignoring argument "tail"'):
permutation_cluster_test(X, tail=0)
def test_thresholds():
"""Test automatic threshold calculations."""
# within subjects
rng = np.random.RandomState(0)
X = rng.randn(10, 1, 1) + 0.08
want_thresh = -stats.t.ppf(0.025, len(X) - 1)
assert 0.03 < stats.ttest_1samp(X[:, 0, 0], 0)[1] < 0.05
my_fun = partial(ttest_1samp_no_p)
with catch_logging() as log:
with pytest.warns(RuntimeWarning, match='threshold is only valid'):
out = permutation_cluster_1samp_test(X, stat_fun=my_fun,
verbose=True)
log = log.getvalue()
assert str(want_thresh)[:6] in log
assert len(out[1]) == 1 # 1 cluster
assert 0.03 < out[2] < 0.05
# between subjects
Y = rng.randn(10, 1, 1)
Z = rng.randn(10, 1, 1) - 0.7
X = [X, Y, Z]
want_thresh = stats.f.ppf(1. - 0.05, 2, sum(len(a) for a in X) - len(X))
p = stats.f_oneway(*X)[1]
assert 0.03 < p < 0.05
my_fun = partial(f_oneway) # just to make the check fail
with catch_logging() as log:
with pytest.warns(RuntimeWarning, match='threshold is only valid'):
out = permutation_cluster_test(X, tail=1, stat_fun=my_fun,
verbose=True)
log = log.getvalue()
assert str(want_thresh)[:6] in log
assert len(out[1]) == 1 # 1 cluster
assert 0.03 < out[2] < 0.05
with pytest.warns(RuntimeWarning, match='Ignoring argument "tail"'):
permutation_cluster_test(X, tail=0)
def test_cluster_permutation_t_test(numba_conditional):
"""Test cluster level permutations T-test."""
condition1_1d, condition2_1d, condition1_2d, condition2_2d = \
_get_conditions()
# use a very large sigma to make sure Ts are not independent
stat_funs = [ttest_1samp_no_p,
partial(ttest_1samp_no_p, sigma=1e-1)]
for stat_fun in stat_funs:
for condition1, p in ((condition1_1d, 0.01),
(condition1_2d, 0.01)):
# these are so significant we can get away with fewer perms
T_obs, clusters, cluster_p_values, hist =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=0, seed=1,
buffer_size=None)
assert_equal(np.sum(cluster_p_values < 0.05), 1)
p_min = np.min(cluster_p_values)
assert_allclose(p_min, p, atol=1e-6)
T_obs_pos, c_1, cluster_p_values_pos, _ =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=1, threshold=1.67, seed=1,
stat_fun=stat_fun,
buffer_size=None)
T_obs_neg, _, cluster_p_values_neg, _ =\
permutation_cluster_1samp_test(-condition1, n_permutations=100,
tail=-1, threshold=-1.67,
seed=1, stat_fun=stat_fun,
buffer_size=None)
assert_array_equal(T_obs_pos, -T_obs_neg)
assert_array_equal(cluster_p_values_pos < 0.05,
cluster_p_values_neg < 0.05)
# test with 2 jobs and buffer_size enabled
buffer_size = condition1.shape[1] // 10
with pytest.warns(None): # sometimes "independently"
T_obs_neg_buff, _, cluster_p_values_neg_buff, _ = \
permutation_cluster_1samp_test(
-condition1, n_permutations=100, tail=-1,
threshold=-1.67, seed=1, n_jobs=2, stat_fun=stat_fun,
buffer_size=buffer_size)
assert_array_equal(T_obs_neg, T_obs_neg_buff)
assert_array_equal(cluster_p_values_neg, cluster_p_values_neg_buff)
def test_cluster_permutation_t_test():
"""Test cluster level permutations T-test."""
my_condition1 = condition1[:, :, None] # to test 2D also
T_obs, clusters, cluster_p_values, hist = permutation_cluster_1samp_test(
my_condition1, n_permutations=500, tail=0)
assert_equal(np.sum(cluster_p_values < 0.05), 1)
T_obs_pos, _, cluster_p_values_pos, _ = permutation_cluster_1samp_test(
my_condition1, n_permutations=500, tail=1,
threshold=1.67)
T_obs_neg, _, cluster_p_values_neg, _ = permutation_cluster_1samp_test(
-my_condition1, n_permutations=500, tail=-1,
threshold=-1.67)
assert_array_equal(T_obs_pos, -T_obs_neg)
assert_array_equal(cluster_p_values_pos < 0.05,
cluster_p_values_neg < 0.05)
def test_cluster_permutation_t_test():
"""Test cluster level permutations T-test
"""
condition1_1d, condition2_1d, condition1_2d, condition2_2d = \
_get_conditions()
# use a very large sigma to make sure Ts are not independent
stat_funs = [ttest_1samp_no_p,
partial(ttest_1samp_no_p, sigma=1e-1)]
for stat_fun in stat_funs:
for condition1 in (condition1_1d, condition1_2d):
# these are so significant we can get away with fewer perms
T_obs, clusters, cluster_p_values, hist =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=0, seed=1,
buffer_size=None)
assert_equal(np.sum(cluster_p_values < 0.05), 1)
T_obs_pos, c_1, cluster_p_values_pos, _ =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=1, threshold=1.67, seed=1,
stat_fun=stat_fun,
buffer_size=None)
T_obs_neg, _, cluster_p_values_neg, _ =\
permutation_cluster_1samp_test(-condition1, n_permutations=100,
tail=-1, threshold=-1.67,
seed=1, stat_fun=stat_fun,
buffer_size=None)
assert_array_equal(T_obs_pos, -T_obs_neg)
assert_array_equal(cluster_p_values_pos < 0.05,
cluster_p_values_neg < 0.05)
# test with 2 jobs and buffer_size enabled
buffer_size = condition1.shape[1] // 10
T_obs_neg_buff, _, cluster_p_values_neg_buff, _ = \
permutation_cluster_1samp_test(-condition1, n_permutations=100,
tail=-1, threshold=-1.67,
seed=1, n_jobs=2,
stat_fun=stat_fun,
buffer_size=buffer_size)
assert_array_equal(T_obs_neg, T_obs_neg_buff)
assert_array_equal(cluster_p_values_neg, cluster_p_values_neg_buff)
def test_cluster_permutation_t_test_with_connectivity():
"""Test cluster level permutations T-test with connectivity matrix."""
try:
try:
from sklearn.feature_extraction.image import grid_to_graph
except ImportError:
from scikits.learn.feature_extraction.image import grid_to_graph
except ImportError:
return
out = permutation_cluster_1samp_test(condition1, n_permutations=500)
connectivity = grid_to_graph(1, condition1.shape[1])
out_connectivity = permutation_cluster_1samp_test(condition1,
n_permutations=500, connectivity=connectivity)
assert_array_equal(out[0], out_connectivity[0])
for a, b in zip(out_connectivity[1], out[1]):
assert_true(np.sum(out[0][a]) == np.sum(out[0][b]))
assert_true(np.all(a[b]))
def test_cluster_permutation_t_test(numba_conditional, stat_fun):
"""Test cluster level permutations T-test."""
condition1_1d, condition2_1d, condition1_2d, condition2_2d = \
_get_conditions()
# use a very large sigma to make sure Ts are not independent
for condition1, p in ((condition1_1d, 0.01), (condition1_2d, 0.01)):
# these are so significant we can get away with fewer perms
T_obs, clusters, cluster_p_values, hist =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=0, seed=1, out_type='mask',
buffer_size=None)
assert_equal(np.sum(cluster_p_values < 0.05), 1)
p_min = np.min(cluster_p_values)
assert_allclose(p_min, p, atol=1e-6)
T_obs_pos, c_1, cluster_p_values_pos, _ =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=1, threshold=1.67, seed=1,
stat_fun=stat_fun, out_type='mask',
buffer_size=None)
T_obs_neg, _, cluster_p_values_neg, _ =\
permutation_cluster_1samp_test(-condition1, n_permutations=100,
tail=-1, threshold=-1.67,
seed=1, stat_fun=stat_fun,
buffer_size=None, out_type='mask')
assert_array_equal(T_obs_pos, -T_obs_neg)
assert_array_equal(cluster_p_values_pos < 0.05,
cluster_p_values_neg < 0.05)
# test with 2 jobs and buffer_size enabled
buffer_size = condition1.shape[1] // 10
with pytest.warns(None): # sometimes "independently"
T_obs_neg_buff, _, cluster_p_values_neg_buff, _ = \
permutation_cluster_1samp_test(
-condition1, n_permutations=100, tail=-1, out_type='mask',
threshold=-1.67, seed=1, n_jobs=2, stat_fun=stat_fun,
buffer_size=buffer_size)
assert_array_equal(T_obs_neg, T_obs_neg_buff)
assert_array_equal(cluster_p_values_neg, cluster_p_values_neg_buff)
# Bad stat_fun
with pytest.raises(TypeError, match='must be .* ndarray'):
permutation_cluster_1samp_test(condition1,
threshold=1,
stat_fun=lambda x: None,
out_type='mask')
with pytest.raises(ValueError, match='not compatible'):
permutation_cluster_1samp_test(condition1,
threshold=1,
stat_fun=lambda x: stat_fun(x)[:-1],
out_type='mask')
def test_cluster_permutation_t_test():
"""Test cluster level permutations T-test."""
condition1_1d, condition2_1d, condition1_2d, condition2_2d = \
_get_conditions()
# use a very large sigma to make sure Ts are not independent
stat_funs = [ttest_1samp_no_p,
partial(ttest_1samp_no_p, sigma=1e-1)]
for stat_fun in stat_funs:
for condition1 in (condition1_1d, condition1_2d):
# these are so significant we can get away with fewer perms
T_obs, clusters, cluster_p_values, hist =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=0, seed=1,
buffer_size=None)
assert_equal(np.sum(cluster_p_values < 0.05), 1)
T_obs_pos, c_1, cluster_p_values_pos, _ =\
permutation_cluster_1samp_test(condition1, n_permutations=100,
tail=1, threshold=1.67, seed=1,
stat_fun=stat_fun,
buffer_size=None)
T_obs_neg, _, cluster_p_values_neg, _ =\
permutation_cluster_1samp_test(-condition1, n_permutations=100,
tail=-1, threshold=-1.67,
seed=1, stat_fun=stat_fun,
buffer_size=None)
assert_array_equal(T_obs_pos, -T_obs_neg)
assert_array_equal(cluster_p_values_pos < 0.05,
cluster_p_values_neg < 0.05)
# test with 2 jobs and buffer_size enabled
buffer_size = condition1.shape[1] // 10
with warnings.catch_warnings(record=True): # independently
T_obs_neg_buff, _, cluster_p_values_neg_buff, _ = \
permutation_cluster_1samp_test(
-condition1, n_permutations=100, tail=-1,
threshold=-1.67, seed=1, n_jobs=2, stat_fun=stat_fun,
buffer_size=buffer_size)
assert_array_equal(T_obs_neg, T_obs_neg_buff)
assert_array_equal(cluster_p_values_neg, cluster_p_values_neg_buff)
def test_permutation_large_n_samples():
"""Test that non-replacement works with large N."""
X = np.random.RandomState(0).randn(72, 1) + 1
for n_samples in (11, 72):
tails = (0, 1) if n_samples <= 20 else (0,)
for tail in tails:
H0 = permutation_cluster_1samp_test(
X[:n_samples], threshold=1e-4, tail=tail)[-1]
assert H0.shape == (1024,)
assert len(np.unique(H0)) >= 1024 - (H0 == 0).sum()
def test_cache_dir():
"""Test use of cache dir
"""
tempdir = _TempDir()
orig_dir = os.getenv('MNE_CACHE_DIR', None)
orig_size = os.getenv('MNE_MEMMAP_MIN_SIZE', None)
rng = np.random.RandomState(0)
X = rng.randn(9, 2, 10)
log_file = op.join(tempdir, 'log.txt')
try:
os.environ['MNE_MEMMAP_MIN_SIZE'] = '1K'
os.environ['MNE_CACHE_DIR'] = tempdir
# Fix error for #1507: in-place when memmapping
permutation_cluster_1samp_test(X,
buffer_size=None,
n_jobs=2,
n_permutations=1,
seed=0,
stat_fun=ttest_1samp_no_p,
verbose=False)
# ensure that non-independence yields warning
stat_fun = partial(ttest_1samp_no_p, sigma=1e-3)
set_log_file(log_file)
permutation_cluster_1samp_test(X,
buffer_size=10,
n_jobs=2,
n_permutations=1,
seed=0,
stat_fun=stat_fun,
verbose=False)
with open(log_file, 'r') as fid:
assert_true('independently' in ''.join(fid.readlines()))
finally:
if orig_dir is not None:
os.environ['MNE_CACHE_DIR'] = orig_dir
else:
del os.environ['MNE_CACHE_DIR']
if orig_size is not None:
os.environ['MNE_MEMMAP_MIN_SIZE'] = orig_size
else:
del os.environ['MNE_MEMMAP_MIN_SIZE']
set_log_file(None)
def test_output_equiv(shape, out_type, adjacency):
"""Test equivalence of output types."""
rng = np.random.RandomState(0)
n_subjects = 10
data = rng.randn(n_subjects, *shape)
data -= data.mean(axis=0, keepdims=True)
data[:, 2:4] += 2
data[:, 6:9] += 2
want_mask = np.zeros(shape, int)
want_mask[2:4] = 1
want_mask[6:9] = 2
if adjacency is not None:
assert adjacency == 'sparse'
adjacency = combine_adjacency(*shape)
clusters = permutation_cluster_1samp_test(X=data,
n_permutations=1,
adjacency=adjacency,
out_type=out_type)[1]
got_mask = np.zeros_like(want_mask)
for n, clu in enumerate(clusters, 1):
if out_type == 'mask':
if len(shape) == 1 and adjacency is None:
assert isinstance(clu, tuple)
assert len(clu) == 1
assert isinstance(clu[0], slice)
else:
assert isinstance(clu, np.ndarray)
assert clu.dtype == bool
assert clu.shape == shape
got_mask[clu] = n
else:
assert isinstance(clu, tuple)
for c in clu:
assert isinstance(c, np.ndarray)
assert c.dtype.kind == 'i'
assert out_type == 'indices'
got_mask[np.ix_(*clu)] = n
assert_array_equal(got_mask, want_mask)
def test_cluster_permutation_t_test_with_connectivity():
"""Test cluster level permutations T-test with connectivity matrix."""
try:
try:
from sklearn.feature_extraction.image import grid_to_graph
except ImportError:
from scikits.learn.feature_extraction.image import grid_to_graph
except ImportError:
return
# we don't care about p-values in any of these, so do fewer permutations
out = permutation_cluster_1samp_test(condition1_1d, n_permutations=50)
connectivity = grid_to_graph(1, condition1_1d.shape[1])
out_connectivity = permutation_cluster_1samp_test(condition1_1d,
n_permutations=50, connectivity=connectivity)
assert_array_equal(out[0], out_connectivity[0])
for a, b in zip(out_connectivity[1], out[1]):
assert_true(np.sum(out[0][a]) == np.sum(out[0][b]))
assert_true(np.all(a[b]))
# test spatio-temporal with no time connectivity (repeat spatial pattern)
connectivity_2 = sparse.coo_matrix(
linalg.block_diag(connectivity.asfptype().todense(),
connectivity.asfptype().todense()))
condition1_2 = np.concatenate((condition1_1d,
condition1_1d), axis=1)
out_connectivity_2 = permutation_cluster_1samp_test(condition1_2,
n_permutations=50, connectivity=connectivity_2)
# make sure we were operating on the same values
split = len(out[0])
assert_array_equal(out[0], out_connectivity_2[0][:split])
assert_array_equal(out[0], out_connectivity_2[0][split:])
# make sure we really got 2x the number of original clusters
n_clust_orig = len(out[1])
assert_true(len(out_connectivity_2[1]) == 2 * n_clust_orig)
# Make sure that we got the old ones back
n_pts = condition1_1d.shape[1]
data_1 = set([np.sum(out[0][b[:n_pts]]) for b in out[1]])
data_2 = set([np.sum(out_connectivity_2[0][a[:n_pts]]) for a in
out_connectivity_2[1][:]])
assert_true(len(data_1.intersection(data_2)) == len(data_1))
# now use the other algorithm
condition1_3 = np.reshape(condition1_2, (40, 2, 350))
out_connectivity_3 = spatio_temporal_cluster_1samp_test(
condition1_3, n_permutations=50,
connectivity=connectivity, max_step=0,
threshold=1.67, check_disjoint=True)
# make sure we were operating on the same values
split = len(out[0])
assert_array_equal(out[0], out_connectivity_3[0][0])
assert_array_equal(out[0], out_connectivity_3[0][1])
# make sure we really got 2x the number of original clusters
assert_true(len(out_connectivity_3[1]) == 2 * n_clust_orig)
# Make sure that we got the old ones back
data_1 = set([np.sum(out[0][b[:n_pts]]) for b in out[1]])
data_2 = set([np.sum(out_connectivity_3[0][a[0], a[1]]) for a in
out_connectivity_3[1]])
assert_true(len(data_1.intersection(data_2)) == len(data_1))
# test new versus old method old method
out_connectivity_4 = spatio_temporal_cluster_1samp_test(
condition1_3, n_permutations=50,
connectivity=connectivity, max_step=2,
threshold=1.67)
out_connectivity_5 = spatio_temporal_cluster_1samp_test(
condition1_3, n_permutations=50,
connectivity=connectivity, max_step=1,
threshold=1.67)
# clutsers could be in a different order
sums_4 = [np.sum(out_connectivity_4[0][a]) for a in out_connectivity_4[1]]
sums_5 = [np.sum(out_connectivity_4[0][a]) for a in out_connectivity_5[1]]
sums_4 = np.sort(sums_4)
sums_5 = np.sort(sums_5)
assert_array_almost_equal(sums_4, sums_5)
def test_permutation_connectivity_equiv():
"""Test cluster level permutations with and without connectivity."""
try:
try:
from sklearn.feature_extraction.image import grid_to_graph
except ImportError:
from scikits.learn.feature_extraction.image import grid_to_graph
except ImportError:
return
rng = np.random.RandomState(0)
# subjects, time points, spatial points
n_time = 2
n_space = 4
X = rng.randn(6, n_time, n_space)
# add some significant points
X[:, :, 0:2] += 10 # span two time points and two spatial points
X[:, 1, 3] += 20 # span one time point
max_steps = [1, 1, 1, 2]
# This will run full algorithm in two ways, then the ST-algorithm in 2 ways
# All of these should give the same results
conns = [
None,
grid_to_graph(n_time, n_space),
grid_to_graph(1, n_space),
grid_to_graph(1, n_space)
]
stat_map = None
thresholds = [2, dict(start=1.5, step=1.0)]
sig_counts = [2, 5]
sdps = [0, 0.05, 0.05]
ots = ['mask', 'mask', 'indices']
stat_fun = partial(ttest_1samp_no_p, sigma=1e-3)
for thresh, count in zip(thresholds, sig_counts):
cs = None
ps = None
for max_step, conn in zip(max_steps, conns):
for sdp, ot in zip(sdps, ots):
t, clusters, p, H0 = \
permutation_cluster_1samp_test(
X, threshold=thresh, connectivity=conn, n_jobs=2,
max_step=max_step, stat_fun=stat_fun,
step_down_p=sdp, out_type=ot)
# make sure our output datatype is correct
if ot == 'mask':
assert_true(isinstance(clusters[0], np.ndarray))
assert_true(clusters[0].dtype == bool)
assert_array_equal(clusters[0].shape, X.shape[1:])
else: # ot == 'indices'
assert_true(isinstance(clusters[0], tuple))
# make sure all comparisons were done; for TFCE, no perm
# should come up empty
if count == 8:
assert_true(not np.any(H0 == 0))
inds = np.where(p < 0.05)[0]
assert_true(len(inds) == count)
this_cs = [clusters[ii] for ii in inds]
this_ps = p[inds]
this_stat_map = np.zeros((n_time, n_space), dtype=bool)
for ci, c in enumerate(this_cs):
if isinstance(c, tuple):
this_c = np.zeros((n_time, n_space), bool)
for x, y in zip(c[0], c[1]):
this_stat_map[x, y] = True
this_c[x, y] = True
this_cs[ci] = this_c
c = this_c
this_stat_map[c] = True
if cs is None:
ps = this_ps
cs = this_cs
if stat_map is None:
stat_map = this_stat_map
assert_array_equal(ps, this_ps)
assert_true(len(cs) == len(this_cs))
for c1, c2 in zip(cs, this_cs):
assert_array_equal(c1, c2)
assert_array_equal(stat_map, this_stat_map)
def test_cluster_permutation_t_test_with_connectivity():
"""Test cluster level permutations T-test with connectivity matrix."""
try:
try:
from sklearn.feature_extraction.image import grid_to_graph
except ImportError:
from scikits.learn.feature_extraction.image import grid_to_graph
except ImportError:
return
out = permutation_cluster_1samp_test(condition1_1d, n_permutations=500)
connectivity = grid_to_graph(1, condition1_1d.shape[1])
out_connectivity = permutation_cluster_1samp_test(
condition1_1d, n_permutations=500, connectivity=connectivity)
assert_array_equal(out[0], out_connectivity[0])
for a, b in zip(out_connectivity[1], out[1]):
assert_true(np.sum(out[0][a]) == np.sum(out[0][b]))
assert_true(np.all(a[b]))
# test spatio-temporal with no time connectivity (repeat spatial pattern)
connectivity_2 = sparse.coo_matrix(
linalg.block_diag(connectivity.asfptype().todense(),
connectivity.asfptype().todense()))
condition1_2 = np.concatenate((condition1_1d, condition1_1d), axis=1)
out_connectivity_2 = permutation_cluster_1samp_test(
condition1_2, n_permutations=500, connectivity=connectivity_2)
# make sure we were operating on the same values
split = len(out[0])
assert_array_equal(out[0], out_connectivity_2[0][:split])
assert_array_equal(out[0], out_connectivity_2[0][split:])
# make sure we really got 2x the number of original clusters
n_clust_orig = len(out[1])
assert_true(len(out_connectivity_2[1]) == 2 * n_clust_orig)
# Make sure that we got the old ones back
n_pts = condition1_1d.shape[1]
data_1 = set([np.sum(out[0][b[:n_pts]]) for b in out[1]])
data_2 = set([
np.sum(out_connectivity_2[0][a[:n_pts]])
for a in out_connectivity_2[1][:]
])
assert_true(len(data_1.intersection(data_2)) == len(data_1))
# now use the other algorithm
condition1_3 = np.reshape(condition1_2, (40, 2, 350))
out_connectivity_3 = mnestats.spatio_temporal_cluster_1samp_test(
condition1_3,
n_permutations=500,
connectivity=connectivity,
max_step=0,
threshold=1.67,
check_disjoint=True)
# make sure we were operating on the same values
split = len(out[0])
assert_array_equal(out[0], out_connectivity_3[0][0])
assert_array_equal(out[0], out_connectivity_3[0][1])
# make sure we really got 2x the number of original clusters
assert_true(len(out_connectivity_3[1]) == 2 * n_clust_orig)
# Make sure that we got the old ones back
data_1 = set([np.sum(out[0][b[:n_pts]]) for b in out[1]])
data_2 = set([
np.sum(out_connectivity_3[0][a[0], a[1]])
for a in out_connectivity_3[1]
])
assert_true(len(data_1.intersection(data_2)) == len(data_1))
def test_permutation_adjacency_equiv(numba_conditional):
"""Test cluster level permutations with and without adjacency."""
from sklearn.feature_extraction.image import grid_to_graph
rng = np.random.RandomState(0)
# subjects, time points, spatial points
n_time = 2
n_space = 4
X = rng.randn(6, n_time, n_space)
# add some significant points
X[:, :, 0:2] += 10 # span two time points and two spatial points
X[:, 1, 3] += 20 # span one time point
max_steps = [1, 1, 1, 2, 1]
# This will run full algorithm in two ways, then the ST-algorithm in 2 ways
# All of these should give the same results
adjs = [
None,
grid_to_graph(n_time, n_space),
grid_to_graph(1, n_space),
grid_to_graph(1, n_space), None
]
stat_map = None
thresholds = [2, 2, 2, 2, dict(start=0.01, step=1.0)]
sig_counts = [2, 2, 2, 2, 5]
stat_fun = partial(ttest_1samp_no_p, sigma=1e-3)
cs = None
ps = None
for thresh, count, max_step, adj in zip(thresholds, sig_counts, max_steps,
adjs):
t, clusters, p, H0 = \
permutation_cluster_1samp_test(
X, threshold=thresh, adjacency=adj, n_jobs=2,
max_step=max_step, stat_fun=stat_fun, seed=0, out_type='mask')
# make sure our output datatype is correct
assert isinstance(clusters[0], np.ndarray)
assert clusters[0].dtype == bool
assert_array_equal(clusters[0].shape, X.shape[1:])
# make sure all comparisons were done; for TFCE, no perm
# should come up empty
inds = np.where(p < 0.05)[0]
assert_equal(len(inds), count)
assert_allclose(p[inds], 0.03125, atol=1e-6)
if isinstance(thresh, dict):
assert_equal(len(clusters), n_time * n_space)
assert np.all(H0 != 0)
continue
this_cs = [clusters[ii] for ii in inds]
this_ps = p[inds]
this_stat_map = np.zeros((n_time, n_space), dtype=bool)
for ci, c in enumerate(this_cs):
if isinstance(c, tuple):
this_c = np.zeros((n_time, n_space), bool)
for x, y in zip(c[0], c[1]):
this_stat_map[x, y] = True
this_c[x, y] = True
this_cs[ci] = this_c
c = this_c
this_stat_map[c] = True
if cs is None:
ps = this_ps
cs = this_cs
if stat_map is None:
stat_map = this_stat_map
assert_array_equal(ps, this_ps)
assert len(cs) == len(this_cs)
for c1, c2 in zip(cs, this_cs):
assert_array_equal(c1, c2)
assert_array_equal(stat_map, this_stat_map)
def test_permutation_connectivity_equiv():
"""Test cluster level permutations with and without connectivity."""
try:
try:
from sklearn.feature_extraction.image import grid_to_graph
except ImportError:
from scikits.learn.feature_extraction.image import grid_to_graph
except ImportError:
return
rng = np.random.RandomState(0)
# subjects, time points, spatial points
n_time = 2
n_space = 4
X = rng.randn(6, n_time, n_space)
# add some significant points
X[:, :, 0:2] += 10 # span two time points and two spatial points
X[:, 1, 3] += 20 # span one time point
max_steps = [1, 1, 1, 2, 1]
# This will run full algorithm in two ways, then the ST-algorithm in 2 ways
# All of these should give the same results
conns = [None,
grid_to_graph(n_time, n_space),
grid_to_graph(1, n_space),
grid_to_graph(1, n_space),
None]
stat_map = None
thresholds = [2, 2, 2, 2, dict(start=0.01, step=1.0)]
sig_counts = [2, 2, 2, 2, 5]
stat_fun = partial(ttest_1samp_no_p, sigma=1e-3)
cs = None
ps = None
for thresh, count, max_step, conn in zip(thresholds, sig_counts,
max_steps, conns):
t, clusters, p, H0 = \
permutation_cluster_1samp_test(
X, threshold=thresh, connectivity=conn, n_jobs=2,
max_step=max_step, stat_fun=stat_fun)
# make sure our output datatype is correct
assert_true(isinstance(clusters[0], np.ndarray))
assert_true(clusters[0].dtype == bool)
assert_array_equal(clusters[0].shape, X.shape[1:])
# make sure all comparisons were done; for TFCE, no perm
# should come up empty
inds = np.where(p < 0.05)[0]
assert_equal(len(inds), count)
if isinstance(thresh, dict):
assert_equal(len(clusters), n_time * n_space)
assert_true(np.all(H0 != 0))
continue
this_cs = [clusters[ii] for ii in inds]
this_ps = p[inds]
this_stat_map = np.zeros((n_time, n_space), dtype=bool)
for ci, c in enumerate(this_cs):
if isinstance(c, tuple):
this_c = np.zeros((n_time, n_space), bool)
for x, y in zip(c[0], c[1]):
this_stat_map[x, y] = True
this_c[x, y] = True
this_cs[ci] = this_c
c = this_c
this_stat_map[c] = True
if cs is None:
ps = this_ps
cs = this_cs
if stat_map is None:
stat_map = this_stat_map
assert_array_equal(ps, this_ps)
assert_true(len(cs) == len(this_cs))
for c1, c2 in zip(cs, this_cs):
assert_array_equal(c1, c2)
assert_array_equal(stat_map, this_stat_map)
def test_permutation_connectivity_equiv():
"""Test cluster level permutations with and without connectivity
"""
try:
try:
from sklearn.feature_extraction.image import grid_to_graph
except ImportError:
from scikits.learn.feature_extraction.image import grid_to_graph
except ImportError:
return
rng = np.random.RandomState(0)
# subjects, time points, spatial points
X = rng.randn(7, 2, 10)
# add some significant points
X[:, 0:2, 0:2] += 10 # span two time points and two spatial points
X[:, 1, 5:9] += 10 # span four time points
max_steps = [1, 1, 1, 2]
# This will run full algorithm in two ways, then the ST-algorithm in 2 ways
# All of these should give the same results
conns = [None, grid_to_graph(2, 10),
grid_to_graph(1, 10), grid_to_graph(1, 10)]
stat_map = None
thresholds = [2, dict(start=0.5, step=0.5)]
sig_counts = [2, 8]
sdps = [0, 0.05, 0.05]
ots = ['mask', 'mask', 'indices']
for thresh, count in zip(thresholds, sig_counts):
cs = None
ps = None
for max_step, conn in zip(max_steps, conns):
for stat_fun in [ttest_1samp_no_p,
partial(ttest_1samp_no_p, sigma=1e-3)]:
for sdp, ot in zip(sdps, ots):
t, clusters, p, H0 = \
permutation_cluster_1samp_test(X,
threshold=thresh,
connectivity=conn,
n_jobs=2,
max_step=max_step,
stat_fun=stat_fun,
step_down_p=sdp,
out_type=ot)
# make sure our output datatype is correct
if ot == 'mask':
assert_true(isinstance(clusters[0], np.ndarray))
assert_true(clusters[0].dtype == bool)
assert_array_equal(clusters[0].shape, X.shape[1:])
else: # ot == 'indices'
assert_true(isinstance(clusters[0], tuple))
# make sure all comparisons were done; for TFCE, no perm
# should come up empty
if count == 8:
assert_true(not np.any(H0 == 0))
inds = np.where(p < 0.05)[0]
assert_true(len(inds) == count)
this_cs = [clusters[ii] for ii in inds]
this_ps = p[inds]
this_stat_map = np.zeros((2, 10), dtype=bool)
for ci, c in enumerate(this_cs):
if isinstance(c, tuple):
this_c = np.zeros((2, 10), bool)
for x, y in zip(c[0], c[1]):
this_stat_map[x, y] = True
this_c[x, y] = True
this_cs[ci] = this_c
c = this_c
this_stat_map[c] = True
if cs is None:
ps = this_ps
cs = this_cs
if stat_map is None:
stat_map = this_stat_map
assert_array_equal(ps, this_ps)
assert_true(len(cs) == len(this_cs))
for c1, c2 in zip(cs, this_cs):
assert_array_equal(c1, c2)
assert_array_equal(stat_map, this_stat_map)
