n_iterations = 20
deltas = np.linspace(0.5, 1.5, 20)
n_deltas = len(deltas)
"""Estimate reproducibility and true positive rate in simulated test-retest
experiments using the two-group model."""
n_reproducible = np.zeros([n_iterations, n_deltas])
n_tp = np.zeros([2, n_iterations, n_deltas]) # test and retest conditions
for i, delta in enumerate(deltas):
print('Estimating reproducibility at effect size d=%1.3f' % delta)
for j in np.arange(0, n_iterations):
"""Generate test and retest data."""
X_test = square_grid_model(nl, sl, N, delta, equal_var=True)[0]
X_retest = square_grid_model(nl, sl, N, delta, equal_var=True)[0]
"""Correct the statistics for multiple testing."""
Y_test = lsu(X_test.flatten(), alpha)
Y_test = Y_test.reshape(nl, nl)
Y_retest = lsu(X_retest.flatten(), alpha)
Y_retest = Y_retest.reshape(nl, nl)
"""Count how many discoveries were reproducible."""
n_reproducible[j, i] = np.sum(Y_retest[d:(nl - d),
d:(nl - d)][Y_test[d:(nl - d),
d:(nl - d)]])
"""Count the number of true positives in test and retest sets."""
n_tp[0, j, i] = np.sum(Y_test[d:(nl - d), d:(nl - d)])
n_tp[1, j, i] = np.sum(Y_retest[d:(nl - d), d:(nl - d)])
"""Visualize the results."""
z = norm.ppf(0.995) # 99% confidence interval
sns.set_style('darkgrid')
from numpy.random import normal
from scipy.stats import ttest_ind
import seaborn as sb
"""Reproduce the simulation in Figure 2. panel C."""
m = 100
n_iter = 99
delta = np.arange(0., 1.6, 0.1)
n_rejected_lsu, n_rejected_tst = (np.ndarray([len(delta), n_iter]),
np.ndarray([len(delta), n_iter]))
n_rejected_nc = np.ndarray([len(delta), n_iter])
for i in range(0, n_iter):
for j, d in enumerate(delta):
_, pvals = two_group_model(N=20, m=m, pi0=0, delta=d)
n_rejected_lsu[j, i] = np.sum(lsu(pvals))
n_rejected_tst[j, i] = np.sum(tst(pvals))
n_rejected_nc[j, i] = np.sum(pvals < 0.05)
print 'iteration %3d' % i
"""Visualize the results (i.e. the proportion of rejected null hypotheses
as a function of the effect size.)"""
fig = plt.figure(figsize=(6, 4))
fig.subplots_adjust(bottom=0.15)
ax = fig.add_subplot(111)
# lsu
ax.errorbar(x=delta,
y=np.mean(n_rejected_lsu / float(m), axis=1),
yerr=np.std(n_rejected_lsu / float(m), axis=1))
# tst
ax.errorbar(x=delta,
from fwer import sidak, holm_bonferroni, hochberg
from data import square_grid_model
from permutation import tfr_permutation_test
from rft import rft_2d
from util import grid_model_counts, roc
from viz import plot_grid_model
"""Generate the test data."""
nl, sl = 90, 30
N, delta = 25, 0.7
X, X_tstats, X_raw, Y_raw = square_grid_model(nl, sl, N, delta, equal_var=True)
alpha = 0.05
"""Apply each correction technique to the generated dataset."""
Y_sidak = sidak(X.flatten(), alpha=alpha)
Y_sidak = Y_sidak.reshape(nl, nl)
Y_fdr = lsu(X.flatten(), q=alpha)
Y_fdr = Y_fdr.reshape(nl, nl)
Y_qvalue, _ = qvalue(X.flatten(), threshold=alpha)
Y_qvalue = Y_qvalue.reshape(nl, nl)
Y_rft, Y_smooth, _ = rft_2d(X_tstats, fwhm=30, alpha=alpha, verbose=True)
# No reshape needed since already in correct form.
Y_tst = tst(X.flatten(), q=alpha)
Y_tst = Y_tst.reshape(nl, nl)
Y_permutation = tfr_permutation_test(X_raw,
Y_raw,
n_permutations=100,
alpha=alpha,
fs = 200
n_iter = 1000
A_pi0, B_pi0 = 0.75, 0.9
A_delta, B_delta = 1.2, 0.7
A_m, B_m = 3 * fs, 8 * fs
power_all, power_separate = [], []
for i in np.arange(0, n_iter):
"""Draw random numbers from two different mixture distributions."""
_, A_pvals = two_group_model(N=25, m=A_m, pi0=A_pi0, delta=A_delta)
_, B_pvals = two_group_model(N=25, m=B_m, pi0=B_pi0, delta=B_delta)
"""Combine the separate classes and perform one FDR."""
all_pvals = np.hstack([A_pvals, B_pvals])
significant = lsu(all_pvals)
"""Compute power of the procedure, i.e. Pr(reject H0 | H1 is true)."""
A_rejections = np.sum(significant[int(A_m * A_pi0):A_m])
B_rejections = np.sum(significant[A_m + int(B_m * B_pi0):B_m])
power_all.append((A_rejections + B_rejections) / ((1 - A_pi0) * A_m +
(1 - B_pi0) * B_m))
"""Apply FDR separately to the two parts and perform a similar
power calculation."""
A_significant, B_significant = lsu(A_pvals), lsu(B_pvals)
A_rejections = np.sum(A_significant[int(A_m * A_pi0):A_m])
B_rejections = np.sum(B_significant[int(B_m * B_pi0):B_m])
power_separate.append((A_rejections + B_rejections) / ((1 - A_pi0) * A_m +
(1 - B_pi0) * B_m))