n_iterations = 20 deltas = np.linspace(0.5, 1.5, 20) n_deltas = len(deltas) """Estimate reproducibility and true positive rate in simulated test-retest experiments using the two-group model.""" n_reproducible = np.zeros([n_iterations, n_deltas]) n_tp = np.zeros([2, n_iterations, n_deltas]) # test and retest conditions for i, delta in enumerate(deltas): print('Estimating reproducibility at effect size d=%1.3f' % delta) for j in np.arange(0, n_iterations): """Generate test and retest data.""" X_test = square_grid_model(nl, sl, N, delta, equal_var=True)[0] X_retest = square_grid_model(nl, sl, N, delta, equal_var=True)[0] """Correct the statistics for multiple testing.""" Y_test = lsu(X_test.flatten(), alpha) Y_test = Y_test.reshape(nl, nl) Y_retest = lsu(X_retest.flatten(), alpha) Y_retest = Y_retest.reshape(nl, nl) """Count how many discoveries were reproducible.""" n_reproducible[j, i] = np.sum(Y_retest[d:(nl - d), d:(nl - d)][Y_test[d:(nl - d), d:(nl - d)]]) """Count the number of true positives in test and retest sets.""" n_tp[0, j, i] = np.sum(Y_test[d:(nl - d), d:(nl - d)]) n_tp[1, j, i] = np.sum(Y_retest[d:(nl - d), d:(nl - d)]) """Visualize the results.""" z = norm.ppf(0.995) # 99% confidence interval sns.set_style('darkgrid')
from numpy.random import normal from scipy.stats import ttest_ind import seaborn as sb """Reproduce the simulation in Figure 2. panel C.""" m = 100 n_iter = 99 delta = np.arange(0., 1.6, 0.1) n_rejected_lsu, n_rejected_tst = (np.ndarray([len(delta), n_iter]), np.ndarray([len(delta), n_iter])) n_rejected_nc = np.ndarray([len(delta), n_iter]) for i in range(0, n_iter): for j, d in enumerate(delta): _, pvals = two_group_model(N=20, m=m, pi0=0, delta=d) n_rejected_lsu[j, i] = np.sum(lsu(pvals)) n_rejected_tst[j, i] = np.sum(tst(pvals)) n_rejected_nc[j, i] = np.sum(pvals < 0.05) print 'iteration %3d' % i """Visualize the results (i.e. the proportion of rejected null hypotheses as a function of the effect size.)""" fig = plt.figure(figsize=(6, 4)) fig.subplots_adjust(bottom=0.15) ax = fig.add_subplot(111) # lsu ax.errorbar(x=delta, y=np.mean(n_rejected_lsu / float(m), axis=1), yerr=np.std(n_rejected_lsu / float(m), axis=1)) # tst ax.errorbar(x=delta,
from fwer import sidak, holm_bonferroni, hochberg from data import square_grid_model from permutation import tfr_permutation_test from rft import rft_2d from util import grid_model_counts, roc from viz import plot_grid_model """Generate the test data.""" nl, sl = 90, 30 N, delta = 25, 0.7 X, X_tstats, X_raw, Y_raw = square_grid_model(nl, sl, N, delta, equal_var=True) alpha = 0.05 """Apply each correction technique to the generated dataset.""" Y_sidak = sidak(X.flatten(), alpha=alpha) Y_sidak = Y_sidak.reshape(nl, nl) Y_fdr = lsu(X.flatten(), q=alpha) Y_fdr = Y_fdr.reshape(nl, nl) Y_qvalue, _ = qvalue(X.flatten(), threshold=alpha) Y_qvalue = Y_qvalue.reshape(nl, nl) Y_rft, Y_smooth, _ = rft_2d(X_tstats, fwhm=30, alpha=alpha, verbose=True) # No reshape needed since already in correct form. Y_tst = tst(X.flatten(), q=alpha) Y_tst = Y_tst.reshape(nl, nl) Y_permutation = tfr_permutation_test(X_raw, Y_raw, n_permutations=100, alpha=alpha,
fs = 200 n_iter = 1000 A_pi0, B_pi0 = 0.75, 0.9 A_delta, B_delta = 1.2, 0.7 A_m, B_m = 3 * fs, 8 * fs power_all, power_separate = [], [] for i in np.arange(0, n_iter): """Draw random numbers from two different mixture distributions.""" _, A_pvals = two_group_model(N=25, m=A_m, pi0=A_pi0, delta=A_delta) _, B_pvals = two_group_model(N=25, m=B_m, pi0=B_pi0, delta=B_delta) """Combine the separate classes and perform one FDR.""" all_pvals = np.hstack([A_pvals, B_pvals]) significant = lsu(all_pvals) """Compute power of the procedure, i.e. Pr(reject H0 | H1 is true).""" A_rejections = np.sum(significant[int(A_m * A_pi0):A_m]) B_rejections = np.sum(significant[A_m + int(B_m * B_pi0):B_m]) power_all.append((A_rejections + B_rejections) / ((1 - A_pi0) * A_m + (1 - B_pi0) * B_m)) """Apply FDR separately to the two parts and perform a similar power calculation.""" A_significant, B_significant = lsu(A_pvals), lsu(B_pvals) A_rejections = np.sum(A_significant[int(A_m * A_pi0):A_m]) B_rejections = np.sum(B_significant[int(B_m * B_pi0):B_m]) power_separate.append((A_rejections + B_rejections) / ((1 - A_pi0) * A_m + (1 - B_pi0) * B_m))