def two_group_reproducibility(effect_sizes,
emphasis_primary,
nl=90,
sl=30,
alpha=0.05,
N=25,
n_iter=10,
method=tst):
"""Function for computing reproducibility in the two-group model under
various effect sizes and amounts of emphasis on the primary study.
Input arguments:
================
effect_sizes : ndarray [n_effect_sizes, ]
The tested effect sizes.
emphasis_primary : ndarray [n_emphasis_values, ]
The tested amounts of emphasis on the primary study.
TODO: document rest of the parameters.
Output arguments
================
reproducibility : ndarray [n_effect_sizes, n_emphasis_values]
The observed reproducibility at the tested effect sizes and amounts
of emphasis on the primary study.
"""
n_effect_sizes, n_emphasis = len(effect_sizes), len(emphasis_primary)
"""Compute the reproducibility rate for each effect size and
primary study emphasis, for several iterations."""
reproducible = np.zeros([n_effect_sizes, n_emphasis, n_iter])
for ind in np.ndindex(n_effect_sizes, n_emphasis, n_iter):
# Simulate new data.
delta, emphasis = effect_sizes[ind[0]], emphasis_primary[ind[1]]
X_pri = square_grid_model(nl, sl, N, delta, equal_var=True)[0]
X_fol = square_grid_model(nl, sl, N, delta, equal_var=True)[0]
X_pri, X_fol = X_pri.flatten(), X_fol.flatten()
# Apply the correction and compute reproducibility.
R = fwer_replicability(X_pri, X_fol, emphasis, method, alpha)
R = np.reshape(R, [nl, nl])
tp, _, _, fn = grid_model_counts(R, nl, sl)
reproducible[ind] = tp / float(tp + fn)
reproducible = np.mean(reproducible, axis=2)
return reproducible
def direct_replication_fwer_partial_conjunction():
"""Perform a comparison of the partial conjuction and FWER
replicability methods using the two-group model."""
N, nl, sl = 25, 90, 30
effect_sizes = np.linspace(0.6, 2.4, 12)
n_effect_sizes = len(effect_sizes)
method = lsu # hochberg #bonferroni
emphasis = np.asarray(
[0.02, 0.05, 0.10, 0.30, 0.50, 0.70, 0.90, 0.95, 0.98])
n_emphasis = len(emphasis)
"""Generate the test data."""
print('Simulating primary and follow-up experiments ..')
# Allocate memory.
pvals_pri = np.zeros([n_effect_sizes, nl, nl])
pvals_sec = np.zeros(np.shape(pvals_pri))
# Obtain the uncorrected p-values.
for i, delta in enumerate(effect_sizes):
pvals_pri[i] = square_grid_model(nl, sl, N, delta)[0]
pvals_sec[i] = square_grid_model(nl, sl, N, delta)[0]
"""Find reproducible effects using the FWER replicability
method."""
print('Estimating reproducibility: FWER replicability ..')
repr_fwer = np.zeros([n_effect_sizes, n_emphasis])
for i in np.ndindex(n_effect_sizes, n_emphasis):
# Find reproducible effects and rearrange the data.
result = fwer_replicability(pvals_pri[i[0]].flatten(),
pvals_sec[i[0]].flatten(), emphasis[i[1]],
method)
result = np.reshape(result, [nl, nl])
# Compute the number reproducible true effects.
repr_fwer[i] = (grid_model_counts(result, nl, sl)[0] / float(sl**2))
"""Find reproducible effects using the partial conjuction
method."""
print('Estimating reproducibility: Partial conjuction ..')
repr_part = np.zeros([n_effect_sizes])
for i in np.ndindex(n_effect_sizes):
result = partial_conjuction(pvals_pri[i].flatten(),
pvals_sec[i].flatten(), method)
result = np.reshape(result, [nl, nl])
repr_part[i] = (grid_model_counts(result, nl, sl)[0] / float(sl**2))
"""Visualize the data."""
sns.set_style('white')
fig = plt.figure(figsize=(8, 5))
ax = fig.add_subplot(111)
plot_logistic(effect_sizes,
repr_fwer[:, emphasis <= 0.5],
ax=ax,
color='k')
plot_logistic(effect_sizes, repr_fwer[:, emphasis > 0.5], ax=ax, color='g')
plot_logistic(effect_sizes, repr_part, ax=ax, color='b')
ax.set_xlabel('Effect size')
ax.set_ylabel('Reproducibility rate')
fig.tight_layout()
plt.show()