def p_adjust_bh(p):
"""Benjamini-Hochberg p-value correction for multiple hypothesis testing."""
p = asfarray(p)
by_descend = p.argsort()[::-1]
by_orig = by_descend.argsort()
steps = float(len(p)) / arange(len(p), 0, -1)
q = minimum(1, minimum.accumulate(steps * p[by_descend]))
return q[by_orig]
def simulateLindleyEfficient(lam, mu, n):
arrDist = stats.expon(scale=1 / lam) # note that this is the MEAN!!!!
servDist = stats.expon(scale=1 / mu) # note that this is the MEAN!!!!
a = arrDist.rvs(n - 1) # interarrival times
b = servDist.rvs(n - 1) # service times
d = append([0], b - a)
cumd = cumsum(d)
w = cumd - minimum.accumulate(cumd) # running minimum
return w
def multiple_test_correction(pvals, alpha=0.05, method='indep'):
"""
p-value correction for false discovery rate.
This covers Benjamini/Hochberg for independent or positively correlated and
Benjamini/Yekutieli for general or negatively correlated tests. Both are
available in the function multipletests, as method=`fdr_bh`, resp. `fdr_by`.
If there is prior information on the fraction of true hypothesis, then alpha
should be set to alpha * m/m_0 where m is the number of tests,
given by the p-values, and m_0 is an estimate of the true hypothesis.
(see Benjamini, Krieger and Yekuteli)
The two-step method of Benjamini, Krieger and Yekutiel that estimates the number
of false hypotheses will be available (soon).
Method names can be abbreviated to first letter, 'i' or 'p' for fdr_bh and 'n' for
fdr_by.
Author: Josef Pktd, H Raja and Vincent Davis (scikits.statsmodels.sandbox.stats.multicomp)
Keyword arguments:
pvals -- List of p-values from the individual tests.
alpha -- Error rate (float). (default 0.05)
method -- {'indep', 'negcorr')
Return:
rejected -- List of booleans. True if a hypothesis is rejected, False otherwise.
pvalue_corrected -- A list with the p-values adjusted for multiple hypothesis testing to limit FDR.
"""
pvals = asarray(pvals)
pvals_sortind = argsort(pvals)
pvals_sorted = pvals[pvals_sortind]
sortrevind = pvals_sortind.argsort()
if method in ['i', 'indep', 'p', 'poscorr']:
ecdffactor = ecdf(pvals_sorted)
elif method in ['n', 'negcorr']:
cm = sum(1./arange(1, len(pvals_sorted)+1))
ecdffactor = ecdf(pvals_sorted) / cm
else:
raise ValueError('only indep and negcorr implemented')
reject = pvals_sorted < ecdffactor*alpha
if reject.any():
rejectmax = max(nonzero(reject)[0])
else:
rejectmax = 0
reject[:rejectmax] = True
pvals_corrected_raw = pvals_sorted / ecdffactor
pvals_corrected = minimum.accumulate(pvals_corrected_raw[::-1])[::-1]
pvals_corrected[pvals_corrected > 1] = 1
return reject[sortrevind], pvals_corrected[sortrevind]
def multiple_test_correction(pvals, alpha=0.05, method='indep'):
"""
p-value correction for false discovery rate.
This covers Benjamini/Hochberg for independent or positively correlated and
Benjamini/Yekutieli for general or negatively correlated tests. Both are
available in the function multipletests, as method=`fdr_bh`, resp. `fdr_by`.
If there is prior information on the fraction of true hypothesis, then alpha
should be set to alpha * m/m_0 where m is the number of tests,
given by the p-values, and m_0 is an estimate of the true hypothesis.
(see Benjamini, Krieger and Yekuteli)
The two-step method of Benjamini, Krieger and Yekutiel that estimates the number
of false hypotheses will be available (soon).
Method names can be abbreviated to first letter, 'i' or 'p' for fdr_bh and 'n' for
fdr_by.
Author: Josef Pktd, H Raja and Vincent Davis (scikits.statsmodels.sandbox.stats.multicomp)
Keyword arguments:
pvals -- List of p-values from the individual tests.
alpha -- Error rate (float). (default 0.05)
method -- {'indep', 'negcorr')
Return:
rejected -- List of booleans. True if a hypothesis is rejected, False otherwise.
pvalue_corrected -- A list with the p-values adjusted for multiple hypothesis testing to limit FDR.
"""
pvals = asarray(pvals)
pvals_sortind = argsort(pvals)
pvals_sorted = pvals[pvals_sortind]
sortrevind = pvals_sortind.argsort()
if method in ['i', 'indep', 'p', 'poscorr']:
ecdffactor = ecdf(pvals_sorted)
elif method in ['n', 'negcorr']:
cm = sum(1./arange(1, len(pvals_sorted)+1))
ecdffactor = ecdf(pvals_sorted) / cm
else:
raise ValueError('only indep and negcorr implemented')
reject = pvals_sorted < ecdffactor*alpha
if reject.any():
rejectmax = max(nonzero(reject)[0])
else:
rejectmax = 0
reject[:rejectmax] = True
pvals_corrected_raw = pvals_sorted / ecdffactor
pvals_corrected = minimum.accumulate(pvals_corrected_raw[::-1])[::-1]
pvals_corrected[pvals_corrected > 1] = 1
return reject[sortrevind], pvals_corrected[sortrevind]
def FDR(df, column, new_col):
#read the column from the data frame, exclude rows with NA and sort in descending order
a = df[df[column] != "NaN"][column].sort_values(ascending=False)
#Get the number of rows in the data frame
b = len(a)
#create a list in reverse order for the range of 1 to the length of the data frame
c = list(reversed(range(1, b + 1)))
#Perform FDR, I can't remember why this works, but it does
d = minimum.accumulate([b / x * y for x, y in zip(c, a)])
#If a value is great than 1, change it to 1, else keep it the same
e = [x if x < 1.0 else 1.0 for x in d]
#create a data frame out of the new FDR column
f = pd.DataFrame(e, columns=[new_col])
#match the index for the new column to the original data frame
f.index = a.index
#Ad the new column to the data frame
df = pd.concat([df, f], axis=1)
#Return the new data frame
return df