def test_mcnemar_chisquare():
f_obs1 = np.array([[101, 121], [59, 33]])
f_obs2 = np.array([[101, 70], [59, 33]])
f_obs3 = np.array([[101, 80], [59, 33]])
#> mcn = mcnemar.test(matrix(c(101, 121, 59, 33),nrow=2))
res1 = [2.067222e01, 5.450095e-06]
res2 = [0.7751938, 0.3786151]
res3 = [2.87769784, 0.08981434]
assert_allclose(mcnemar(f_obs1, exact=False), res1, rtol=1e-6)
assert_allclose(mcnemar(f_obs2, exact=False), res2, rtol=1e-6)
assert_allclose(mcnemar(f_obs3, exact=False), res3, rtol=1e-6)
# compare table versus observations
x, y = _expand_table(f_obs2).T # tuple unpack
assert_allclose(mcnemar(f_obs2, exact=False),
mcnemar(x, y, exact=False),
rtol=1e-13)
# test correction = False
res1 = [2.135556e01, 3.815136e-06]
res2 = [0.9379845, 0.3327967]
res3 = [3.17266187, 0.07488031]
res = mcnemar(f_obs1, exact=False, correction=False)
assert_allclose(res, res1, rtol=1e-6)
res = mcnemar(f_obs2, exact=False, correction=False)
assert_allclose(res, res2, rtol=1e-6)
res = mcnemar(f_obs3, exact=False, correction=False)
assert_allclose(res, res3, rtol=1e-6)
def test_mcnemar_exact():
f_obs1 = np.array([[101, 121], [59, 33]])
f_obs2 = np.array([[101, 70], [59, 33]])
f_obs3 = np.array([[101, 80], [59, 33]])
f_obs4 = np.array([[101, 30], [60, 33]])
f_obs5 = np.array([[101, 10], [30, 33]])
f_obs6 = np.array([[101, 10], [10, 33]])
#vassar college online computation
res1 = 0.000004
res2 = 0.378688
res3 = 0.089452
res4 = 0.00206
res5 = 0.002221
res6 = 1.
assert_almost_equal(mcnemar(f_obs1, exact=True), [59, res1], decimal=6)
assert_almost_equal(mcnemar(f_obs2, exact=True), [59, res2], decimal=6)
assert_almost_equal(mcnemar(f_obs3, exact=True), [59, res3], decimal=6)
assert_almost_equal(mcnemar(f_obs4, exact=True), [30, res4], decimal=6)
assert_almost_equal(mcnemar(f_obs5, exact=True), [10, res5], decimal=6)
assert_almost_equal(mcnemar(f_obs6, exact=True), [10, res6], decimal=6)
x, y = _expand_table(f_obs2).T # tuple unpack
assert_allclose(mcnemar(f_obs2, exact=True),
mcnemar(x, y, exact=True),
rtol=1e-13)
def test_mcnemar_exact():
f_obs1 = np.array([[101, 121], [59, 33]])
f_obs2 = np.array([[101, 70], [59, 33]])
f_obs3 = np.array([[101, 80], [59, 33]])
f_obs4 = np.array([[101, 30], [60, 33]])
f_obs5 = np.array([[101, 10], [30, 33]])
f_obs6 = np.array([[101, 10], [10, 33]])
#vassar college online computation
res1 = 0.000004
res2 = 0.378688
res3 = 0.089452
res4 = 0.00206
res5 = 0.002221
res6 = 1.
assert_almost_equal(mcnemar(f_obs1, exact=True), [59, res1], decimal=6)
assert_almost_equal(mcnemar(f_obs2, exact=True), [59, res2], decimal=6)
assert_almost_equal(mcnemar(f_obs3, exact=True), [59, res3], decimal=6)
assert_almost_equal(mcnemar(f_obs4, exact=True), [30, res4], decimal=6)
assert_almost_equal(mcnemar(f_obs5, exact=True), [10, res5], decimal=6)
assert_almost_equal(mcnemar(f_obs6, exact=True), [10, res6], decimal=6)
x, y = _expand_table(f_obs2).T # tuple unpack
assert_allclose(mcnemar(f_obs2, exact=True),
mcnemar(x, y, exact=True), rtol=1e-13)
def test_mcnemar_chisquare():
f_obs1 = np.array([[101, 121], [59, 33]])
f_obs2 = np.array([[101, 70], [59, 33]])
f_obs3 = np.array([[101, 80], [59, 33]])
#> mcn = mcnemar.test(matrix(c(101, 121, 59, 33),nrow=2))
res1 = [2.067222e01, 5.450095e-06]
res2 = [0.7751938, 0.3786151]
res3 = [2.87769784, 0.08981434]
assert_allclose(mcnemar(f_obs1, exact=False), res1, rtol=1e-6)
assert_allclose(mcnemar(f_obs2, exact=False), res2, rtol=1e-6)
assert_allclose(mcnemar(f_obs3, exact=False), res3, rtol=1e-6)
# compare table versus observations
x, y = _expand_table(f_obs2).T # tuple unpack
assert_allclose(mcnemar(f_obs2, exact=False),
mcnemar(x, y, exact=False), rtol=1e-13)
# test correction = False
res1 = [2.135556e01, 3.815136e-06]
res2 = [0.9379845, 0.3327967]
res3 = [3.17266187, 0.07488031]
res = mcnemar(f_obs1, exact=False, correction=False)
assert_allclose(res, res1, rtol=1e-6)
res = mcnemar(f_obs2, exact=False, correction=False)
assert_allclose(res, res2, rtol=1e-6)
res = mcnemar(f_obs3, exact=False, correction=False)
assert_allclose(res, res3, rtol=1e-6)
def test_cochransq():
# example from dataplot docs, Conovover p. 253
# http://www.itl.nist.gov/div898/software/dataplot/refman1/auxillar/cochran.htm
x = np.array(
[
[1, 1, 1],
[1, 1, 1],
[0, 1, 0],
[1, 1, 0],
[0, 0, 0],
[1, 1, 1],
[1, 1, 1],
[1, 1, 0],
[0, 0, 1],
[0, 1, 0],
[1, 1, 1],
[1, 1, 1],
]
)
res_qstat = 2.8
res_pvalue = 0.246597
assert_almost_equal(cochrans_q(x), [res_qstat, res_pvalue])
# equivalence of mcnemar and cochranq for 2 samples
a, b = x[:, :2].T
assert_almost_equal(mcnemar(a, b, exact=False, correction=False), cochrans_q(x[:, :2]))
def mcnemar_test(y, y_a, y_b):
'''
Given true labels and predictions from two binary classifiers a, b,
perfrom McNemar test on the these predictions
Return (chi2, p): Chi2 of the test statistic, p-value of the null hypothesis
'''
chi2, p = mcnemar(build_contingency_table(y, y_a, y_b))
print('Chi2 = {}, P = {}'.format(chi2, p))
return chi2, p
def test_mcnemar_vectorized():
ttk = np.random.randint(5, 15, size=(2, 2, 3))
mcnemar(ttk)
res = mcnemar(ttk, exact=False)
res1 = list(zip(*[mcnemar(ttk[:, :, i], exact=False) for i in range(3)]))
assert_allclose(res, res1, rtol=1e-13)
res = mcnemar(ttk, exact=False, correction=False)
res1 = list(zip(*[mcnemar(ttk[:, :, i], exact=False, correction=False) for i in range(3)]))
assert_allclose(res, res1, rtol=1e-13)
res = mcnemar(ttk, exact=True)
res1 = list(zip(*[mcnemar(ttk[:, :, i], exact=True) for i in range(3)]))
assert_allclose(res, res1, rtol=1e-13)
def test_mcnemar_vectorized():
ttk = np.random.randint(5,15, size=(2,2,3))
mcnemar(ttk)
res = mcnemar(ttk, exact=False)
res1 = zip(*[mcnemar(ttk[:,:,i], exact=False) for i in range(3)])
assert_allclose(res, res1, rtol=1e-13)
res = mcnemar(ttk, exact=False, correction=False)
res1 = zip(*[mcnemar(ttk[:,:,i], exact=False, correction=False)
for i in range(3)])
assert_allclose(res, res1, rtol=1e-13)
res = mcnemar(ttk, exact=True)
res1 = zip(*[mcnemar(ttk[:,:,i], exact=True) for i in range(3)])
assert_allclose(res, res1, rtol=1e-13)
def test_cochransq():
#example from dataplot docs, Conovover p. 253
#http://www.itl.nist.gov/div898/software/dataplot/refman1/auxillar/cochran.htm
x = np.array([[1, 1, 1], [1, 1, 1], [0, 1, 0], [1, 1, 0], [0, 0, 0],
[1, 1, 1], [1, 1, 1], [1, 1, 0], [0, 0, 1], [0, 1, 0],
[1, 1, 1], [1, 1, 1]])
res_qstat = 2.8
res_pvalue = 0.246597
assert_almost_equal(cochrans_q(x), [res_qstat, res_pvalue])
#equivalence of mcnemar and cochranq for 2 samples
a, b = x[:, :2].T
assert_almost_equal(mcnemar(a, b, exact=False, correction=False),
cochrans_q(x[:, :2]))
def tryMcnemar():
"""McNemars Test should be run in the "exact" version, even though approximate formulas are
typically given in the lecture scripts. Just ignore the statistic that is returned, because
it is different for the two options.
In the following example, a researcher attempts to determine if a drug has an effect on a
particular disease. Counts of individuals are given in the table, with the diagnosis
(disease: present or absent) before treatment given in the rows, and the diagnosis
after treatment in the columns. The test requires the same subjects to be included in
the before-and-after measurements (matched pairs).
"""
f_obs = np.array([[101, 121], [59, 33]])
(statistic, pVal) = mcnemar(f_obs)
print("\nMCNEMAR'S TEST -----------------------------------------------------")
print("p = {0:5.3f}".format(pVal))
if pVal < 0.05:
print("There was a significant change in the disease by the treatment.")
def tryMcnemar():
'''McNemars Test should be run in the "exact" version, even though approximate formulas are
typically given in the lecture scripts. Just ignore the statistic that is returned, because
it is different for the two options.
In the following example, a researcher attempts to determine if a drug has an effect on a
particular disease. Counts of individuals are given in the table, with the diagnosis
(disease: present or absent) before treatment given in the rows, and the diagnosis
after treatment in the columns. The test requires the same subjects to be included in
the before-and-after measurements (matched pairs).
'''
f_obs = np.array([[101, 121],[59, 33]])
(statistic, pVal) = mcnemar(f_obs)
print('\nMCNEMAR\'S TEST -----------------------------------------------------')
print('p = {0:5.3f}'.format(pVal))
if pVal < 0.05:
print("There was a significant change in the disease by the treatment.")
def mcnemar_test(methodA_res, methodB_res):
assert methodA_res['dataset'] == methodB_res['dataset']
assert methodA_res['length'] == methodB_res['length']
# calculate the contingency table for mcnemar's test
# methodB
# correct incorrect
# methodA correct A B
# incorrect C D
A, B, C, D = 0, 0, 0, 0
for idx in range(0, methodA_res['length']):
if idx in methodA_res['correct'] and idx in methodB_res['correct']:
A += 1
if idx in methodA_res['correct'] and idx in methodB_res['incorrect']:
B += 1
if idx in methodA_res['incorrect'] and idx in methodB_res['correct']:
C += 1
if idx in methodA_res['incorrect'] and idx in methodB_res['incorrect']:
D += 1
table = [[A, B], [C, D]]
test_result = mcnemar(table, exact=False, correction=True)
return test_result
def mcnemar_test(pdf, var_names):
chi2, p = mcnemar(pdf[var_names[0]], pdf[var_names[1]], exact=False)
return _('Result of the McNemar test') + ': χ<sup>2</sup>(1, <i>N</i> = %d) = %0.3g, %s\n' % \
(len(pdf[var_names[0]]), chi2, cs_util.print_p(p))
def compute_mcnemar_test(method1_data, method2_data):
stat, pvalue = mcnemar(method1_data, method2_data, exact=False)
if pvalue < alpha:
return "%.3g*" % pvalue
return "%.3g" % pvalue
res1 = model1.predict(test3D_matrix)
res2 = model2.predict(test3D_matrix1)
test1 = [0] * 1580
i = 0
for cl in res1:
if np.argmax(cl) == np.argmax(test_classes[i]):
test1[i] = 1
else:
test1[i] = 0
i += 1
test2 = [0] * 1580
i = 0
for cl in res2:
if np.argmax(cl) == np.argmax(test_classes[i]):
test2[i] = 1
else:
test2[i] = 0
i += 1
stts, pvalue = mcnemar(test1, test2)
print(pvalue)
alpha = 0.05
if pvalue > alpha:
print('Same proportions of errors (fail to reject H0)')
else:
print('Different proportions of errors (reject H0)')
from statsmodels.sandbox.stats.runs import mcnemar
#subject_list=['sub001', 'sub002', 'sub003', 'sub004', 'sub005', 'sub006', 'sub007', 'sub008', 'sub009', 'sub010', 'sub011', 'sub012', 'sub013', 'sub014', 'sub015', 'sub016', 'sub017', 'sub018', 'sub019']
#req_filter='bp'
#fwhm_list=['0', '6']
subject_list = sys.argv[1].split(', ')
req_filter = sys.argv[2]
fwhm_list = sys.argv[3].split(', ')
MVPA_result_folder = sys.argv[4]
mask_name = sys.argv[5]
hits_dict = {}
for fwhm in fwhm_list:
fwhm_hits = []
for subject in subject_list:
data = h5load(MVPA_result_folder + '/' + subject + '_' + mask_name +
'_' + req_filter + '_' + str(fwhm) + '.hdf5')
fwhm_hits += list(data.samples[:, 0] == data.sa.targets)
hits_dict[fwhm] = fwhm_hits
#~ paired_list_of_hits=zip(hits_dict[FWHM_list[0]],hits_dict[FWHM_list[1]])
#~ stat=mcnemar_midp(paired_list_of_hits.count((0, 1)), paired_list_of_hits.count((1, 0)))
stat_pval = mcnemar(hits_dict[fwhm_list[0]],
hits_dict[fwhm_list[1]],
exact=False,
correction=True)
print stat_pval[0]
print stat_pval[1] * len(subject_list)
V, p, df, expected = stats.chi2_contingency(data, correction=False)
print("Chi-square value = %1.2f, df = %1.2f, p = %1.2f"%(V, df, p))
## McNear Test
- One dependent variable (categorical)
- dependent samples
data = pd.read_table(DEMO_DATA_ROOT + "/gries_sflwr/_inputfiles/04-1-2-3_accjudg.csv")
data.head()
from statsmodels.sandbox.stats.runs import mcnemar
crosstab = pd.crosstab(data['BEFORE'],data['AFTER'])
x2, p = mcnemar(crosstab, correction=False)
print('Chi-square=%1.2f, p = %1.2f'%(x2, p))
## Independent *t*-test
vowels = pd.read_table(DEMO_DATA_ROOT + "/gries_sflwr/_inputfiles/04-3-2-1_f1-freq.csv")
vowels.head()
t, p = stats.ttest_ind(vowels[vowels['SEX']=='M']['HZ_F1'], vowels[vowels['SEX']=='F']['HZ_F1'])
print("t-score=%1.2f, p=%1.2f"%(t,p))
## One-way ANOVA
data = pd.read_table(DEMO_DATA_ROOT + "/gries_sflwr/_inputfiles/05-2_reactiontimes.csv")
data
'''Solution for Exercise "Categorical Data"
McNemar's Test
'''
from scipy import stats
from statsmodels.sandbox.stats.runs import mcnemar
obs = [[19,1], [6, 14]]
obs2 = [[20,0], [6, 14]]
_, p = mcnemar(obs)
_, p2 = mcnemar(obs2)
print('\n--- McNemar Test ---')
if p < 0.05:
print('The results from the neurologist are significanlty different from the questionnaire (p={0:5.3f}).'.format(p))
else:
print('The results from the neurologist are NOT significanlty different from the questionnaire (p={0:5.3f}).'.format(p))
if (p<0.05 == p2<0.05):
print('The results would NOT change if the expert had diagnosed all "sane" people correctly.')
else:
print('The results would change if the expert had diagnosed all "sane" people correctly.')
import numpy as np
from statsmodels.sandbox.stats.runs import mcnemar
from statsmodels.stats.contingency_tables import SquareTable
from statsmodels.stats.inter_rater import cohens_kappa
from statsmodels.discrete.conditional_models import (ConditionalLogit,
ConditionalPoisson,
ConditionalMNLogit)
#1.检验量
#(1)对称性检验(齐性检验)
data = pd.read_csv(r"D:/书籍资料整理/属性数据分析/环保.csv")
temp = np.array([[227, 132], [107, 678]])
#使用此种参数与书中结果最接近。这个应该是书中提到的方形列联表的检测方法.
#其结果与那个说的更相近
result = mcnemar(temp, exact=False, correction=False)
tmp = pd.DataFrame()
for i in range(0, 4):
tmp = tmp.append([data.loc[i]] * data.iloc[i]['值'])
tmp = tmp.reset_index()
del tmp['值']
del tmp['index']
temp2 = SquareTable.from_data(tmp)
temp2.summary()
#可求边缘分布概率
row, col = temp2.marginal_probabilities
#方形列联表检验
temp2.symmetry()
p_val = f1.pdf(fval)
# Chi square using contingency table
# -----------------------------------------------------------------------------
from scipy.stats import chi2_contingency
import numpy as np
observed_table = np.array([[5, 744], [9, 1489]])
chi2, p, dof, expectedTable = chi2_contingency(observed_table)
# McNemar test for paired case-control study using contingency table
# -----------------------------------------------------------------------------
from statsmodels.sandbox.stats.runs import mcnemar
contingencyTable = np.array([[13, 25], [4, 92]])
statistic, p = mcnemar(contingencyTable)
# kappa test for inter-observer agreement testing using contingency table
# -----------------------------------------------------------------------------
import numpy as np
def kappa_table(table):
row_marg = np.sum(table, 0)
col_marg = np.sum(table, 1)
total = sum(row_marg)
n = table.shape[1]
diagonal_sum = sum(table[i][i] for i in range(n))
observed = diagonal_sum / total
'''Solution for Exercise "Categorical Data"
McNemar's Test
'''
# author: Thomas Haslwanter, date: Sept-2015
from scipy import stats
from statsmodels.sandbox.stats.runs import mcnemar
obs = [[19, 1], [6, 14]]
obs2 = [[20, 0], [6, 14]]
_, p = mcnemar(obs)
_, p2 = mcnemar(obs2)
print('\n--- McNemar Test ---')
if p < 0.05:
print(
'The results from the neurologist are significanlty different from the questionnaire (p={0:5.3f}).'
.format(p))
else:
print(
'The results from the neurologist are NOT significanlty different from the questionnaire (p={0:5.3f}).'
.format(p))
if (p < 0.05 == p2 < 0.05):
print(
'The results would NOT change if the expert had diagnosed all "sane" people correctly.'
)
else:
print(
