def test_twoway_confusion_phi():
cm = ConfusionMatrix2.from_ccw(116, 21, 18, 21)
assert_almost_equal(cm.matthews_corr(), 0.31, 2)
assert_almost_equal(cm.yule_q(), 0.6512, 4)
assert_almost_equal(cm.DOR(), 4.7347, 4)
cm = ConfusionMatrix2.from_ccw(35, 60, 41, 9)
assert_almost_equal(cm.chisq_score(), 5.50, 2)
def test_negative_correlation():
"""Some metrics should have negative sign
"""
cm = ConfusionMatrix2.from_ccw(10, 120, 8, 300)
assert_almost_equal(cm.g_score(), 384.52, 2)
assert_almost_equal(cm.chisq_score(), 355.70, 2)
mic0, mic1, mic2 = cm.mic_scores()
assert_almost_equal(mic0, -0.8496, 4)
assert_almost_equal(mic1, -0.8524, 4)
assert_almost_equal(mic2, -0.8510, 4)
assert_almost_equal(cm.matthews_corr(), -0.9012, 4)
assert_almost_equal(cm.informedness(), -0.9052, 4)
assert_almost_equal(cm.markedness(), -0.8971, 4)
assert_almost_equal(cm.kappa(), -0.6407, 4)
inform, marked = cm.informedness(), cm.markedness()
expected_matt = geometric_mean(inform, marked)
assert_almost_equal(expected_matt, cm.matthews_corr(), 6)
def test_0000():
m = (0, 0, 0, 0)
cm = ConfusionMatrix2.from_ccw(*m)
assert_almost_equal(cm.chisq_score(), 0.0, 4)
assert_almost_equal(cm.g_score(), 0.0, 4)
assert_true(np.isnan(cm.dice_coeff()))
assert_true(np.isnan(cm.ochiai_coeff()))
assert_true(np.isnan(cm.ochiai_coeff_adj()))
assert_true(np.isnan(cm.matthews_corr()))
assert_true(np.isnan(cm.mp_corr()))
assert_true(np.isnan(cm.kappa()))
assert_true(np.isnan(cm.loevinger_coeff()))
assert_true(np.isnan(cm.cole_coeff()))
assert_true(np.isnan(cm.yule_q()))
assert_true(np.isnan(cm.yule_y()))
assert_true(np.isnan(cm.informedness()))
assert_true(np.isnan(cm.markedness()))
def test_1110():
m = (1, 1, 1, 0)
cm = ConfusionMatrix2.from_ccw(*m)
assert_almost_equal(cm.chisq_score(), 0.75, 4)
assert_almost_equal(cm.g_score(), 1.0465, 4)
assert_almost_equal(cm.dice_coeff(), 0.6667, 4)
assert_almost_equal(cm.ochiai_coeff(), 0.7071, 4)
assert_almost_equal(cm.ochiai_coeff_adj(), 0.4459, 4)
assert_almost_equal(cm.matthews_corr(), 0.5, 4)
assert_almost_equal(cm.mp_corr(), 0.5, 4)
assert_almost_equal(cm.kappa(), 0.4, 4)
assert_almost_equal(cm.loevinger_coeff(), 1.0, 4)
assert_almost_equal(cm.cole_coeff(), 1.0, 4)
assert_almost_equal(cm.yule_q(), 1.0, 4)
assert_almost_equal(cm.yule_y(), 1.0, 4)
assert_almost_equal(cm.informedness(), 0.5, 4)
assert_almost_equal(cm.markedness(), 0.5, 4)
def test_0101():
m = (0, 1, 0, 1)
cm = ConfusionMatrix2.from_ccw(*m)
assert_almost_equal(cm.chisq_score(), 2.0, 4)
assert_almost_equal(cm.g_score(), 2.7726, 4)
assert_almost_equal(cm.dice_coeff(), 0.0, 4)
assert_almost_equal(cm.ochiai_coeff(), 0.0, 4)
assert_almost_equal(cm.ochiai_coeff_adj(), -1.0, 4)
assert_almost_equal(cm.matthews_corr(), -1.0, 4)
assert_almost_equal(cm.mp_corr(), -1.0, 4)
assert_almost_equal(cm.kappa(), -1.0, 4)
assert_almost_equal(cm.loevinger_coeff(), -1.0, 4)
assert_almost_equal(cm.cole_coeff(), -1.0, 4)
assert_almost_equal(cm.yule_q(), -1.0, 4)
assert_almost_equal(cm.yule_y(), -1.0, 4)
assert_almost_equal(cm.informedness(), -1.0, 4)
assert_almost_equal(cm.markedness(), -1.0, 4)
def test_twoway_confusion_2():
"""Finley's tornado data (listed in Goodman and Kruskal)
"""
cm = ConfusionMatrix2.from_ccw(11, 14, 906, 3)
assert_almost_equal(cm.g_score(), 70.83, 2)
assert_almost_equal(cm.chisq_score(), 314.3, 1)
mic0, mic1, mic2 = cm.mic_scores()
assert_almost_equal(mic0, 0.555, 3)
assert_almost_equal(mic1, 0.698, 3)
assert_almost_equal(mic2, 0.614, 3)
assert_almost_equal(cm.matthews_corr(), 0.580, 3)
assert_almost_equal(cm.informedness(), 0.770, 3)
assert_almost_equal(cm.markedness(), 0.437, 3)
kappa0, kappa1, kappa2 = cm.kappas()
assert_almost_equal(kappa0, 0.431, 3)
assert_almost_equal(kappa1, 0.780, 3)
assert_almost_equal(kappa2, 0.556, 3)
def test_twoway_confusion_1():
"""Finley's tornado data
http://www.cawcr.gov.au/projects/verification/Finley/Finley_Tornados.html
"""
cm = ConfusionMatrix2.from_ccw(28, 72, 2680, 23)
assert_almost_equal(cm.g_score(), 126.1, 1)
assert_almost_equal(cm.chisq_score(), 397.9, 1)
mic0, mic1, mic2 = cm.mic_scores()
assert_almost_equal(mic2, 0.429, 3)
assert_almost_equal(mic1, 0.497, 3)
assert_almost_equal(mic0, 0.382, 3)
assert_almost_equal(cm.matthews_corr(), 0.377, 3)
assert_almost_equal(cm.informedness(), 0.523, 3)
assert_almost_equal(cm.markedness(), 0.271, 3)
kappa0, kappa1, kappa2 = cm.kappas()
assert_almost_equal(kappa0, 0.267, 3)
assert_almost_equal(kappa1, 0.532, 3)
assert_almost_equal(kappa2, 0.355, 3)
def test_0100():
m = (0, 1, 0, 0)
cm = ConfusionMatrix2.from_ccw(*m)
assert_almost_equal(cm.chisq_score(), 0.0, 4)
assert_almost_equal(cm.g_score(), 0.0, 4)
h, c, v = cm.entropy_scores()
assert_almost_equal(h, 1.0, 4)
assert_almost_equal(c, 1.0, 4)
assert_almost_equal(v, 1.0, 4)
assert_almost_equal(cm.dice_coeff(), 0.0, 4)
assert_almost_equal(cm.ochiai_coeff(), 0.0, 4)
assert_almost_equal(cm.ochiai_coeff_adj(), 0.0, 4)
assert_almost_equal(cm.matthews_corr(), -0.5, 4)
assert_almost_equal(cm.mp_corr(), -0.5, 4)
assert_almost_equal(cm.kappa(), 0.0, 4)
assert_almost_equal(cm.loevinger_coeff(), 0.0, 4)
assert_almost_equal(cm.cole_coeff(), -0.5, 4)
assert_almost_equal(cm.yule_y(), -1.0, 4)
assert_almost_equal(cm.yule_q(), -1.0, 4)
assert_almost_equal(cm.informedness(), 0.0, 4)
assert_almost_equal(cm.markedness(), 0.0, 4)
def test_2x2_invariants():
"""Alternative implementations should coincide for 2x2 matrices
"""
for _ in xrange(100):
cm = ConfusionMatrix2.from_random_counts(low=0, high=10)
# object idempotency
assert_equal(
cm.to_ccw(),
ConfusionMatrix2.from_ccw(*cm.to_ccw()).to_ccw(),
msg="must be able to convert to tuple and create from tuple")
# pairwise H, C, V
h, c, v = cm.pairwise_hcv()[:3]
check_with_nans(v, geometric_mean(h, c), ensure_nans=False)
# informedness
actual_info = cm.informedness()
expected_info_1 = cm.TPR() + cm.TNR() - 1.0
expected_info_2 = cm.TPR() - cm.FPR()
check_with_nans(actual_info, expected_info_1, 4, ensure_nans=False)
check_with_nans(actual_info, expected_info_2, 4, ensure_nans=False)
# markedness
actual_mark = cm.markedness()
expected_mark_1 = cm.PPV() + cm.NPV() - 1.0
expected_mark_2 = cm.PPV() - cm.FOR()
check_with_nans(actual_mark, expected_mark_1, 4, ensure_nans=False)
check_with_nans(actual_mark, expected_mark_2, 4, ensure_nans=False)
# matthews corr coeff
# actual_mcc = cm.matthews_corr()
# expected_mcc = geometric_mean(actual_info, actual_mark)
# check_with_nans(actual_mcc, expected_mcc, 4, ensure_nans=False)
# kappas
actual_kappa = cm.kappa()
# kappa is the same as harmonic mean of kappa components
expected_kappa_1 = harmonic_mean(*cm.kappas()[:2])
check_with_nans(actual_kappa, expected_kappa_1, 4, ensure_nans=False)
# kappa is the same as accuracy adjusted for chance
expected_kappa_2 = harmonic_mean(
*cm.adjust_to_null(cm.accuracy, model='m3'))
check_with_nans(actual_kappa, expected_kappa_2, 4, ensure_nans=False)
# kappa is the same as Dice coeff adjusted for chance
expected_kappa_3 = harmonic_mean(
*cm.adjust_to_null(cm.dice_coeff, model='m3'))
check_with_nans(actual_kappa, expected_kappa_3, 4, ensure_nans=False)
# odds ratio and Yule's Q
actual_odds_ratio = cm.DOR()
actual_yule_q = cm.yule_q()
expected_yule_q = _div(actual_odds_ratio - 1.0,
actual_odds_ratio + 1.0)
expected_odds_ratio = _div(cm.PLL(), cm.NLL())
check_with_nans(actual_odds_ratio,
expected_odds_ratio,
4,
ensure_nans=False)
check_with_nans(actual_yule_q, expected_yule_q, 4, ensure_nans=False)
# F-score and Dice
expected_f = harmonic_mean(cm.precision(), cm.recall())
actual_f = cm.fscore()
check_with_nans(expected_f, actual_f, 6)
check_with_nans(expected_f, cm.dice_coeff(), 6, ensure_nans=False)
# association coefficients (1)
dice = cm.dice_coeff()
expected_jaccard = _div(dice, 2.0 - dice)
actual_jaccard = cm.jaccard_coeff()
check_with_nans(actual_jaccard, expected_jaccard, 6, ensure_nans=False)
# association coefficients (2)
jaccard = cm.jaccard_coeff()
expected_ss2 = _div(jaccard, 2.0 - jaccard)
actual_ss2 = cm.sokal_sneath_coeff()
check_with_nans(actual_ss2, expected_ss2, 6, ensure_nans=False)
# adjusted ochiai
actual = cm.ochiai_coeff_adj()
expected = harmonic_mean(
*cm.adjust_to_null(cm.ochiai_coeff, model='m3'))
check_with_nans(actual, expected, 6, ensure_nans=False)
def matrix_from_matrices(*args):
arr = args[0]
return ConfusionMatrix2.from_ccw(*arr)