def test_krippendorffs_alpha2(self):
# test example from
# http://en.wikipedia.org/wiki/Krippendorff%27s_Alpha#A_computational_example
kr = self.krippendorff_example
alpha = pma.krippendorffs_alpha(kr.annotations,
metric_func=pmd.binary_distance)
self.assertAlmostEqual(alpha, kr.alpha_binary, 3)
alpha = pma.krippendorffs_alpha(kr.annotations,
metric_func=pmd.diagonal_distance)
self.assertAlmostEqual(alpha, kr.alpha_diagonal, 3)
def test_all_invalid(self):
# behavior: all measures should return np.nan for a set of
# annotations with no valid entry
anno = self.invalid_test.annotations
self.assertTrue(
np.isnan(pma.scotts_pi(anno[:, 0], anno[:, 1], nclasses=4)))
self.assertTrue(
np.isnan(pma.cohens_kappa(anno[:, 0], anno[:, 1], nclasses=3)))
self.assertTrue(
np.isnan(
pma.cohens_weighted_kappa(anno[:, 0], anno[:, 1], nclasses=5)))
self.assertTrue(np.isnan(pma.fleiss_kappa(anno, nclasses=4)))
self.assertTrue(np.isnan(pma.krippendorffs_alpha(anno, nclasses=7)))
self.assertTrue(
np.isnan(pmc.pearsons_rho(anno[:, 0], anno[:, 1], nclasses=4)))
self.assertTrue(
np.isnan(pmc.spearmans_rho(anno[:, 0], anno[:, 1], nclasses=4)))
self.assertTrue(np.isnan(pmc.cronbachs_alpha(anno, nclasses=4)))
def test_all_invalid(self):
# behavior: all measures should return np.nan for a set of
# annotations with no valid entry
anno = self.invalid_test.annotations
self.assert_(
np.isnan(pma.scotts_pi(anno[:,0], anno[:,1], nclasses=4))
)
self.assert_(
np.isnan(pma.cohens_kappa(anno[:,0], anno[:,1], nclasses=3))
)
self.assert_(
np.isnan(pma.cohens_weighted_kappa(anno[:,0], anno[:,1],
nclasses=5))
)
self.assert_(
np.isnan(pma.fleiss_kappa(anno, nclasses=4))
)
self.assert_(
np.isnan(pma.krippendorffs_alpha(anno, nclasses=7))
)
self.assert_(
np.isnan(pmc.pearsons_rho(anno[:,0], anno[:,1], nclasses=4))
)
self.assert_(
np.isnan(pmc.spearmans_rho(anno[:,0], anno[:,1], nclasses=4))
)
self.assert_(
np.isnan(pmc.cronbachs_alpha(anno, nclasses=4))
)
def test_krippendorffs_alpha(self):
# test basic functionality with full agreement, missing annotations
fa = self.full_agreement
self.assertAlmostEqual(pma.krippendorffs_alpha(fa.annotations),
1.0, 6)