Esempio n. 1
0
 def test_matrix(self):
     ke = self.krippendorff_example
     mat = pyanno.measures.pairwise_matrix(pma.cohens_kappa,
                                           ke.annotations, nclasses=4)
     self.assertAlmostEqual(mat[1,1], 1., 6)
     kappa = pma.cohens_kappa(ke.annotations[:,2], ke.annotations[:,0])
     self.assertAlmostEqual(mat[2,0], kappa)
Esempio n. 2
0
    def test_cohens_kappa(self):
        # test basic functionality with full agreement, missing annotations
        fa = self.full_agreement

        self.assertAlmostEqual(pma.cohens_kappa(fa.annotations[:,0],
                                                fa.annotations[:,1]),
                               1.0, 6)
Esempio n. 3
0
    def test_all_invalid(self):
        # behavior: all measures should return np.nan for a set of
        # annotations with no valid entry

        anno = self.invalid_test.annotations

        self.assertTrue(
            np.isnan(pma.scotts_pi(anno[:, 0], anno[:, 1], nclasses=4)))

        self.assertTrue(
            np.isnan(pma.cohens_kappa(anno[:, 0], anno[:, 1], nclasses=3)))

        self.assertTrue(
            np.isnan(
                pma.cohens_weighted_kappa(anno[:, 0], anno[:, 1], nclasses=5)))

        self.assertTrue(np.isnan(pma.fleiss_kappa(anno, nclasses=4)))

        self.assertTrue(np.isnan(pma.krippendorffs_alpha(anno, nclasses=7)))

        self.assertTrue(
            np.isnan(pmc.pearsons_rho(anno[:, 0], anno[:, 1], nclasses=4)))

        self.assertTrue(
            np.isnan(pmc.spearmans_rho(anno[:, 0], anno[:, 1], nclasses=4)))

        self.assertTrue(np.isnan(pmc.cronbachs_alpha(anno, nclasses=4)))
Esempio n. 4
0
 def test_cohens_weighted_kappa2(self):
     # cohen's weighted kappa is the same as cohen's kappa when
     # the weights are 0. on the diagonal and 1. elsewhere
     anno1 = np.array([0, 0, 1, 2, 1, MV, 3])
     anno2 = np.array([0, MV, 1, 0, 1, MV, 2])
     weighted_kappa = pma.cohens_weighted_kappa(anno1, anno2,
                                                pmd.binary_distance)
     cohens_kappa = pma.cohens_kappa(anno1, anno2)
     self.assertAlmostEqual(weighted_kappa, cohens_kappa, 6)
Esempio n. 5
0
    def test_all_invalid(self):
        # behavior: all measures should return np.nan for a set of
        # annotations with no valid entry

        anno = self.invalid_test.annotations

        self.assert_(
            np.isnan(pma.scotts_pi(anno[:,0], anno[:,1], nclasses=4))
        )

        self.assert_(
            np.isnan(pma.cohens_kappa(anno[:,0], anno[:,1], nclasses=3))
        )

        self.assert_(
            np.isnan(pma.cohens_weighted_kappa(anno[:,0], anno[:,1],
                                               nclasses=5))
        )

        self.assert_(
            np.isnan(pma.fleiss_kappa(anno, nclasses=4))
        )

        self.assert_(
            np.isnan(pma.krippendorffs_alpha(anno, nclasses=7))
        )

        self.assert_(
            np.isnan(pmc.pearsons_rho(anno[:,0], anno[:,1], nclasses=4))
        )

        self.assert_(
            np.isnan(pmc.spearmans_rho(anno[:,0], anno[:,1], nclasses=4))
        )

        self.assert_(
            np.isnan(pmc.cronbachs_alpha(anno, nclasses=4))
        )
Esempio n. 6
0
    def test_cohens_kappa2(self):
        ce = self.cohen_example

        kappa = pma.cohens_kappa(ce.annotations[:,0],
                                ce.annotations[:,1])
        self.assertAlmostEqual(kappa, ce.kappa, 6)