def test_all_invalid(self): # behavior: all measures should return np.nan for a set of # annotations with no valid entry anno = self.invalid_test.annotations self.assertTrue( np.isnan(pma.scotts_pi(anno[:, 0], anno[:, 1], nclasses=4))) self.assertTrue( np.isnan(pma.cohens_kappa(anno[:, 0], anno[:, 1], nclasses=3))) self.assertTrue( np.isnan( pma.cohens_weighted_kappa(anno[:, 0], anno[:, 1], nclasses=5))) self.assertTrue(np.isnan(pma.fleiss_kappa(anno, nclasses=4))) self.assertTrue(np.isnan(pma.krippendorffs_alpha(anno, nclasses=7))) self.assertTrue( np.isnan(pmc.pearsons_rho(anno[:, 0], anno[:, 1], nclasses=4))) self.assertTrue( np.isnan(pmc.spearmans_rho(anno[:, 0], anno[:, 1], nclasses=4))) self.assertTrue(np.isnan(pmc.cronbachs_alpha(anno, nclasses=4)))
def test_fleiss_kappa(self): # test basic functionality with full agreement, missing annotations fa = self.full_agreement self.assertAlmostEqual(pma.fleiss_kappa(fa.annotations, fa.nclasses), 1.0, 6) # unequal number of annotators per row fa.annotations[0,:] = MV self.assertRaises(PyannoValueError, pma.fleiss_kappa, fa.annotations)
def test_all_invalid(self): # behavior: all measures should return np.nan for a set of # annotations with no valid entry anno = self.invalid_test.annotations self.assert_( np.isnan(pma.scotts_pi(anno[:,0], anno[:,1], nclasses=4)) ) self.assert_( np.isnan(pma.cohens_kappa(anno[:,0], anno[:,1], nclasses=3)) ) self.assert_( np.isnan(pma.cohens_weighted_kappa(anno[:,0], anno[:,1], nclasses=5)) ) self.assert_( np.isnan(pma.fleiss_kappa(anno, nclasses=4)) ) self.assert_( np.isnan(pma.krippendorffs_alpha(anno, nclasses=7)) ) self.assert_( np.isnan(pmc.pearsons_rho(anno[:,0], anno[:,1], nclasses=4)) ) self.assert_( np.isnan(pmc.spearmans_rho(anno[:,0], anno[:,1], nclasses=4)) ) self.assert_( np.isnan(pmc.cronbachs_alpha(anno, nclasses=4)) )