示例#1
0
 def apply(self, tasks):
     print('Create regions...')
     random.shuffle(tasks)
     regions = self.create_regions(tasks[:100])
     print(f'Num regions: {len(regions)}')
     L_train = self.applier.apply(regions)
     lfa = LFAnalysis(L=L_train, lfs=self.lfs)
     confl = lfa.lf_conflicts()
     cov = lfa.lf_coverages()
     confli = np.argsort(confl)
     lfs_sorted = [self.lfs[i] for i in confli]
     out = []
     for lf, cf, cv in zip(lfs_sorted, confl[confli], cov[confli]):
         print(lf.name, cf, cv)
         out.append({'lop': lf.name, 'conflict': cf, 'coverage': cv})
     return out
class TestAnalysis(unittest.TestCase):
    def setUp(self) -> None:
        self.lfa = LFAnalysis(np.array(L))
        self.lfa_wo_abstain = LFAnalysis(np.array(L_wo_abstain))
        self.Y = np.array(Y)

    def test_label_coverage(self) -> None:
        self.assertEqual(self.lfa.label_coverage(), 5 / 6)

    def test_label_overlap(self) -> None:
        self.assertEqual(self.lfa.label_overlap(), 4 / 6)

    def test_label_conflict(self) -> None:
        self.assertEqual(self.lfa.label_conflict(), 3 / 6)

    def test_lf_polarities(self) -> None:
        polarities = self.lfa.lf_polarities()
        self.assertEqual(polarities, [[1, 2], [], [0, 2], [2], [0, 1], [0]])

    def test_lf_coverages(self) -> None:
        coverages = self.lfa.lf_coverages()
        coverages_expected = [3 / 6, 0, 3 / 6, 2 / 6, 2 / 6, 4 / 6]
        np.testing.assert_array_almost_equal(coverages,
                                             np.array(coverages_expected))

    def test_lf_overlaps(self) -> None:
        overlaps = self.lfa.lf_overlaps(normalize_by_coverage=False)
        overlaps_expected = [3 / 6, 0, 3 / 6, 1 / 6, 2 / 6, 4 / 6]
        np.testing.assert_array_almost_equal(overlaps,
                                             np.array(overlaps_expected))

        overlaps = self.lfa.lf_overlaps(normalize_by_coverage=True)
        overlaps_expected = [1, 0, 1, 1 / 2, 1, 1]
        np.testing.assert_array_almost_equal(overlaps,
                                             np.array(overlaps_expected))

    def test_lf_conflicts(self) -> None:
        conflicts = self.lfa.lf_conflicts(normalize_by_overlaps=False)
        conflicts_expected = [3 / 6, 0, 2 / 6, 1 / 6, 2 / 6, 3 / 6]
        np.testing.assert_array_almost_equal(conflicts,
                                             np.array(conflicts_expected))

        conflicts = self.lfa.lf_conflicts(normalize_by_overlaps=True)
        conflicts_expected = [1, 0, 2 / 3, 1, 1, 3 / 4]
        np.testing.assert_array_almost_equal(conflicts,
                                             np.array(conflicts_expected))

    def test_lf_empirical_accuracies(self) -> None:
        accs = self.lfa.lf_empirical_accuracies(self.Y)
        accs_expected = [1 / 3, 0, 1 / 3, 1 / 2, 1 / 2, 2 / 4]
        np.testing.assert_array_almost_equal(accs, np.array(accs_expected))

    def test_lf_empirical_probs(self) -> None:
        P_emp = self.lfa.lf_empirical_probs(self.Y, 3)
        P = np.array([
            [[1 / 2, 1, 0], [0, 0, 0], [1 / 2, 0, 1 / 2], [0, 0, 1 / 2]],
            [[1, 1, 1], [0, 0, 0], [0, 0, 0], [0, 0, 0]],
            [[0, 1, 1 / 2], [1 / 2, 0, 1 / 2], [0, 0, 0], [1 / 2, 0, 0]],
            [[1, 1 / 2, 1 / 2], [0, 0, 0], [0, 0, 0], [0, 1 / 2, 1 / 2]],
            [[1 / 2, 1, 1 / 2], [1 / 2, 0, 0], [0, 0, 1 / 2], [0, 0, 0]],
            [[0, 1, 0], [1, 0, 1], [0, 0, 0], [0, 0, 0]],
        ])
        np.testing.assert_array_almost_equal(P, P_emp)

    def test_lf_summary(self) -> None:
        df = self.lfa.lf_summary(self.Y, est_weights=None)
        df_expected = pd.DataFrame({
            "Polarity": [[1, 2], [], [0, 2], [2], [0, 1], [0]],
            "Coverage": [3 / 6, 0, 3 / 6, 2 / 6, 2 / 6, 4 / 6],
            "Overlaps": [3 / 6, 0, 3 / 6, 1 / 6, 2 / 6, 4 / 6],
            "Conflicts": [3 / 6, 0, 2 / 6, 1 / 6, 2 / 6, 3 / 6],
            "Correct": [1, 0, 1, 1, 1, 2],
            "Incorrect": [2, 0, 2, 1, 1, 2],
            "Emp. Acc.": [1 / 3, 0, 1 / 3, 1 / 2, 1 / 2, 2 / 4],
        })
        pd.testing.assert_frame_equal(df.round(6), df_expected.round(6))

        df = self.lfa.lf_summary(Y=None, est_weights=None)
        df_expected = pd.DataFrame({
            "Polarity": [[1, 2], [], [0, 2], [2], [0, 1], [0]],
            "Coverage": [3 / 6, 0, 3 / 6, 2 / 6, 2 / 6, 4 / 6],
            "Overlaps": [3 / 6, 0, 3 / 6, 1 / 6, 2 / 6, 4 / 6],
            "Conflicts": [3 / 6, 0, 2 / 6, 1 / 6, 2 / 6, 3 / 6],
        })
        pd.testing.assert_frame_equal(df.round(6), df_expected.round(6))

        est_weights = [1, 0, 1, 1, 1, 0.5]
        names = list("abcdef")
        lfs = [LabelingFunction(s, f) for s in names]
        lfa = LFAnalysis(np.array(L), lfs)
        df = lfa.lf_summary(self.Y, est_weights=est_weights)
        df_expected = pd.DataFrame({
            "j": [0, 1, 2, 3, 4, 5],
            "Polarity": [[1, 2], [], [0, 2], [2], [0, 1], [0]],
            "Coverage": [3 / 6, 0, 3 / 6, 2 / 6, 2 / 6, 4 / 6],
            "Overlaps": [3 / 6, 0, 3 / 6, 1 / 6, 2 / 6, 4 / 6],
            "Conflicts": [3 / 6, 0, 2 / 6, 1 / 6, 2 / 6, 3 / 6],
            "Correct": [1, 0, 1, 1, 1, 2],
            "Incorrect": [2, 0, 2, 1, 1, 2],
            "Emp. Acc.": [1 / 3, 0, 1 / 3, 1 / 2, 1 / 2, 2 / 4],
            "Learned Weight": [1, 0, 1, 1, 1, 0.5],
        }).set_index(pd.Index(names))
        pd.testing.assert_frame_equal(df.round(6), df_expected.round(6))

    def test_wrong_number_of_lfs(self) -> None:
        with self.assertRaisesRegex(ValueError, "Number of LFs"):
            LFAnalysis(np.array(L), [LabelingFunction(s, f) for s in "ab"])

    def test_lf_summary_without_abstain(self) -> None:
        df = self.lfa_wo_abstain.lf_summary(self.Y + 4, est_weights=None)
        df_expected = pd.DataFrame({
            "Polarity": [[3, 4, 5], [3, 4], [3, 4, 5], [4, 5], [3, 4, 5], [3]],
            "Coverage": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
            "Overlaps": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
            "Conflicts": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
            "Correct": [1, 1, 1, 3, 1, 0],
            "Incorrect": [5, 5, 5, 3, 5, 6],
            "Emp. Acc.": [1 / 6, 1 / 6, 1 / 6, 3 / 6, 1 / 6, 0],
        })
        pd.testing.assert_frame_equal(df.round(6), df_expected.round(6))