def test_failing_rule_application(self): chart = cky.make_chart() grammar = MockGrammar({ ('the',): [('DT', -4)], ('potato',): [('N', -3), ('V', -20)],}) parser = cky.CKYParser(grammar) self.assertFalse(parser._apply_preterminal_rules( 'the rock'.split(), chart))
def test_failing_rule_application(self): chart = cky.make_chart() grammar = MockGrammar({ ('the', ): [('DT', -4)], ('potato', ): [('N', -3), ('V', -20)], }) parser = cky.CKYParser(grammar) self.assertFalse( parser._apply_preterminal_rules('the rock'.split(), chart))
def test_rule_application(self): chart = cky.make_chart() grammar = MockGrammar({ ('the', ): [('DT', -4)], ('potato', ): [('N', -3), ('V', -20)], ('DT', 'N'): [('NP', -1), ('VP', -300)], ('DT', 'V'): [('VP', -2)] }) parser = cky.CKYParser(grammar) sentence = 'the potato'.split() # Verify preterminal rule application. self.assertTrue(parser._apply_preterminal_rules(sentence, chart)) self.assertSetEqual(set([(0, 1), (1, 2)]), set(chart.keys())) self.assertSetEqual(set(['DT']), self.finite_keys(chart[(0, 1)])) self.assertEqual(chart[(0, 1)]['DT'].label(), 'DT') self.assertEqual(chart[(0, 1)]['DT'].logprob(), -4) self.assertEqual(len(chart[(0, 1)]['DT'].leaves()), 1) self.assertEqual(chart[(0, 1)]['DT'].leaves()[0], 'the') self.assertSetEqual(set(['N', 'V']), set(chart[(1, 2)].keys())) self.assertEqual(chart[(1, 2)]['N'].label(), 'N') self.assertEqual(chart[(1, 2)]['N'].logprob(), -3) self.assertEqual(len(chart[(1, 2)]['N'].leaves()), 1) self.assertEqual(chart[(1, 2)]['N'].leaves()[0], 'potato') self.assertEqual(chart[(1, 2)]['V'].label(), 'V') self.assertEqual(chart[(1, 2)]['V'].logprob(), -20) self.assertEqual(len(chart[(1, 2)]['V'].leaves()), 1) self.assertEqual(chart[(1, 2)]['V'].leaves()[0], 'potato') # Verify binary rule application. parser._apply_binary_rules(len(sentence), chart) self.assertSetEqual(set([(0, 1), (1, 2), (0, 2)]), set(chart.keys())) self.assertEqual(self.finite_keys(chart[(0, 2)]), set(['NP', 'VP'])) self.assertEqual(chart[(0, 2)]['NP'].label(), 'NP') self.assertEqual(chart[(0, 2)]['NP'].logprob(), -8) self.assertEqual(chart[(0, 2)]['VP'].label(), 'VP') self.assertEqual(chart[(0, 2)]['VP'].logprob(), -26) self.assertEqual(chart[(0, 2)]['VP'][1].label(), 'V')
def test_rule_application(self): chart = cky.make_chart() grammar = MockGrammar({ ('the',): [('DT', -4)], ('potato',): [('N', -3), ('V', -20)], ('DT', 'N'): [('NP', -1), ('VP', -300)], ('DT', 'V'): [('VP', -2)]}) parser = cky.CKYParser(grammar) sentence = 'the potato'.split() # Verify preterminal rule application. self.assertTrue(parser._apply_preterminal_rules( sentence, chart)) self.assertSetEqual(set([(0, 1), (1, 2)]), set(chart.keys())) self.assertSetEqual(set(['DT']), self.finite_keys(chart[(0, 1)])) self.assertEqual(chart[(0, 1)]['DT'].label(), 'DT') self.assertEqual(chart[(0, 1)]['DT'].logprob(), -4) self.assertEqual(len(chart[(0, 1)]['DT'].leaves()), 1) self.assertEqual(chart[(0, 1)]['DT'].leaves()[0], 'the') self.assertSetEqual(set(['N', 'V']), set(chart[(1, 2)].keys())) self.assertEqual(chart[(1, 2)]['N'].label(), 'N') self.assertEqual(chart[(1, 2)]['N'].logprob(), -3) self.assertEqual(len(chart[(1, 2)]['N'].leaves()), 1) self.assertEqual(chart[(1, 2)]['N'].leaves()[0], 'potato') self.assertEqual(chart[(1, 2)]['V'].label(), 'V') self.assertEqual(chart[(1, 2)]['V'].logprob(), -20) self.assertEqual(len(chart[(1, 2)]['V'].leaves()), 1) self.assertEqual(chart[(1, 2)]['V'].leaves()[0], 'potato') # Verify binary rule application. parser._apply_binary_rules(len(sentence), chart) self.assertSetEqual(set([(0, 1), (1, 2), (0, 2)]), set(chart.keys())) self.assertEqual(self.finite_keys(chart[(0, 2)]), set(['NP', 'VP'])) self.assertEqual(chart[(0, 2)]['NP'].label(), 'NP') self.assertEqual(chart[(0, 2)]['NP'].logprob(), -8) self.assertEqual(chart[(0, 2)]['VP'].label(), 'VP') self.assertEqual(chart[(0, 2)]['VP'].logprob(), -26) self.assertEqual(chart[(0, 2)]['VP'][1].label(), 'V')
def test_rule_application_three_words(self): chart = cky.make_chart() grammar = MockGrammar({ ('James', ): [('N', -4)], ('was', ): [('V', -3), ('DT', -20)], ('hungry', ): [('JJ', -5), ('N', -21)], ('N', 'JJ'): [('NP', -1), ('VP', -300)], ('N', 'V'): [('NP', -300)], ('N', 'DT'): [('NP', -51)], ('V', 'JJ'): [('VP', -7)], ('N', 'VP'): [('S', -1)], ('JJ', 'N'): [('NP', -2)] }) parser = cky.CKYParser(grammar) sentence = 'James was hungry'.split() # Verify preterminal rule application. self.assertTrue(parser._apply_preterminal_rules(sentence, chart)) self.assertSetEqual(set([(0, 1), (1, 2), (2, 3)]), set(chart.keys())) self.assertSetEqual(set(['N']), self.finite_keys(chart[(0, 1)])) self.assertEqual(chart[(0, 1)]['N'].label(), 'N') self.assertEqual(chart[(0, 1)]['N'].logprob(), -4) self.assertEqual(len(chart[(0, 1)]['N'].leaves()), 1) self.assertEqual(chart[(0, 1)]['N'].leaves()[0], 'James') self.assertSetEqual(set(['DT', 'V']), self.finite_keys(chart[(1, 2)])) self.assertEqual(chart[(1, 2)]['V'].label(), 'V') self.assertEqual(chart[(1, 2)]['V'].logprob(), -3) self.assertEqual(len(chart[(1, 2)]['V'].leaves()), 1) self.assertEqual(chart[(1, 2)]['V'].leaves()[0], 'was') self.assertEqual(chart[(1, 2)]['DT'].label(), 'DT') self.assertEqual(chart[(1, 2)]['DT'].logprob(), -20) self.assertEqual(len(chart[(1, 2)]['DT'].leaves()), 1) self.assertEqual(chart[(1, 2)]['DT'].leaves()[0], 'was') self.assertSetEqual(set(['N', 'JJ']), self.finite_keys(chart[(2, 3)])) self.assertEqual(chart[(2, 3)]['JJ'].label(), 'JJ') self.assertEqual(chart[(2, 3)]['JJ'].logprob(), -5) self.assertEqual(len(chart[(2, 3)]['JJ'].leaves()), 1) self.assertEqual(chart[(2, 3)]['JJ'].leaves()[0], 'hungry') self.assertEqual(chart[(2, 3)]['N'].label(), 'N') self.assertEqual(chart[(2, 3)]['N'].logprob(), -21) self.assertEqual(len(chart[(2, 3)]['N'].leaves()), 1) self.assertEqual(chart[(2, 3)]['N'].leaves()[0], 'hungry') # Verify binary rule application. parser._apply_binary_rules(len(sentence), chart) self.assertSetEqual( set([(0, 1), (1, 2), (0, 2), (2, 3), (1, 3), (0, 3)]), set(chart.keys())) self.assertEqual(self.finite_keys(chart[(0, 2)]), set(['NP'])) self.assertEqual(self.finite_keys(chart[(1, 3)]), set(['VP'])) self.assertEqual(self.finite_keys(chart[(0, 3)]), set(['S'])) self.assertEqual(chart[(0, 2)]['NP'].logprob(), -4 - 20 - 51) self.assertEqual(chart[(1, 3)]['VP'].logprob(), -3 - 5 - 7) self.assertEqual(chart[(0, 3)]['S'].logprob(), -4 - 3 - 5 - 7 - 1)
def test_rule_application_three_words(self): chart = cky.make_chart() grammar = MockGrammar({ ('James',): [('N', -4)], ('was',): [('V', -3), ('DT', -20)], ('hungry',): [('JJ', -5), ('N', -21)], ('N', 'JJ'): [('NP', -1), ('VP', -300)], ('N', 'V'): [('NP', -300)], ('N', 'DT'): [('NP', -51)], ('V', 'JJ'): [('VP', -7)], ('N', 'VP'): [('S', -1)], ('JJ', 'N'): [('NP', -2)]}) parser = cky.CKYParser(grammar) sentence = 'James was hungry'.split() # Verify preterminal rule application. self.assertTrue(parser._apply_preterminal_rules( sentence, chart)) self.assertSetEqual(set([(0, 1), (1, 2), (2, 3)]), set(chart.keys())) self.assertSetEqual(set(['N']), self.finite_keys(chart[(0, 1)])) self.assertEqual(chart[(0, 1)]['N'].label(), 'N') self.assertEqual(chart[(0, 1)]['N'].logprob(), -4) self.assertEqual(len(chart[(0, 1)]['N'].leaves()), 1) self.assertEqual(chart[(0, 1)]['N'].leaves()[0], 'James') self.assertSetEqual(set(['DT', 'V']), self.finite_keys(chart[(1, 2)])) self.assertEqual(chart[(1, 2)]['V'].label(), 'V') self.assertEqual(chart[(1, 2)]['V'].logprob(), -3) self.assertEqual(len(chart[(1, 2)]['V'].leaves()), 1) self.assertEqual(chart[(1, 2)]['V'].leaves()[0], 'was') self.assertEqual(chart[(1, 2)]['DT'].label(), 'DT') self.assertEqual(chart[(1, 2)]['DT'].logprob(), -20) self.assertEqual(len(chart[(1, 2)]['DT'].leaves()), 1) self.assertEqual(chart[(1, 2)]['DT'].leaves()[0], 'was') self.assertSetEqual(set(['N', 'JJ']), self.finite_keys(chart[(2, 3)])) self.assertEqual(chart[(2, 3)]['JJ'].label(), 'JJ') self.assertEqual(chart[(2, 3)]['JJ'].logprob(), -5) self.assertEqual(len(chart[(2, 3)]['JJ'].leaves()), 1) self.assertEqual(chart[(2, 3)]['JJ'].leaves()[0], 'hungry') self.assertEqual(chart[(2, 3)]['N'].label(), 'N') self.assertEqual(chart[(2, 3)]['N'].logprob(), -21) self.assertEqual(len(chart[(2, 3)]['N'].leaves()), 1) self.assertEqual(chart[(2, 3)]['N'].leaves()[0], 'hungry') # Verify binary rule application. parser._apply_binary_rules(len(sentence), chart) self.assertSetEqual( set([(0, 1), (1, 2), (0, 2), (2, 3), (1, 3), (0, 3)]), set(chart.keys())) self.assertEqual(self.finite_keys(chart[(0, 2)]), set(['NP'])) self.assertEqual(self.finite_keys(chart[(1, 3)]), set(['VP'])) self.assertEqual(self.finite_keys(chart[(0, 3)]), set(['S'])) self.assertEqual(chart[(0, 2)]['NP'].logprob(), -4 -20 -51) self.assertEqual(chart[(1, 3)]['VP'].logprob(), -3 -5 -7) self.assertEqual(chart[(0, 3)]['S'].logprob(), -4 -3 -5 -7 -1)