def test_failing_rule_application(self):
chart = cky.make_chart()
grammar = MockGrammar({
('the', ): [('DT', -4)],
('potato', ): [('N', -3), ('V', -20)],
})
parser = cky.CKYParser(grammar)
self.assertFalse(
parser._apply_preterminal_rules('the rock'.split(), chart))
def main_(options, args):
assert options.dep
assert options.out
pairsStream = open(options.dep)
pairsIterator = csiparse.instanceIterator(pairsStream)
# print >> sys.stderr, "C_dep constraints enabled"
if options.dir:
dirStream = open(options.dir)
dirIterator = csiparse.instanceIterator(dirStream)
# print >> sys.stderr, "C_dir constraints enabled"
else:
dirIterator = None
if options.mod:
relsStream = open(options.mod)
relsIterator = csiparse.instanceIterator(relsStream)
# print >> sys.stderr, "C_mod constraints enabled"
else:
relsIterator = None
for sentence in sentenceIterator(fileinput.input(args)):
outfile = open(options.out, "w")
domains, constraints = csiparse.formulateWCSP(sentence, dirIterator,
relsIterator,
pairsIterator, options)
parser = cky.CKYParser(len(sentence))
for constraint in constraints:
parser.addConstraint(constraint)
chart = parser.parse()
#item = chart[0, self.numTokens - 1, "r", True]
#for token in sentence:
# token[DEPREL] = "__"
# token[HEAD] = "__"
for token in sentence:
if len(token) <= DEPREL:
token.extend([None, None])
#print chart[0, len(sentence) - 1, "r", True].r + 1
rightComplete(chart, 0, len(sentence) - 1 + 1, sentence)
if options.non_projective:
approxNonProjective(sentence, parser)
for token in sentence:
outfile.write(" ".join(map(str, token)))
outfile.write("\n")
def test_rule_application(self):
chart = cky.make_chart()
grammar = MockGrammar({
('the', ): [('DT', -4)],
('potato', ): [('N', -3), ('V', -20)],
('DT', 'N'): [('NP', -1), ('VP', -300)],
('DT', 'V'): [('VP', -2)]
})
parser = cky.CKYParser(grammar)
sentence = 'the potato'.split()
# Verify preterminal rule application.
self.assertTrue(parser._apply_preterminal_rules(sentence, chart))
self.assertSetEqual(set([(0, 1), (1, 2)]), set(chart.keys()))
self.assertSetEqual(set(['DT']), self.finite_keys(chart[(0, 1)]))
self.assertEqual(chart[(0, 1)]['DT'].label(), 'DT')
self.assertEqual(chart[(0, 1)]['DT'].logprob(), -4)
self.assertEqual(len(chart[(0, 1)]['DT'].leaves()), 1)
self.assertEqual(chart[(0, 1)]['DT'].leaves()[0], 'the')
self.assertSetEqual(set(['N', 'V']), set(chart[(1, 2)].keys()))
self.assertEqual(chart[(1, 2)]['N'].label(), 'N')
self.assertEqual(chart[(1, 2)]['N'].logprob(), -3)
self.assertEqual(len(chart[(1, 2)]['N'].leaves()), 1)
self.assertEqual(chart[(1, 2)]['N'].leaves()[0], 'potato')
self.assertEqual(chart[(1, 2)]['V'].label(), 'V')
self.assertEqual(chart[(1, 2)]['V'].logprob(), -20)
self.assertEqual(len(chart[(1, 2)]['V'].leaves()), 1)
self.assertEqual(chart[(1, 2)]['V'].leaves()[0], 'potato')
# Verify binary rule application.
parser._apply_binary_rules(len(sentence), chart)
self.assertSetEqual(set([(0, 1), (1, 2), (0, 2)]), set(chart.keys()))
self.assertEqual(self.finite_keys(chart[(0, 2)]), set(['NP', 'VP']))
self.assertEqual(chart[(0, 2)]['NP'].label(), 'NP')
self.assertEqual(chart[(0, 2)]['NP'].logprob(), -8)
self.assertEqual(chart[(0, 2)]['VP'].label(), 'VP')
self.assertEqual(chart[(0, 2)]['VP'].logprob(), -26)
self.assertEqual(chart[(0, 2)]['VP'][1].label(), 'V')
def test_rule_application_three_words(self):
chart = cky.make_chart()
grammar = MockGrammar({
('James', ): [('N', -4)],
('was', ): [('V', -3), ('DT', -20)],
('hungry', ): [('JJ', -5), ('N', -21)],
('N', 'JJ'): [('NP', -1), ('VP', -300)],
('N', 'V'): [('NP', -300)],
('N', 'DT'): [('NP', -51)],
('V', 'JJ'): [('VP', -7)],
('N', 'VP'): [('S', -1)],
('JJ', 'N'): [('NP', -2)]
})
parser = cky.CKYParser(grammar)
sentence = 'James was hungry'.split()
# Verify preterminal rule application.
self.assertTrue(parser._apply_preterminal_rules(sentence, chart))
self.assertSetEqual(set([(0, 1), (1, 2), (2, 3)]), set(chart.keys()))
self.assertSetEqual(set(['N']), self.finite_keys(chart[(0, 1)]))
self.assertEqual(chart[(0, 1)]['N'].label(), 'N')
self.assertEqual(chart[(0, 1)]['N'].logprob(), -4)
self.assertEqual(len(chart[(0, 1)]['N'].leaves()), 1)
self.assertEqual(chart[(0, 1)]['N'].leaves()[0], 'James')
self.assertSetEqual(set(['DT', 'V']), self.finite_keys(chart[(1, 2)]))
self.assertEqual(chart[(1, 2)]['V'].label(), 'V')
self.assertEqual(chart[(1, 2)]['V'].logprob(), -3)
self.assertEqual(len(chart[(1, 2)]['V'].leaves()), 1)
self.assertEqual(chart[(1, 2)]['V'].leaves()[0], 'was')
self.assertEqual(chart[(1, 2)]['DT'].label(), 'DT')
self.assertEqual(chart[(1, 2)]['DT'].logprob(), -20)
self.assertEqual(len(chart[(1, 2)]['DT'].leaves()), 1)
self.assertEqual(chart[(1, 2)]['DT'].leaves()[0], 'was')
self.assertSetEqual(set(['N', 'JJ']), self.finite_keys(chart[(2, 3)]))
self.assertEqual(chart[(2, 3)]['JJ'].label(), 'JJ')
self.assertEqual(chart[(2, 3)]['JJ'].logprob(), -5)
self.assertEqual(len(chart[(2, 3)]['JJ'].leaves()), 1)
self.assertEqual(chart[(2, 3)]['JJ'].leaves()[0], 'hungry')
self.assertEqual(chart[(2, 3)]['N'].label(), 'N')
self.assertEqual(chart[(2, 3)]['N'].logprob(), -21)
self.assertEqual(len(chart[(2, 3)]['N'].leaves()), 1)
self.assertEqual(chart[(2, 3)]['N'].leaves()[0], 'hungry')
# Verify binary rule application.
parser._apply_binary_rules(len(sentence), chart)
self.assertSetEqual(
set([(0, 1), (1, 2), (0, 2), (2, 3), (1, 3), (0, 3)]),
set(chart.keys()))
self.assertEqual(self.finite_keys(chart[(0, 2)]), set(['NP']))
self.assertEqual(self.finite_keys(chart[(1, 3)]), set(['VP']))
self.assertEqual(self.finite_keys(chart[(0, 3)]), set(['S']))
self.assertEqual(chart[(0, 2)]['NP'].logprob(), -4 - 20 - 51)
self.assertEqual(chart[(1, 3)]['VP'].logprob(), -3 - 5 - 7)
self.assertEqual(chart[(0, 3)]['S'].logprob(), -4 - 3 - 5 - 7 - 1)