def parse(self, lcls, head, tokens, output):
grammar = ParseRuleSet()
for rule_name, rule in sorted(lcls.items(), reverse=True):
if isinstance(rule, ParseRule):
grammar.add(rule)
logging_builder = LoggingBuilder(lcls)
parse(grammar, head, tokens).apply(logging_builder)
print(logging_builder.get_text())
logging_builder.compare(output, self)
def parse(self, lcls, head, tokens, output):
grammar = ParseRuleSet()
for rule_name, rule in sorted(lcls.items(), reverse=True):
if isinstance(rule, ParseRule):
grammar.add(rule)
logging_builder = LoggingBuilder(lcls)
parse(grammar, head, tokens).apply(logging_builder)
print(logging_builder.get_text())
logging_builder.compare(output, self)
def ambig(self, text, start_index=None, end_index=None, values=None):
with self.assertRaises(AmbiguousParseError) as cm:
parse(self.p, "top", lex(text)).single()
e = cm.exception
if start_index is not None:
self.assertEqual(e.start_index, start_index)
if end_index is not None:
self.assertEqual(e.end_index, end_index)
if values is not None:
self.assertEqual(e.values, values)
def ambig(self, text, start_index=None, end_index=None, values=None):
with self.assertRaises(AmbiguousParseError) as cm:
parse(self.p, "top", lex(text)).single()
e = cm.exception
if start_index is not None:
self.assertEqual(e.start_index, start_index)
if end_index is not None:
self.assertEqual(e.end_index, end_index)
if values is not None:
self.assertEqual(e.values, values)
def no_parse(self, text, at_index, encountered=None, expected_terminals=None, expected=None):
with self.assertRaises(NoParseError) as cm:
parse(self.p, "top", lex(text)).single()
e = cm.exception
self.assertEqual(e.start_index, at_index)
self.assertEqual(e.end_index, at_index)
if encountered is not None:
self.assertEqual(e.encountered, encountered)
if expected_terminals is not None:
self.assertEqual(set(map(repr,e.expected_terminals)), set(map(repr,expected_terminals)))
if expected is not None:
self.assertEqual(set(map(repr,e.expected)), set(map(repr,expected)))
def test_complexity_lr(self):
p = self.p
p.add(ParseRule("1", "top", [Terminal("a"), NonTerminal("top")]))
p.add(ParseRule("2", "top", []))
n = 1000
forest = parse(self.p, "top", lex("a " * n))
self.assertEqual(forest.count(), 1)
self.assertEqual(forest.internal_node_count, 3 + 3 * n)
def test_complexity_lr(self):
p = self.p
p.add(ParseRule("1","top",[Terminal("a"), NonTerminal("top")]))
p.add(ParseRule("2","top",[]))
n = 1000
forest = parse(self.p, "top", lex("a " * n))
self.assertEqual(forest.count(), 1)
self.assertEqual(forest.internal_node_count, 3 + 3 * n)
def no_parse(self,
text,
at_index,
encountered=None,
expected_terminals=None,
expected=None):
with self.assertRaises(NoParseError) as cm:
parse(self.p, "top", lex(text)).single()
e = cm.exception
self.assertEqual(e.start_index, at_index)
self.assertEqual(e.end_index, at_index)
if encountered is not None:
self.assertEqual(e.encountered, encountered)
if expected_terminals is not None:
self.assertEqual(set(map(repr, e.expected_terminals)),
set(map(repr, expected_terminals)))
if expected is not None:
self.assertEqual(set(map(repr, e.expected)),
set(map(repr, expected)))
def test_complexity(self):
# Correctly written an Earley parse should be O(n) for several sorts of grammar
# and O(n^3) worse case
p = self.p
p.add(ParseRule("1", "top", [NonTerminal("a", star=True)]))
p.add(ParseRule("2", "a", [Terminal("a")]))
p.add(ParseRule("3", "a", [Terminal("a")]))
# Assuming the EarleyParser is correct, this should only take linear time/space to compute, despite 2^n possible
# parses. Note that you can set this value well beyond Python's recursion depth
n = 1000
forest = parse(self.p, "top", lex("a " * n))
self.assertEqual(forest.count(), 2**n)
self.assertEqual(forest.internal_node_count, 4 + 5 * n)
def test_complexity(self):
# Correctly written an Earley parse should be O(n) for several sorts of grammar
# and O(n^3) worse case
p = self.p
p.add(ParseRule("1","top",[NonTerminal("a", star=True)]))
p.add(ParseRule("2","a",[Terminal("a")]))
p.add(ParseRule("3","a",[Terminal("a")]))
# Assuming the EarleyParser is correct, this should only take linear time/space to compute, despite 2^n possible
# parses. Note that you can set this value well beyond Python's recursion depth
n = 1000
forest = parse(self.p, "top", lex("a " * n))
self.assertEqual(forest.count(), 2**n)
self.assertEqual(forest.internal_node_count, 4 + 5 * n)
def parse(self, text, trees):
parse_forest = parse(self.p, "top", lex(text))
result_trees1 = set(map(simplify_parse_tree, parse_forest))
result_trees2 = set(map(simplify_parse_tree, parse_forest.all()))
expected_trees = set(trees)
self.assertSetEqual(result_trees1, result_trees2)
extra_results = result_trees1 - expected_trees
missing_results = expected_trees - result_trees1
t = []
if extra_results:
t.append("Parsed unexpected parse trees:\n" + "\n".join(extra_results))
if missing_results:
t.append("Expected parse trees:\n" + "\n".join(missing_results))
if t:
self.assertFalse(extra_results or missing_results, "\n".join(t))
def parse(self, text, trees):
parse_forest = parse(self.p, "top", lex(text))
result_trees1 = set(map(simplify_parse_tree, parse_forest))
result_trees2 = set(map(simplify_parse_tree, parse_forest.all()))
expected_trees = set(trees)
self.assertSetEqual(result_trees1, result_trees2)
extra_results = result_trees1 - expected_trees
missing_results = expected_trees - result_trees1
t = []
if extra_results:
t.append("Parsed unexpected parse trees:\n" +
"\n".join(extra_results))
if missing_results:
t.append("Expected parse trees:\n" + "\n".join(missing_results))
if t:
self.assertFalse(extra_results or missing_results, "\n".join(t))
# This is the example is explained in more detail in usage.rst
from axaxaxas import ParseRule, ParseRuleSet, Terminal as T, NonTerminal as NT
grammar = ParseRuleSet()
grammar.add(ParseRule("sentence", [NT("noun"), NT("verb"), NT("noun")]))
grammar.add(ParseRule("noun", [T("man")]))
grammar.add(ParseRule("noun", [T("dog")]))
grammar.add(ParseRule("verb", [T("bites")]))
###
from axaxaxas import parse
parse_forest = parse(grammar, "sentence", "man bites dog".split())
print(parse_forest.single())
###
grammar.add(ParseRule("relative", [T("step", optional=True), T("sister")]))
grammar.add(ParseRule("relative", [T("great", star=True), T("grandfather")]))
print(parse(grammar, "relative", "sister".split()).single())
print(parse(grammar, "relative", "step sister".split()).single())
print(parse(grammar, "relative", "grandfather".split()).single())
print(parse(grammar, "relative", "great great grandfather".split()).single())
###
grammar.add(ParseRule("described relative", [NT("adjective", star=True), NT("relative")]))
grammar.add(ParseRule("adjective", [T("awesome")]))
grammar.add(ParseRule("adjective", [T("great")]))
def infinite(self, text):
with self.assertRaises(InfiniteParseError):
parse(self.p, "top", lex(text)).single()
def roundtrip(self, text):
round_trip = unlex(unparse(parse(self.p, "top", lex(text)).single()))
self.assertEqual(text.strip(), round_trip.strip())
# This is the example is explained in more detail in usage.rst
from axaxaxas import ParseRule, ParseRuleSet, Terminal as T, NonTerminal as NT
grammar = ParseRuleSet()
grammar.add(ParseRule("sentence", [NT("noun"), NT("verb"), NT("noun")]))
grammar.add(ParseRule("noun", [T("man")]))
grammar.add(ParseRule("noun", [T("dog")]))
grammar.add(ParseRule("verb", [T("bites")]))
###
from axaxaxas import parse
parse_forest = parse(grammar, "sentence", "man bites dog".split())
print(parse_forest.single())
###
grammar.add(ParseRule("relative", [T("step", optional=True), T("sister")]))
grammar.add(ParseRule("relative", [T("great", star=True), T("grandfather")]))
print(parse(grammar, "relative", "sister".split()).single())
print(parse(grammar, "relative", "step sister".split()).single())
print(parse(grammar, "relative", "grandfather".split()).single())
print(parse(grammar, "relative", "great great grandfather".split()).single())
###
grammar.add(
ParseRule("described relative",
[NT("adjective", star=True),
def infinite(self, text):
with self.assertRaises(InfiniteParseError):
parse(self.p, "top", lex(text)).single()
def roundtrip(self, text):
round_trip = unlex(unparse(parse(self.p, "top", lex(text)).single()))
self.assertEqual(text.strip(), round_trip.strip())