def test_group_complex():
grammar_str = r'''
@obj
s: (b c)*[comma];
s: (b c)*[comma] a=(a+ (b | c)*)+[comma];
terminals
a: "a";
b: "b";
c: "c";
comma: ",";
'''
grammar = Grammar.from_string(grammar_str)
assert len(grammar.get_productions('s_g1')) == 1
# B | C
prods = grammar.get_productions('s_g3')
assert len(prods) == 2
assert prods[0].rhs[0].name == 'b'
assert prods[1].rhs[0].name == 'c'
# Nesting
prods = grammar.get_productions('s_g2')
assert len(prods) == 1
assert prods[0].rhs[0].name == 'a_1'
assert prods[0].rhs[1].name == 's_g3_0'
assert grammar.get_productions('s')[1].rhs[1].name == 's_g2_1_comma'
assert 's_g5' not in grammar
parser = GLRParser(grammar)
forest = parser.parse('b c, b c a a a b c c b, a b b')
result = parser.call_actions(forest[0])
assert result.a == [[['a', 'a', 'a'], ['b', 'c', 'c', 'b']],
[['a'], ['b', 'b']]]
def test_glr_recovery_default():
"""
Test default error recovery in GLR parsing. Default recovery should report
the error, drop current input at position and try to recover.
In case of multiple subsequent errouneous chars only one error should be
reported.
"""
parser = GLRParser(g, actions=actions, error_recovery=True)
results = parser.parse('1 + 2 + * 3 & 89 - 5')
assert len(parser.errors) == 2
e1, e2 = parser.errors
# First errors is '*' at position 8 and of length 2
assert e1.location.start_position == 8
assert e1.location.end_position == 10
# Second error is '& 89' at position 12 and length 5
assert e2.location.start_position == 12
assert e2.location.end_position == 17
# There are 5 trees for '1 + 2 + 3 - 5'
# All results are the same
assert len(results) == 5
result_set = set([parser.call_actions(tree) for tree in results])
assert len(result_set) == 1
assert 1 in set(result_set)
def test_lexical_ambiguity():
g = Grammar.from_string("""
expression: a a
| b
;
terminals
a: "x";
b: "xx";
""")
p = GLRParser(g)
assert all(
[p.call_actions(x) in ['xx', ['x', 'x']] for x in p.parse('xx')])
disambig_p = GLRParser(g, lexical_disambiguation=True)
assert p.call_actions(disambig_p.parse("xx")[0]) == 'xx'
def test_dynamic_disambiguation_glr():
"""
Test disambiguation determined at run-time based on the input.
This tests GLR parsing.
"""
p = GLRParser(g,
actions=actions,
dynamic_filter=custom_disambiguation_filter)
# * operation will be of higher priority as it appears later in the stream.
result1 = p.parse(instr1)
assert len(result1) == 1
assert p.call_actions(result1[0]) == 1 + (2 * 5) + 3
# + operation will be of higher priority here.
result2 = p.parse(instr2)
assert len(result2) == 1
assert p.call_actions(result2[0]) == 1 * (2 + 5) * 3
def test_glr_list_building_bug():
"""Test regression for a bug in building lists from default `collect` actions.
"""
grammar = r"""
S: B+;
B: "b"? A+;
A: "a";
"""
g = Grammar.from_string(grammar)
parser = GLRParser(g, prefer_shifts=True)
result = parser.parse('b a b a a a')
assert len(result) == 1
assert parser.call_actions(result[0]) == [['b', ['a']],
['b', ['a', 'a', 'a']]]
def test_glr_recovery_custom_new_position():
"""
Test that custom recovery that increment position works.
"""
def custom_recovery(head, error):
# This recovery will just skip over erroneous part of input '& 89'.
head.position += 4
return head.parser.default_error_recovery(head)
parser = GLRParser(g, actions=actions, error_recovery=custom_recovery)
results = parser.parse('1 + 5 & 89 - 2')
assert len(parser.errors) == 1
assert len(results) == 2
result_set = set([parser.call_actions(tree) for tree in results])
assert len(result_set) == 1
# Calculated result should be '1 + 5 - 2'
assert result_set.pop() == 4
def test_glr_recovery_custom_new_token():
"""
Test that custom recovery that introduces new token works.
"""
def custom_recovery(head, error):
# Here we will introduce missing operation token
head.token_ahead = Token(g.get_terminal('-'),
'-',
head.position,
length=0)
return True
parser = GLRParser(g, actions=actions, error_recovery=custom_recovery)
results = parser.parse('1 + 5 8 - 2')
assert len(parser.errors) == 1
assert len(results) == 5
result_set = set([parser.call_actions(tree) for tree in results])
assert len(result_set) == 2
assert -4 in result_set
assert 0 in result_set
def test_epsilon_grammar():
grammar = r"""
Model: Prods;
Prods: Prod | Prods Prod | EMPTY;
Prod: ID "=" ProdRefs;
ProdRefs: ID | ProdRefs ID;
terminals
ID: /\w+/;
"""
g = Grammar.from_string(grammar)
p = GLRParser(g)
txt = """
First = One Two three
Second = Foo Bar
Third = Baz
"""
forest = p.parse(txt)
expected = [
# First solution
[[['First', '=', [['One', 'Two'], 'three']],
['Second', '=', ['Foo', 'Bar']]], ['Third', '=', 'Baz']],
# Second solution
[[[[], ['First', '=', [['One', 'Two'], 'three']]],
['Second', '=', ['Foo', 'Bar']]], ['Third', '=', 'Baz']]
]
assert [p.call_actions(tree) for tree in forest] == expected
results = p.parse("")
assert len(results) == 1
def test_expressions():
actions = {
"s":
lambda _, c: c[0],
"E": [
lambda _, nodes: nodes[0] + nodes[2],
lambda _, nodes: nodes[0] * nodes[2], lambda _, nodes: nodes[1],
lambda _, nodes: int(nodes[0])
]
}
# This grammar is highly ambiguous if priorities and
# associativities are not defined to disambiguate.
grammar = r"""
E: E "+" E | E "*" E | "(" E ")" | Number;
terminals
Number: /\d+/;
"""
g = Grammar.from_string(grammar)
p = GLRParser(g, actions=actions)
# Even this simple expression has 2 different interpretations
# (4 + 2) * 3 and
# 4 + (2 * 3)
forest = p.parse("4 + 2 * 3")
assert len(forest) == 2
results = [p.call_actions(tree) for tree in forest]
assert 18 in results and 10 in results
# Adding one more operand rises number of interpretations to 5
results = p.parse("4 + 2 * 3 + 8")
assert len(results) == 5
# One more and there are 14 interpretations
results = p.parse("4 + 2 * 3 + 8 * 5")
assert len(results) == 14
# The number of interpretation will be the Catalan number of n
# where n is the number of operations.
# https://en.wikipedia.org/wiki/Catalan_number
# This number rises very fast. For 10 operations number of interpretations
# will be 16796!
# If we rise priority for multiplication operation we reduce ambiguity.
# Default production priority is 10. Here we will raise it to 15 for
# multiplication.
grammar = r"""
E: E "+" E | E "*" E {15}| "(" E ")" | Number;
terminals
Number: /\d+/;
"""
g = Grammar.from_string(grammar)
p = GLRParser(g, actions=actions)
# This expression now has 2 interpretation:
# (4 + (2*3)) + 8
# 4 + ((2*3) + 8)
# This is due to associativity of + operation which is not defined.
results = p.parse("4 + 2 * 3 + 8")
assert len(results) == 2
# If we define associativity for both + and * we have resolved all
# ambiguities in the grammar.
grammar = r"""
E: E "+" E {left}| E "*" E {left, 15}| "(" E ")" | Number;
terminals
Number: /\d+/;
"""
g = Grammar.from_string(grammar)
p = GLRParser(g, actions=actions)
results = p.parse("4 + 2 * 3 + 8 * 5 * 3")
assert len(results) == 1
assert p.call_actions(results[0]) == 4 + 2 * 3 + 8 * 5 * 3
