def test_recovery_custom_unsuccessful():
"""
Test unsuccessful error recovery.
"""
def custom_recovery(context, error):
return False
parser = Parser(g, actions=actions, error_recovery=custom_recovery)
with pytest.raises(ParseError) as e:
parser.parse('1 + 5 8 - 2')
error = e.value
assert error.location.start_position == 6
def test_error_recovery_parse_error():
"""In this test we have error that can't be recovered from by a simple
dropping of characters as we'll end up with invalid expression at the EOF.
The current solution is to throw ParseError at the beggining of the last
error that couldn't be recovered from.
"""
parser = Parser(g, actions=actions, error_recovery=True)
with pytest.raises(ParseError) as einfo:
parser.parse("1 + 2 + * 3 + & -")
assert einfo.value.location.start_position == 14
def todo_test_grammar_without_valid_inputs():
"""
TODO: There is no valid input for this grammar. This should be detected by
the parser.
"""
grammar = """
S: A | B;
A: '1' S '1';
B: '0' S '0';
"""
g = Grammar.from_string(grammar)
p = Parser(g)
p.parse('0101000110001010')
def test_whitespace_redefinition():
grammar = get_grammar()
# Make newline treated as non-ws characted
p = Parser(grammar, ws=' \t')
p.parse("""id+ id * (id +id ) """)
try:
p.parse("""id+ id * (id
+id )
""")
except ParseError as e:
assert e.location.start_position == 13
def test_most_specific(cf):
grammar = """
S: First | Second | Third;
First: /\d+\.\d+/;
Second: '14';
Third: /\d+/;
"""
g = Grammar.from_string(grammar)
parser = Parser(g, actions=actions, debug=True)
# String match in rule Second is more specific than Third regexp rule.
parser.parse('14')
assert called == [False, True, False]
def test_priority(cf):
grammar = """
S: First | Second | Third;
First: /\d+\.\d+/ {15};
Second: '14.75';
Third: /\d+\.75/;
"""
g = Grammar.from_string(grammar)
parser = Parser(g, actions=actions, debug=True)
# Priority is used first
parser.parse('14.75')
assert called == [True, False, False]
def test_error_recovery_complete():
"""
In this test we are using complete parse.
"""
parser = Parser(g, actions=actions, error_recovery=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# Both '*' and '& 89' should be dropped now as the parser expects to
# consume all the input. Thus the parser should calculate '1 + 2 + 3 - 5'
assert result == 1
assert len(parser.errors) == 2
e1, e2 = parser.errors
assert e1.location.start_position == 8
assert e1.location.end_position == 10
# Characters of the second error should be packed as a single error
# spanning the whole erroneous region. Whitespaces should be included too.
assert e2.location.start_position == 12
assert e2.location.end_position == 17
assert 'Error at 1:12:"+ 2 + * 3 **> & 89 - 5" => '\
'Expected: ) or * or + or - or / or STOP or ^' in str(e2)
def test_error_recovery_uncomplete():
"""
Test default recovery for partial parse.
parglare will try to parse as much as possible for the given grammar and
input. If the current input can be reduced to the start rule the parse
will succeed.
"""
parser = Parser(g,
actions=actions,
consume_input=False,
error_recovery=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# '*' after '+' will be dropped but when the parser reach '&'
# it has a complete expression and will terminate successfully and
# report only one error ('*' after '+').
# The parser should thus calculate '1 + 2 + 3'
assert result == 6
assert len(parser.errors) == 1
e = parser.errors[0]
assert e.location.start_position == 8
assert e.location.end_position == 10
assert 'Error at 1:8:"1 + 2 + **> * 3 & 89 -" => '\
'Expected: ( or number but found <*(*)>' in str(e)
def test_parglare_builtin_action_override_repetition():
"""
Test that user given action can override actions attached to
repetition operator generated rule actions.
"""
# B+ will product B_1 rule with `collect` common action
grammar = """
S: B+;
B: "b";
"""
called = [False]
def my_collect(_, __):
called[0] = True
return "pass"
my_actions = {
"collect": my_collect,
}
g = Grammar.from_string(grammar)
p = Parser(g, actions=my_actions)
assert p.parse("b b") == 'pass'
assert called[0]
def test_user_grammar_actions():
"""
Test that user supplied actions are used.
"""
grammar = """
S: A B C;
@nonterm_action
C: A B;
A: "a";
@term_action
B: "b";
"""
called = [False, False]
def nonterm_action(_, __):
called[0] = True
def term_action(_, __):
called[1] = True
my_actions = {
"nonterm_action": nonterm_action,
"term_action": term_action,
}
g = Grammar.from_string(grammar)
p = Parser(g, actions=my_actions)
assert p.parse("a b a b")
assert all(called)
def refinement_checker(scriptFile):
try:
g = Grammar.from_file("grammar")
parser = Parser(g, actions=actions)
except Exception as e:
print(e)
print("Parse generation: Failed.")
print("Terminating.")
sys.exit()
print("Parser generation: Done.")
try:
script = parser.parse(scriptFile)
print("Parse input: Done.")
except Exception as e:
print(e)
print("Parse input: Failed.")
print("Terminating.")
sys.exit()
try:
execute(script)
except Exception as e:
print(e)
print("Script execution: Failed.")
print("Terminating.")
sys.exit()
print("Script execution: Done.")
print("Terminating.")
def test_grammar_with_unicode():
this_folder = os.path.dirname(__file__)
grammar = Grammar.from_file(os.path.join(this_folder, "names.pg"))
parser = Parser(grammar, consume_input=False)
inp = 'МИША МЫЛ РАМУ'
result = parser.parse(inp)
assert result
def test_error_recovery_uncomplete():
"""
Test default recovery for partial parse.
parglare will try to parse as much as possible for the given grammar and
input. If the current input can be reduced to the start rule the parse
will succeed. In order to prevent partial parse first grammar rule should
be ended with EOF like in the case of 'Result' rule.
"""
# By setting start_production to 'E' parser will understand only +
# operation
parser = Parser(g,
actions=actions,
start_production='E',
error_recovery=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# '*' after '+' will be droped but when the parser reach '&'
# it has a complete expression and will terminate successfuly and
# report only one error ('*' after '+').
# The parser should thus calculate '1 + 2 + 3'
assert result == 6
assert len(parser.errors) == 1
e = parser.errors[0]
assert e.location.start_position == 8
assert e.location.end_position == 9
assert 'Error at 1:8:"1 + 2 + ** 3 & 89 -" => '\
'Expected: ( or number but found <*(*)>' in str(e)
def test_longest_match(cf):
grammar = """
S: First | Second | Third;
First: /\d+\.\d+/;
Second: '13';
Third: /\d+/;
"""
g = Grammar.from_string(grammar)
parser = Parser(g, actions=actions)
# If all matches are regexes of the same priority use longest match
# disambiguation.
parser.parse('14.17')
assert called == [True, False, False]
def test_error_recovery_complete():
"""
In this test we start from the 'Result' rule so parglare will require
input to end with 'EOF' for the parse to be successful.
"""
parser = Parser(g, actions=actions, error_recovery=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# Both '*' and '& 89' should be dropped now as the parser expects EOF at
# the end. Thus the parser should calculate '1 + 2 + 3 - 5'
assert result == 1
assert len(parser.errors) == 2
e1, e2 = parser.errors
assert e1.location.start_position == 8
assert e1.location.end_position == 9
# Characters of the second error should be packed as a single error
# spanning the whole erroneous region. Whitespaces should be included too.
assert e2.location.start_position == 12
assert e2.location.end_position == 16
assert 'Error at 1:12:"+ 2 + * 3 *& 89 - 5" => '\
'Expected: ) or * or + or - or / or EOF or ^' in str(e2)
def test_custom_error_recovery():
"""
Test that registered callable for error recovery is called with the
right parameters.
"""
called = [False]
def my_recovery(context, error):
expected_symbols = context.state.actions.keys()
called[0] = True
assert isinstance(context.parser, Parser)
assert context.input_str == '1 + 2 + * 3 - 5'
assert context.position == 8
open_par = g.get_terminal('(')
assert open_par in expected_symbols
number = g.get_terminal('number')
assert number in expected_symbols
return None, context.position + 1
parser = Parser(g, actions=actions, error_recovery=my_recovery, debug=True)
result = parser.parse("1 + 2 + * 3 - 5")
assert result == 1
# Assert that recovery handler is called.
assert called[0]
def test_multiple_assignment_with_repetitions():
"""
Test assignment of repetition.
"""
grammar = """
S: "1" first=some_match+[comma] second?=some_match* "3";
terminals
some_match: "2";
comma: ",";
"""
g = Grammar.from_string(grammar)
assert assignment_in_productions(g.productions, 'S', 'first')
assert assignment_in_productions(g.productions, 'S', 'second')
called = [False]
def act_s(_, nodes, first, second):
called[0] = True
assert first == ["2", "2"]
assert second is True
return nodes
actions = {"S": act_s}
p = Parser(g, actions=actions)
input_str = '1 2, 2 2 2 2 3'
result = p.parse(input_str)
assert result == ["1", ["2", "2"], ["2", "2", "2"], "3"]
assert called[0]
def test_obj_position():
"""
Test that object start/end position is set properly.
"""
grammar = r"""
S: "first" seconds=Second+;
Second: value=digits;
terminals
digits:/\d+/;
"""
g = Grammar.from_string(grammar)
parser = Parser(g)
result = parser.parse("""
first 45 56
66 3434342
""")
n = result.seconds[1]
assert n._pg_start_position == 14
assert n._pg_end_position == 16
n = result.seconds[3]
assert n._pg_start_position == 24
assert n._pg_end_position == 31
def test_custom_error_recovery():
"""
Test that registered callable for error recovery is called with the
right parameters.
"""
called = [False]
def my_recovery(parser, input, position, expected_symbols):
called[0] = True
assert isinstance(parser, Parser)
assert input == '1 + 2 + * 3 - 5'
assert position == 8
assert type(expected_symbols) is set
assert Terminal('(') in expected_symbols
assert Terminal('number') in expected_symbols
return None, None, position + 1
parser = Parser(g, actions=actions, error_recovery=my_recovery, debug=True)
result = parser.parse("1 + 2 + * 3 - 5")
assert result == 1
# Assert that recovery handler is called.
assert called[0]
def test_assignment_of_repetition():
"""
Test assignment of repetition.
"""
grammar = """
S: "1" first=some_match+ "3";
terminals
some_match: "2";
"""
g = Grammar.from_string(grammar)
assert assignment_in_productions(g.productions, 'S', 'first')
called = [False]
def act_s(_, nodes, first):
called[0] = True
assert first == ["2", "2"]
return nodes
actions = {"S": act_s}
p = Parser(g, actions=actions)
input_str = '1 2 2 3'
result = p.parse(input_str)
assert result == ["1", ["2", "2"], "3"]
assert called[0]
def test_error_recovery_uncomplete():
"""
Test default recovery for partial parse.
parglare will try to parse as much as possible for the given grammar and
input. If the current input can be reduced to the start rule the parse
will succeed. In order to prevent partial parse first grammar rule should
be ended with EOF like in the case of 'Result' rule.
"""
parser = Parser(g, start_production=2, actions=actions,
error_recovery=True, debug=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# '*' after '+' will be droped but when the parser reach '&'
# it has a complete expression and will terminate successfuly and
# report only one error ('*' after '+').
# The parser should thus calculate '1 + 2 + 3'
assert result == 6
assert len(parser.errors) == 1
e = parser.errors[0]
assert e.position == 8
assert e.length == 1
assert 'Unexpected input at position (1, 8). Expected' in str(e)
def test_assignment_bool():
"""
Test bool assignment.
"""
grammar = """
S: "1" first?=some_match "3";
some_match: "2";
"""
g = Grammar.from_string(grammar)
assert assignment_in_productions(g.productions, 'S', 'first')
called = [False]
def act_s(_, nodes, first):
called[0] = True
assert first is True
return nodes
actions = {"S": act_s}
p = Parser(g, actions=actions)
input_str = '1 2 3'
result = p.parse(input_str)
assert result == ["1", "2", "3"]
assert called[0]
def test_prefer_shifts_no_sr_conflicts():
"""
Test that grammar with S/R conflict will be resolved to SHIFT actions
if prefer_shift option is used.
"""
# This grammar has S/R conflict as B+ may consume multiple single "a" A
# because "b" is optional. Thus, parser can't decide if it should shift "a"
# or reduce by 'B: "b"? A+' and later by 'S: B+'; Most of the time we want
# gready behavior so in case of doubt parser will choose shift if
# prefer_shift is set to `True`. This means that the parser will first
# consume all "a" using A+ and that reduce B at the end.
grammar = r"""
S: B+;
B: "b"? A+;
terminals
A: "a";
"""
g = Grammar.from_string(grammar)
# There is a shift reduce conflict so we can't use LR parser.
table = create_table(g)
assert len(table.sr_conflicts) == 1
# But we can eliminate conflict by prefer_shifts strategy.
table = create_table(g, prefer_shifts=True)
assert len(table.sr_conflicts) == 0
# With prefer_shifts we get a greedy behavior
input_str = 'b a a a b a a'
output = [['b', ['a', 'a', 'a']], ['b', ['a', 'a']]]
parser = Parser(g, prefer_shifts=True)
result = parser.parse(input_str)
assert result == output
# GLR parser can parse without prefer_shifts strategy. This grammar is
# ambiguous but, in this case, implicit elimination of empty trees (due to
# optional "b") will yield a single result.
parser = GLRParser(g)
result = parser.parse(input_str)
assert len(result) == 1
assert result[0] == output
parser = GLRParser(g, prefer_shifts=True)
result = parser.parse(input_str)
assert len(result) == 1
assert result[0] == output
def test_most_specific_longest_match(cf):
grammar = """
S: First | Second | Third;
First: '147';
Second: '14';
Third: /\d+/;
"""
g = Grammar.from_string(grammar)
parser = Parser(g, actions=actions, debug=True)
# All three rules could match. First is tried first because it is
# more specific (str match) and longest. It succeeds so other two
# are not tried at all.
parser.parse('147')
assert called == [True, False, False]
def test_priority(cf):
grammar = r"""
M: First | Second | Third "5";
terminals
First: /\d+\.75/;
Second: '14.75';
Third: /\d+\.\d/ {15};
"""
g = Grammar.from_string(grammar)
parser = Parser(g, actions=actions, debug=False)
# Priority is used first
parser.parse('14.75')
assert called == [False, False, True]
def test_parse_list_of_integers():
grammar = """
Numbers: all_less_than_five;
all_less_than_five: all_less_than_five int_less_than_five
| int_less_than_five;
terminals
int_less_than_five:;
"""
def int_less_than_five(input, pos):
if input[pos] < 5:
return [input[pos]]
recognizers = {
'int_less_than_five': int_less_than_five
}
g = Grammar.from_string(grammar, recognizers=recognizers, debug=True)
actions = {
'Numbers': pass_single,
'all_less_than_five': collect,
'int_less_than_five': pass_single
}
# Test that `ws` must be set to `None` for non-textual content
parser = Parser(g, actions=actions)
ints = [3, 4, 1, 4]
with pytest.raises(
ParserInitError,
match=r'For parsing non-textual content please '
'set `ws` to `None`'):
parser.parse(ints)
parser = Parser(g, actions=actions, ws=None)
ints = [3, 4, 1, 4]
p = parser.parse(ints)
assert p == ints
# Test that error is correctly reported.
with pytest.raises(ParseError) as e:
parser.parse([4, 2, 1, 6, 3])
assert '1:3:"[4, 2, 1]*[6, 3]"' in str(e)
assert 'int_less_than_five' in str(e)
def test_layout_terminal():
"""
Test that layout definition may be just a terminal rule.
"""
grammar = r"""
S: "a" "b";
LAYOUT: "c";
"""
g = Grammar.from_string(grammar)
parser = Parser(g)
with pytest.raises(ParseError):
parser.parse("a b")
parser.parse("cacbc")
grammar = r"""
S: "a" "b";
LAYOUT: DIGITS;
terminals
DIGITS: /\d*/;
"""
g = Grammar.from_string(grammar)
parser = Parser(g)
with pytest.raises(ParseError):
parser.parse("a b")
result = parser.parse("4444a23b545")
assert result == ['a', 'b']
def test_associativity_variant_2():
"""
See https://github.com/igordejanovic/parglare/issues/22
"""
g = Grammar.from_string(grammar_2)
parser = Parser(g, actions=action.all)
r = parser.parse(expression)
assert r == result
def test_actions():
grammar = get_grammar()
p = Parser(grammar, actions=get_actions())
result = p.parse("""34.7+78*34 +89+
12.223*4""")
assert result == 34.7 + 78 * 34 + 89 + 12.223 * 4
def test_table_from_cache_different_than_calculated():
this_folder = os.path.dirname(__file__)
grammar_file = os.path.join(this_folder, 'grammar.pg')
table_file = os.path.join(this_folder, 'grammar.pgt')
try:
os.remove(table_file)
except Exception:
pass
g = Grammar.from_file(grammar_file)
p = Parser(g)
without_cache = p.parse('dynamic("OS")')
p = Parser(g)
with_cache = p.parse('dynamic("OS")')
assert without_cache == with_cache