def test_failed_disambiguation(cf):
grammar = r"""
S: First | Second | Third;
terminals
First: /\d+\.\d+/ {15};
Second: '14.7';
Third: /\d+\.75/ {15};
"""
g = Grammar.from_string(grammar)
parser = Parser(g, actions=actions, debug=True)
# All rules will match but First and Third have higher priority.
# Both are regexes so longest match will be used.
# Both have the same length.
with pytest.raises(DisambiguationError) as e:
parser.parse('14.75')
assert 'disambiguate' in str(e.value)
assert 'First' in str(e.value)
assert 'Second' not in str(e.value)
assert 'Third' in str(e.value)
def test_repeatable_zero_or_more_with_separator():
"""
Tests zero or more repeatable operator with separator.
"""
grammar = """
S: "2" b*[comma] "3";
terminals
b: "1";
comma: ",";
"""
g = Grammar.from_string(grammar)
assert g.get_nonterminal('b_0_comma')
p = Parser(g)
input_str = '2 1, 1 , 1 3'
result = p.parse(input_str)
assert result == ["2", ["1", "1", "1"], "3"]
input_str = '2 3'
result = p.parse(input_str)
assert result == ["2", [], "3"]
def refinementChecker(scriptFile):
try:
g = Grammar.from_file("grammar")
parser = Parser(g, actions=actions)
except Exception as e:
print e
print "Parse generation: Failed."
print "Terminating."
sys.exit()
print "Parser generation: Done."
try:
script = parser.parse_file(scriptFile)
print "Parse input: Done."
except Exception as e:
print e
print "Parse input: Failed."
print "Terminating."
sys.exit()
try:
execute(script)
except Exception as e:
print e
print "Script execution: Failed."
print "Terminating."
sys.exit()
print "Script execution: Done."
print "Terminating."
def test_assignment_bool():
"""
Test bool assignment.
"""
grammar = """
S: "1" first?=some_match "3";
some_match: "2";
"""
g = Grammar.from_string(grammar)
assert assignment_in_productions(g.productions, 'S', 'first')
called = [False]
def act_s(_, nodes, first):
called[0] = True
assert first is True
return nodes
actions = {"S": act_s}
p = Parser(g, actions=actions)
input_str = '1 2 3'
result = p.parse(input_str)
assert result == ["1", "2", "3"]
assert called[0]
def test_invalid_number_of_actions():
"""
Test that parser error is raised if rule is given list of actions
where there is less/more actions than rule productions.
"""
grammar = '''
S: A+ | B+;
A: 'a';
B: 'b';
'''
g = Grammar.from_string(grammar)
def some_action(_, nodes):
return nodes[0]
actions = {
'S': [some_action, some_action]
}
Parser(g, actions=actions)
actions = {
'S': [some_action]
}
with pytest.raises(ParserInitError,
match=r'Length of list of actions must match.*'):
Parser(g, actions=actions)
actions = {
'S': [some_action, some_action, some_action]
}
with pytest.raises(ParserInitError,
match=r'Length of list of actions must match.*'):
Parser(g, actions=actions)
def test_object_children_order():
"""Children may depend on the concrete production that matched. Test that order
given in `_pg_children` is the same as the order provided in the grammar
(this may be important for tree traversal order).
"""
grammar = r'''
S: a=A b=B
| b=B a=A
| b=B;
A: val="a";
B: val="b";
'''
g = Grammar.from_string(grammar)
p = Parser(g)
ast = p.parse('a b')
res = ['a', 'b']
assert len(res) == len(ast._pg_children)
assert all((x.val == y for x, y in zip(ast._pg_children, res)))
ast = p.parse('b a')
res = ['b', 'a']
assert len(res) == len(ast._pg_children)
assert all((x.val == y for x, y in zip(ast._pg_children, res)))
ast = p.parse('b')
res = ['b']
assert len(res) == len(ast._pg_children)
assert all((x.val == y for x, y in zip(ast._pg_children, res)))
def test_error_recovery_uncomplete():
"""
Test default recovery for partial parse.
parglare will try to parse as much as possible for the given grammar and
input. If the current input can be reduced to the start rule the parse
will succeed.
"""
parser = Parser(g,
actions=actions,
consume_input=False,
error_recovery=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# '*' after '+' will be dropped but when the parser reach '&'
# it has a complete expression and will terminate successfully and
# report only one error ('*' after '+').
# The parser should thus calculate '1 + 2 + 3'
assert result == 6
assert len(parser.errors) == 1
e = parser.errors[0]
assert e.location.start_position == 8
assert e.location.end_position == 10
assert 'Error at 1:8:"1 + 2 + **> * 3 & 89 -" => '\
'Expected: ( or number but found <*(*)>' in str(e)
def test_custom_error_recovery():
"""
Test that registered callable for error recovery is called with the
right parameters.
"""
called = [False]
def my_recovery(parser, input, position, expected_symbols):
called[0] = True
assert isinstance(parser, Parser)
assert input == '1 + 2 + * 3 - 5'
assert position == 8
assert type(expected_symbols) is set
assert Terminal('(') in expected_symbols
assert Terminal('number') in expected_symbols
return None, None, position + 1
parser = Parser(g, actions=actions, error_recovery=my_recovery, debug=True)
result = parser.parse("1 + 2 + * 3 - 5")
assert result == 1
# Assert that recovery handler is called.
assert called[0]
def test_user_grammar_actions():
"""
Test that user supplied actions are used.
"""
grammar = """
S: A B C;
@nonterm_action
C: A B;
A: "a";
@term_action
B: "b";
"""
called = [False, False]
def nonterm_action(_, __):
called[0] = True
def term_action(_, __):
called[1] = True
my_actions = {
"nonterm_action": nonterm_action,
"term_action": term_action,
}
g = Grammar.from_string(grammar)
p = Parser(g, actions=my_actions)
assert p.parse("a b a b")
assert all(called)
def test_parglare_builtin_action_override_repetition():
"""
Test that user given action can override actions attached to
repetition operator generated rule actions.
"""
# B+ will product B_1 rule with `collect` common action
grammar = """
S: B+;
B: "b";
"""
called = [False]
def my_collect(_, __):
called[0] = True
return "pass"
my_actions = {
"collect": my_collect,
}
g = Grammar.from_string(grammar)
p = Parser(g, actions=my_actions)
assert p.parse("b b") == 'pass'
assert called[0]
def test_cyclic_grammar_3():
"""
Grammar with indirect cycle.
r:EMPTY->A ; r:A->S; r:EMPTY->A; r:SA->S; r:EMPTY->A; r:SA->S;...
"""
grammar = """
S: S A | A;
A: "a" | EMPTY;
"""
g = Grammar.from_string(grammar)
# In this grammar we have 3 S/R conflicts where each reduction is EMPTY.
# If we turn off prefer shifts over empty strategy in LR parser
# we will get S/R conflict
with pytest.raises(SRConflicts):
Parser(g, prefer_shifts_over_empty=False)
# By default there is no S/R conflict with prefer shifts over
# empty strategy
Parser(g)
p = GLRParser(g)
results = p.parse('aa')
with pytest.raises(LoopError):
len(results)
def test_grammar_with_unicode():
this_folder = os.path.dirname(__file__)
grammar = Grammar.from_file(os.path.join(this_folder, "names.pg"))
parser = Parser(grammar, consume_input=False)
inp = 'МИША МЫЛ РАМУ'
result = parser.parse(inp)
assert result
def __init__(self, input_file_path, rule_file_path):
super().__init__()
root_path = get_root_path()
self.user_lib = importlib.import_module('model_converter.'
'user_libs.functions',
'functions')
self.input_grammar = \
Grammar.from_file(os.path.join(root_path,
'converter',
'grammars',
'XMLGrammar.pg'))
self.input_parser = Parser(self.input_grammar)
self.input_file_path = input_file_path
self.rule_file_path = rule_file_path
self.conversion_dict = {"Subsystem": []}
self.temp_subsystem_dict = {}
self.path_dict = {}
self.file_input = None
def test_error_recovery_uncomplete():
"""
Test default recovery for partial parse.
parglare will try to parse as much as possible for the given grammar and
input. If the current input can be reduced to the start rule the parse
will succeed. In order to prevent partial parse first grammar rule should
be ended with EOF like in the case of 'Result' rule.
"""
# By setting start_production to 'E' parser will understand only +
# operation
parser = Parser(g,
actions=actions,
start_production='E',
error_recovery=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# '*' after '+' will be droped but when the parser reach '&'
# it has a complete expression and will terminate successfuly and
# report only one error ('*' after '+').
# The parser should thus calculate '1 + 2 + 3'
assert result == 6
assert len(parser.errors) == 1
e = parser.errors[0]
assert e.location.start_position == 8
assert e.location.end_position == 9
assert 'Error at 1:8:"1 + 2 + ** 3 & 89 -" => '\
'Expected: ( or number but found <*(*)>' in str(e)
def test_error_recovery_complete():
"""
In this test we are using complete parse.
"""
parser = Parser(g, actions=actions, error_recovery=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# Both '*' and '& 89' should be dropped now as the parser expects to
# consume all the input. Thus the parser should calculate '1 + 2 + 3 - 5'
assert result == 1
assert len(parser.errors) == 2
e1, e2 = parser.errors
assert e1.location.start_position == 8
assert e1.location.end_position == 10
# Characters of the second error should be packed as a single error
# spanning the whole erroneous region. Whitespaces should be included too.
assert e2.location.start_position == 12
assert e2.location.end_position == 17
assert 'Error at 1:12:"+ 2 + * 3 **> & 89 - 5" => '\
'Expected: ) or * or + or - or / or STOP or ^' in str(e2)
def test_repeatable_one_or_more_with_separator():
"""
Tests one or more repeatable operator with separator.
"""
grammar = """
S: "2" b+[comma] "3";
terminals
b: "1";
comma: ",";
"""
g = Grammar.from_string(grammar)
assert g.get_nonterminal('b_1_comma')
p = Parser(g)
input_str = '2 1, 1 , 1 3'
result = p.parse(input_str)
assert result == ["2", ["1", "1", "1"], "3"]
input_str = '2 3'
with pytest.raises(ParseError) as e:
p.parse(input_str)
assert 'Expected: b' in str(e)
def test_partial_parse():
"""
Not giving EOF at the end of the sequence enables parsing of the beginning
of the input string.
"""
grammar = """
S: 'a' B;
B: 'b';
"""
g = Grammar.from_string(grammar)
parser = Parser(g)
# Parser should succesfuly parse 'ab' at the beggining.
parser.parse('abc')
# But if EOF is given it will match only at the end of the string,
# thus, the whole string must be parsed in order for parsing to
# succeed.
grammar = """
S: 'a' B EOF;
B: 'b';
"""
g = Grammar.from_string(grammar)
parser = Parser(g)
parser.parse('a b')
with pytest.raises(ParseError):
parser.parse('a b c')
def test_error_recovery_uncomplete():
"""
Test default recovery for partial parse.
parglare will try to parse as much as possible for the given grammar and
input. If the current input can be reduced to the start rule the parse
will succeed. In order to prevent partial parse first grammar rule should
be ended with EOF like in the case of 'Result' rule.
"""
parser = Parser(g, start_production=2, actions=actions,
error_recovery=True, debug=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# '*' after '+' will be droped but when the parser reach '&'
# it has a complete expression and will terminate successfuly and
# report only one error ('*' after '+').
# The parser should thus calculate '1 + 2 + 3'
assert result == 6
assert len(parser.errors) == 1
e = parser.errors[0]
assert e.position == 8
assert e.length == 1
assert 'Unexpected input at position (1, 8). Expected' in str(e)
def test_repeatable_zero_or_more():
"""
Tests zero or more repeatable operator.
"""
grammar = """
S: "2" b* "3";
terminals
b: "1";
"""
g = Grammar.from_string(grammar)
assert g.get_nonterminal('b_0')
assert g.get_nonterminal('b_1')
p = Parser(g)
input_str = '2 1 1 1 3'
result = p.parse(input_str)
assert result == ["2", ["1", "1", "1"], "3"]
input_str = '2 3'
result = p.parse(input_str)
assert result == ["2", [], "3"]
def test_highly_ambiguous_grammar():
"""
This grammar has both Shift/Reduce and Reduce/Reduce conflicts and
thus can't be parsed by a deterministic LR parsing.
Shift/Reduce can be resolved by prefer_shifts strategy.
"""
grammar = """
S: "b" | S S | S S S;
"""
g = Grammar.from_string(grammar)
with pytest.raises(SRConflicts):
Parser(g, prefer_shifts=False)
# S/R are resolved by selecting prefer_shifts strategy.
# But R/R conflicts remain.
with pytest.raises(RRConflicts):
Parser(g, prefer_shifts=True)
# GLR parser handles this fine.
p = GLRParser(g, build_tree=True)
# For three tokens we have 3 valid derivations/trees.
results = p.parse("bbb")
assert len(results) == 3
# For 4 tokens we have 10 valid derivations.
results = p.parse("bbbb")
assert len(results) == 10
def test_parse_list_of_integers():
grammar = """
Numbers: all_less_than_five EOF;
all_less_than_five: all_less_than_five int_less_than_five
| int_less_than_five;
int_less_than_five:;
"""
def int_less_than_five(input, pos):
if input[pos] < 5:
return [input[pos]]
recognizers = {'int_less_than_five': int_less_than_five}
g = Grammar.from_string(grammar, recognizers=recognizers, debug=True)
actions = {
'Numbers': pass_single,
'all_less_than_five': collect,
'int_less_than_five': pass_single
}
parser = Parser(g, actions=actions)
ints = [3, 4, 1, 4]
p = parser.parse(ints)
assert p == ints
# Test that error is correctly reported.
with pytest.raises(ParseError) as e:
parser.parse([4, 2, 1, 6, 3])
assert 'Error at position 1,3 => "[4, 2, 1]*[6, 3]".' in str(e)
assert 'int_less_than_five' in str(e)
def test_custom_error_recovery():
"""
Test that registered callable for error recovery is called with the
right parameters.
"""
called = [False]
def my_recovery(context, error):
expected_symbols = context.state.actions.keys()
called[0] = True
assert isinstance(context.parser, Parser)
assert context.input_str == '1 + 2 + * 3 - 5'
assert context.position == 8
open_par = g.get_terminal('(')
assert open_par in expected_symbols
number = g.get_terminal('number')
assert number in expected_symbols
return None, context.position + 1
parser = Parser(g, actions=actions, error_recovery=my_recovery, debug=True)
result = parser.parse("1 + 2 + * 3 - 5")
assert result == 1
# Assert that recovery handler is called.
assert called[0]
def test_error_recovery_complete():
"""
In this test we start from the 'Result' rule so parglare will require
input to end with 'EOF' for the parse to be successful.
"""
parser = Parser(g, actions=actions, error_recovery=True)
result = parser.parse("1 + 2 + * 3 & 89 - 5")
# Both '*' and '& 89' should be dropped now as the parser expects EOF at
# the end. Thus the parser should calculate '1 + 2 + 3 - 5'
assert result == 1
assert len(parser.errors) == 2
e1, e2 = parser.errors
assert e1.location.start_position == 8
assert e1.location.end_position == 9
# Characters of the second error should be packed as a single error
# spanning the whole erroneous region. Whitespaces should be included too.
assert e2.location.start_position == 12
assert e2.location.end_position == 16
assert 'Error at 1:12:"+ 2 + * 3 *& 89 - 5" => '\
'Expected: ) or * or + or - or / or EOF or ^' in str(e2)
def test_partial_parse():
"""
Test `consume_input` parser parameter.
"""
grammar = """
S: 'a' B;
B: 'b';
"""
g = Grammar.from_string(grammar)
parser = Parser(g, consume_input=False)
# Parser should succesfuly parse 'ab' at the beginning.
parser.parse('abc')
# But if `consume_input` is not set to `False` it should be `True` by
# default and the parser will not accept partial parses.
grammar = """
S: 'a' B;
B: 'b';
"""
g = Grammar.from_string(grammar)
parser = Parser(g)
parser.parse('a b')
with pytest.raises(ParseError):
parser.parse('a b c')
def test_obj_position():
"""
Test that object start/end position is set properly.
"""
grammar = r"""
S: "first" seconds=Second+;
Second: value=digits;
terminals
digits:/\d+/;
"""
g = Grammar.from_string(grammar)
parser = Parser(g)
result = parser.parse("""
first 45 56
66 3434342
""")
n = result.seconds[1]
assert n._pg_start_position == 14
assert n._pg_end_position == 16
n = result.seconds[3]
assert n._pg_start_position == 24
assert n._pg_end_position == 31
def test_case_insensitive_parsing():
"""
By default parglare is case sensitive. This test parsing without case
sensitivity.
"""
grammar = """
S: "one" "Two" Astart;
terminals
Astart: /Aa\w+/;
"""
g = Grammar.from_string(grammar)
# By default parsing is case sensitive for both string and regex matches.
parser = Parser(g)
with pytest.raises(ParseError):
parser.parse('One Two Aaa')
with pytest.raises(ParseError):
parser.parse('one Two AAa')
g = Grammar.from_string(grammar, ignore_case=True)
parser = Parser(g)
parser.parse('One Two Aaa')
parser.parse('one Two AAa')
def test_multiple_assignment_with_repetitions():
"""
Test assignment of repetition.
"""
grammar = """
S: "1" first=some_match+[comma] second?=some_match* "3";
terminals
some_match: "2";
comma: ",";
"""
g = Grammar.from_string(grammar)
assert assignment_in_productions(g.productions, 'S', 'first')
assert assignment_in_productions(g.productions, 'S', 'second')
called = [False]
def act_s(_, nodes, first, second):
called[0] = True
assert first == ["2", "2"]
assert second is True
return nodes
actions = {"S": act_s}
p = Parser(g, actions=actions)
input_str = '1 2, 2 2 2 2 3'
result = p.parse(input_str)
assert result == ["1", ["2", "2"], ["2", "2", "2"], "3"]
assert called[0]
def test_assignment_of_repetition():
"""
Test assignment of repetition.
"""
grammar = """
S: "1" first=some_match+ "3";
terminals
some_match: "2";
"""
g = Grammar.from_string(grammar)
assert assignment_in_productions(g.productions, 'S', 'first')
called = [False]
def act_s(_, nodes, first):
called[0] = True
assert first == ["2", "2"]
return nodes
actions = {"S": act_s}
p = Parser(g, actions=actions)
input_str = '1 2 2 3'
result = p.parse(input_str)
assert result == ["1", ["2", "2"], "3"]
assert called[0]
def test_action_override():
"""
Explicitely provided action in `actions` param overrides default or
grammar provided.
"""
grammar = """
S: Foo Bar;
@pass_nochange
Bar: "1" a;
terminals
@pass_nochange
Foo: 'foo';
a: "a";
"""
g = Grammar.from_string(grammar)
p = Parser(g)
input_str = "foo 1 a"
result = p.parse(input_str)
assert result == ["foo", ["1", "a"]]
actions = {"Foo": lambda _, __: "eggs", "Bar": lambda _, __: "bar reduce"}
p = Parser(g, actions=actions)
result = p.parse(input_str)
assert result == ["eggs", "bar reduce"]
# Test with actions call postponing
p = Parser(g, build_tree=True, actions=actions)
tree = p.parse(input_str)
result = p.call_actions(tree)
assert result == ["eggs", "bar reduce"]
def test_default_whitespaces():
grammar = get_grammar()
p = Parser(grammar)
p.parse("""id+ id * (id
+id )
""")
