def resolve_left_recursion_simple(literals, bad_literals):
from grammar.models import Literal
for bt in bad_literals:
index = literals.index(bt)
rules_1 = [
rule[1:] for rule in bt.rules if len(rule) and rule[0] == bt
]
rules_2 = [rule for rule in bt.rules if not len(rule) or rule[0] != bt]
new_literal = Literal(bt.text + '-prime')
bt.rules = [rule2 + [new_literal] for rule2 in rules_2]
new_literal.rules = [rule1 + [new_literal] for rule1 in rules_1] + [[]]
literals.insert(index + 1, new_literal)
return literals
def test_success_order(self):
for filename, expected_parse_tree, grammar_filename in TestParser.test_cases:
with open(
os.path.join(
os.path.join(BASE_DIR, 'resources/test/grammar'),
grammar_filename)) as grammar_file:
with open(
os.path.join(
os.path.join(BASE_DIR, 'resources/test/code'),
filename)) as test_file:
non_terminals = Literal.parse(grammar_file)
all_literals = get_all_literals_from_non_terminals(
non_terminals)
start, states = create_transition_diagram(non_terminals)
parser = Parser(states, start,
Scanner(test_file.read(), all_literals))
errors = parser.parse()
if errors:
print(errors)
parse_tree = parser.tree
self.assertEqual(parse_tree[0], expected_parse_tree[0],
"Error in " + filename)
self.assertListEqual(parse_tree[1], expected_parse_tree[1],
"Error in " + filename)
def setUp(self):
with open(
os.path.join(
BASE_DIR,
'resources/src/predictable_grammar.txt')) as grammar_file:
non_terminals = Literal.parse(grammar_file)
self.all_literals = get_all_literals_from_non_terminals(
non_terminals)
def test_predictability(self):
with open(
os.path.join(BASE_DIR, 'resources', 'src',
'predictable_grammar.txt')) as grammar_file:
grammar = Literal.parse(grammar_file)
first = compute_non_terminals_firsts(grammar)
self.assertTrue(
check_predictability(
grammar, first,
compute_non_terminals_follows(grammar, first)))
def compile_nc_code(input_string):
with open(os.path.join(
BASE_DIR,
'resources/src/predictable_grammar.txt')) as grammar_file:
non_terminals = Literal.parse(grammar_file)
all_literals = get_all_literals_from_non_terminals(non_terminals)
start, states = create_transition_diagram(non_terminals)
parser = Parser(states, start, Scanner(input_string, all_literals))
errors = parser.parse()
if errors:
for error in errors:
print(sys.stderr, error)
return ''
return str(parser.program)
def factorize(grammar):
# only works for factorizations shared among all rules of the non terminal
counter = 0
new_grammar = copy(grammar)
for non_terminal in grammar:
counter += 1
if len(non_terminal.rules) <= 1:
continue
prefix = os.path.commonprefix(non_terminal.rules)
if len(prefix) > 0:
new_non_terminal = Literal(
'rest-of-' + non_terminal.text,
[rule[len(prefix):] for rule in non_terminal.rules])
non_terminal.rules = [prefix + [new_non_terminal]]
new_grammar.insert(counter, new_non_terminal)
counter += 1
return new_grammar
def test(self):
with open(
os.path.join(
BASE_DIR,
'resources/src/predictable_grammar.txt')) as grammar_file:
non_terminals = Literal.parse(grammar_file)
all_literals = get_all_literals_from_non_terminals(non_terminals)
start, states = create_transition_diagram(non_terminals)
os.chdir(os.path.join(BASE_DIR, 'resources/test/executable'))
for filename in Test.test_cases:
with open(os.path.join('..', 'code',
filename + '.nc')) as test_file:
parser = Parser(states, start,
Scanner(test_file.read(), all_literals))
errors = parser.parse()
if errors:
print(errors)
self.fail()
else:
with open(os.path.join(filename + '.nco'),
mode='w') as output_file:
output_file.write(str(parser.program))
os.system('cp ' + filename + '.nco output.txt')
if platform.system() == 'Darwin':
os.system('../../../bin/tester_mac.out > tmptmp')
elif platform.system() == 'Linux':
os.system('../../../bin/tester_linux.out > tmptmp')
else:
assert 0, "I don't give a F*** to Windows"
if 0 != os.system('diff tmptmp ../output/' + filename +
'.txt'):
print(
"Integration test {} failed.".format(filename))
pprint.pprint(parser.tree)
self.fail()
self.assertEqual(0, os.system('rm output.txt tmptmp'))
def test_first_follow(self):
opened_file = open(
os.path.join(BASE_DIR, 'resources', 'test', 'grammar',
'complex.txt'))
non_terminals = Literal.parse(opened_file)
literals_map = {}
for non_terminal in non_terminals:
literals_map[non_terminal.text] = non_terminal
for rule in non_terminal.rules:
for literal in rule:
literals_map[literal.text] = literal
first = compute_non_terminals_firsts(non_terminals)
follow = compute_non_terminals_follows(non_terminals, first)
self.assertDictEqual(
dict(first), {
literals_map['s']: {literals_map['f']},
literals_map['s-prime']: {literals_map['f']},
literals_map['a']: {literals_map['f']},
literals_map['b']: {()},
literals_map['c']: {literals_map['f']},
literals_map['d']: {()},
})
self.assertDictEqual(
follow, {
literals_map['s']: set(),
literals_map['s-prime']: {literals_map['EOF']},
literals_map['a']: {literals_map['f']},
literals_map['b']: {literals_map['f']},
literals_map['c']: {literals_map['EOF'], literals_map['f']},
literals_map['d']: {literals_map['EOF']},
})
opened_file.close()
def test_success_order(self):
for filename, expected_parse_errors, grammar_filename in TestPanicMode.test_cases:
with open(
os.path.join(
os.path.join(BASE_DIR, 'resources/test/grammar'),
grammar_filename)) as grammar_file:
with open(
os.path.join(
os.path.join(BASE_DIR, 'resources/test/code'),
filename)) as test_file:
non_terminals = Literal.parse(grammar_file)
all_literals = get_all_literals_from_non_terminals(
non_terminals)
start, states = create_transition_diagram(non_terminals)
parser = Parser(states, start,
Scanner(test_file.read(), all_literals))
errors = parser.parse()
print(errors)
self.assertEqual(len(expected_parse_errors), len(errors))
for error in errors:
self.assertIn((error.lookahead_literal.text,
error.non_terminal.text),
expected_parse_errors)
def generate():
from grammar.utils import check_left_recursion, resolve_left_recursion_simple, print_to_file, factorize, \
compute_non_terminals_firsts, requires_factorization, compute_non_terminals_follows
with open(os.path.join(BASE_DIR, 'resources/src/raw_grammar.txt')) as file:
new_grammar = current_grammar = Literal.parse(file)
while True:
bad_literals = check_left_recursion(current_grammar)
if not bad_literals:
break
new_grammar = resolve_left_recursion_simple(current_grammar,
bad_literals)
current_grammar = new_grammar
print_to_file(
new_grammar,
os.path.join(BASE_DIR, 'resources/src/recursion_free_grammar.txt'))
print("Left recursion resolved.")
while requires_factorization(current_grammar):
current_grammar = factorize(current_grammar)
print("Factorized one time.")
print_to_file(
current_grammar,
os.path.join(BASE_DIR, 'resources/src/partially_factored_grammar.txt'))
with open(os.path.join(BASE_DIR,
'resources/src/predictable_grammar.txt')) as file:
current_grammar = Literal.parse(file)
first = OrderedDict(compute_non_terminals_firsts(current_grammar))
follow = OrderedDict(compute_non_terminals_follows(current_grammar, first))
print("Computed first and follow sets.")
check_predictability(current_grammar, first, follow)
start_state, state_machines = create_transition_diagram(current_grammar)
with open(path.join(BASE_DIR, 'doc/README.md'), 'w') as doc_file:
with open(path.join(
BASE_DIR,
'resources/src/raw_grammar.txt')) as raw_grammar_file:
with open(
path.join(BASE_DIR,
'resources/src/recursion_free_grammar.txt')
) as recursion_free_grammar_file:
with open(
path.join(
BASE_DIR,
'resources/src/partially_factored_grammar.txt')
) as partially_factored_grammar_file:
with open(
path.join(BASE_DIR,
'resources/src/predictable_grammar.txt')
) as predictable_grammar_file:
doc_file.writelines([
'# ANCC Automatically Generated Documentation\n',
'## Raw Grammar\n',
'```\n',
])
for line in raw_grammar_file:
doc_file.write(line)
doc_file.writelines([
'```\n',
'## Recursion Free Grammar\n',
'```\n',
])
for line in recursion_free_grammar_file:
doc_file.write(line)
doc_file.writelines([
'```\n',
'## Partially Factored Grammar\n',
'```\n',
])
for line in partially_factored_grammar_file:
doc_file.write(line)
doc_file.writelines([
'```\n',
'## Predictable Grammar\n',
'```\n',
])
for line in predictable_grammar_file:
doc_file.write(escape(line))
doc_file.writelines([
'\n',
'```\n',
'## State Diagram\n',
'```\n',
])
print_diagram(state_machines, doc_file)
doc_file.writelines([
'```\n', '## First and Follow\n'
'|Non-terminal|First|Follow|\n'
'|:----------:|:---:|:----:|\n'
])
for non_terminal in sorted(first.keys()):
doc_file.writelines([
'|{}|{}|{}|\n'.format(
non_terminal.text, ' '.join(
sorted([
'ε'
if literal == () else literal.text
for literal in first[non_terminal]
])),
' '.join(
sorted([
'ε'
if literal == () else literal.text
for literal in follow[non_terminal]
]))),
])