def test_lexer(self):
M = MetaGrammar.from_declaration(
('LETTER', list('ABCDEF')),
('NUMBER', range(4)),
('LBR', '['),
('RBR', ']'),
('string', 'symbol', 'string'),
('string', 'empty'),
('symbol', 'LETTER', 'number'),
('number', 'NUMBER', 'number'),
('number', 'empty'),
)
lexer = lex.lex(module=Lexer(M))
lexer.input('A21[]')
self.assertEqual(list(get_tokens(lexer)), [
('LETTER', 'A'),
('NUMBER', '2'),
('NUMBER', '1'),
('LBR', '['),
('RBR', ']')
]
)
lexer.input('A21X[]')
with self.assertWarns(UserWarning):
list(get_tokens(lexer))
def test_parser(self):
M = MetaGrammar.from_declaration(
('LETTER', list('ABCDEF')),
('NUMBER', range(4)),
('LBR', '['),
('RBR', ']'),
('string', 'string', 'symbol'),
('string', 'string', 'level'),
('string', 'empty'),
('symbol', 'LETTER', 'jump'),
('jump', 'jump', 'NUMBER'),
('jump', 'empty'),
('level', 'LBR', 'substring', 'RBR', 'jump'),
('substring', 'substring', 'symbol'),
('substring', 'empty'),
)
# d = to_agraph(M)
# d.layout('dot')
# d.draw('parser.png')
# parser = Parser(M)
# print(parser.__dict__)
# # yacc.yacc(module=parser, start=M.axiom)
print(SimpleGenerator.generate(M, ['string'], 10))
def _build_parser_module_class(G):
P = MetaGrammar(G.subgraph(G.nonterminals))
item_factory = ast_item_factory(P.terminals)
clsdict = {'G':P}
for lhs, rhs in P.productions:
p_name = 'p_%s_%s' % (lhs, '_'.join(rhs))
if lhs == rhs[0]:
clsdict[p_name] = p_left_recursion(lhs, rhs, item_factory)
elif lhs == rhs[-1]:
clsdict[p_name] = p_right_recursion(lhs, rhs, item_factory)
else:
clsdict[p_name] = p_no_recursion(lhs, rhs, item_factory)
cls = type('ParserModule', (BaseModuleParser,), clsdict)
for lhs, rhs in P.productions:
p_doc_string = """%s : %s""" % (lhs, ' '.join(rhs))
p_name = 'p_%s_%s' % (lhs, '_'.join(rhs))
getattr(cls, p_name).__doc__ = p_doc_string
return cls
def setUp(self):
M = MetaGrammar.from_declaration(
('START', 'S'),
('LETTER', list('ABCDEF')),
('NUMBER', range(4)),
('LBR', '['),
('RBR', ']'),
('REWRITE', ':'),
('expression', 'string'),
('expression', 'production'),
('expression', 'grammar'),
('string', 'symbol', 'string'),
('string', 'level', 'string'),
('string', 'empty'),
('symbol', 'LETTER', 'jump'),
('jump', 'jump', 'NUMBER'),
('jump', 'empty'),
('level', 'LBR', 'substring', 'RBR', 'jump'),
('substring', 'substring', 'symbol'),
('substring', 'substring', 'level'),
('substring', 'empty'),
('grammar', 'axiom', 'NEWLINE', 'productions'),
('productions', 'production', 'NEWLINE', 'productions'),
('productions', 'empty'),
('axiom', 'START', 'REWRITE', 'string'),
('production', 'symbol', 'REWRITE', 'string'),
)
self.parser = Parser(M)
def __init__(self, G):
super().__init__()
self.tokens = []
lex_nodes = []
for u,v in G.edges():
if v in G.terminals:
if v == 'NEWLINE':
if 'NEWLINE' not in self.tokens:
def t_NEWLINE(t):
r'\n+'
t.lexer.lineno += len(t.value)
return t
setattr(self, 't_NEWLINE', t_NEWLINE)
self.tokens.append('NEWLINE')
else:
lex_nodes += [u,v]
L = MetaGrammar(G.subgraph(list(set(lex_nodes))))
for nterm in L.nonterminals:
self.tokens.append(nterm)
if L.out_degree(nterm) == 1:
constant = list(L.successors(nterm))[0]
if isinstance(constant, str):
setattr(self, 't_%s' % nterm, '%s' % re.escape(constant))
elif isinstance(constant, re._pattern_type):
setattr(self, 't_%s' % nterm, constant.pattern)
else:
raise Exception()
else:
values = list(L.successors(nterm))
setattr(self, 't_%s' % nterm, r'|'.join(['%s' % i for i in values]))
self.L = L
def test_meta_grammar(self):
M = MetaGrammar.from_declaration(
('S', 'F'),
('S', '(', 'S', '+', 'F', ')'),
('F', '1'),
)
self.assertEqual(M.axiom, 'S')
self.assertEqual(M.terminals, {'(', ')', '1', '+'})
self.assertEqual(M.nonterminals, {'S', 'F'})
self.assertEqual(sorted(list(M.productions)), sorted([
('S', ('(', 'S', '+', 'F', ')')),
('S', ('F',)),
('F', ('1',)),
]))
self.assertEqual(M.alphabet, {'(', '+', 'S', 'F', '1', ')'})
def test_lsys_grammar(self):
M = MetaGrammar.from_declaration(
('START', 'S'),
('LETTER', list('ABCDEF')),
('NUMBER', range(4)),
('LBR', '['),
('RBR', ']'),
('REWRITE', ':'),
('expression', 'string'),
('expression', 'production'),
('expression', 'grammar'),
('string', 'symbol', 'string'),
('string', 'level', 'string'),
('string', 'empty'),
('symbol', 'LETTER', 'jump'),
('jump', 'jump', 'NUMBER'),
('jump', 'empty'),
('level', 'LBR', 'string', 'RBR', 'jump'),
('grammar', 'axiom', 'NEWLINE', 'productions'),
('productions', 'production', 'NEWLINE', 'productions'),
('productions', 'empty'),
('axiom', 'START', 'REWRITE', 'string'),
('production', 'symbol', 'REWRITE', 'string'),
)
# d = to_agraph(M)
# d.layout('dot')
# d.draw('parser.png')
L = Grammar.from_string(M, """S:A
A:A[C]B
B:A
""")
gen = Generator(L)
def test_parser(self):
M = MetaGrammar.from_declaration(
('LETTER', list('ABCDEF')),
('NUMBER', range(4)),
('LBR', '['),
('RBR', ']'),
('string', 'string', 'symbol'),
('string', 'string', 'level'),
('string', 'empty'),
('symbol', 'LETTER', 'jump'),
('jump', 'jump', 'NUMBER'),
('jump', 'empty'),
('level', 'LBR', 'substring', 'RBR', 'jump'),
('substring', 'substring', 'symbol'),
('substring', 'empty'),
)
# d = to_agraph(M)
# d.layout('dot')
# d.draw('parser.png')
# parser = Parser(M)
# print(parser.__dict__)
# # yacc.yacc(module=parser, start=M.axiom)
parser = Parser(M)
self.assertEqual(str(parser.parse('AB12[CDA]')), 'AB12[CDA]')
with self.assertWarns(UserWarning):
parser.parse('AB12[CDA')
with self.assertWarns(UserWarning):
parser.parse('AB12XCDA')
with self.assertWarns(UserWarning):
parser.parse('AB12]CDA')