def setUp(self):
self.plus, self.mul, self.num = CFGTerminal.create("+", "*", "num")
E, T, F = CFGNonterminal.create("E", "T", "F")
E.production(lambda i: i, T)
T.production(lambda i, j, k: i + k, T, self.plus, F)
T.production(lambda i: i, F)
F.production(lambda i, j, k: i * k, F, self.mul, self.num)
F.production(lambda i: i, self.num)
self.parser = CFGParser(E, (self.plus, self.mul, self.num), (E, T, F))
def setUp(self):
S, L = CFGNonterminal.create("S", "L")
self.op, self.cp, self.c, self.x = CFGTerminal.create("(", ")", ",", "x")
S.production(lambda i, j, k: j, self.op, L, self.cp)
S.production(lambda i: i, self.x)
L.production(lambda i: [i], S)
def append(l, i):
l.append(i)
return l
L.production(lambda i, j, k: append(i, k), L, self.c, S)
self.parser = CFGParser(S, (self.op, self.cp, self.c, self.x), (S, L))
def setUp( self ):
plus, times, num = CFGTerminal.create("+", "*", "num")
E, T, F = CFGNonterminal.create("E", "T", "F")
E.production( lambda i: i, T )
T.production( lambda i, j, k: i.add(k), T, plus, F )
T.production( lambda i: i, F )
F.production( lambda i, j, k: i.mul(k), F, times, num )
F.production( lambda i: i, num )
tokens = {
'\\+': plus, '\\*': times,
"[0-9][0-9]*": num.data_wrapper(Number)
}
self.lexer = CFGLexer( tokens )
self.parser = CFGParser( E, (plus, times, num), (E, T, F) )
def _build_parser( self ):
self.terminals = ( start_section, end_section,
tokens_id, productions_id,
start_name, end_name, produces, colon,
newline, indent, dedent, operator, comment, identifier,
literal, number, eof ) = (
CFGTerminal.create( "start_section", "end_section",
"tokens_id", "productions_id",
"start_name", "end_name", "produces",
"colon", "newline", "indent", "dedent",
"operator", "comment", "identifier", "literal", "number",
"eof" ) )
for t in self.terminals:
setattr( self, t.identifier, t )
make_tuple = lambda *args: tuple(args)
self.tokens = []
self.tokens.append(CFGToken(r"{%", start_section))
self.tokens.append(CFGToken(r"%}", end_section))
self.tokens.append(CFGToken(r"tokens",
tokens_id.data_wrapper(Identifier)))
self.tokens.append(CFGToken(r"productions",
productions_id.data_wrapper(Identifier)))
self.tokens.append(CFGToken(r"\(%", start_name))
self.tokens.append(CFGToken(r"%\)", end_name))
self.tokens.append(CFGToken(r"->", produces))
self.tokens.append(CFGToken(r":",
colon.data_wrapper(Operator)))
operators = [
",", ";", "@",
"+", "-", "\\*", "/", "//", "!", "\\|", "&",
"<<", ">>", "<", ">", "=", "\\.",
"%", "`", "~", "\\^"
]
for o in operators:
self.tokens.append(CFGToken(o, operator.data_wrapper(Operator)))
assign_operators = [
"+=", "-=", "\\*=", "/=", "//=",
"!=", "\\|=", "&=", "<=", ">=", "==", "%=",
"<<=", ">>=" ]
for o in assign_operators:
self.tokens.append(CFGToken(o, operator.data_wrapper(Operator)))
paren_operators = [ "\(", "\[", "{", "\)", "\]", "}" ]
for p in paren_operators[:3]:
self.tokens.append(CFGToken(p, self._paren_open))
for p in paren_operators[3:]:
self.tokens.append(CFGToken(p, self._paren_close))
self.tokens.append(CFGToken(r"[a-zA-Z_][a-zA-Z0-9_]*",
identifier.data_wrapper( Identifier )))
self.tokens.append(CFGToken(
r"([0-9][0-9]*(\.[0-9]*)?|\.[0-9][0-9]*)([eE][+-]?[0-9][0-9]*)?",
number.data_wrapper( Number )))
self.tokens.append(CFGToken(r'"([^\\"\n]*(\\.)?)*"',
literal.data_wrapper( Literal )))
self.tokens.append(CFGToken(r"'([^\\'\n]*(\\.)?)*'",
literal.data_wrapper( Literal )))
self.tokens.append(CFGToken(r'r"([^"\n]*(\\")?)*"',
literal.data_wrapper( Literal )))
self.tokens.append(CFGToken(r"r'([^'\n]*(\\')?)*'",
literal.data_wrapper( Literal )))
self.tokens.append(CFGToken(
r'"""([^"]*("|"")?)*"""',
literal.data_wrapper( Literal )))
self.tokens.append(CFGToken(
r"'''([^']*('|'')?)*'''",
literal.data_wrapper( Literal )))
self.tokens.append(CFGToken(r"^[ \t]*\n", self._newline_handler(True)))
self.tokens.append(CFGToken(r"^[ \t]*", self._indent_handler, True))
self.tokens.append(CFGToken(r"\n", self._newline_handler(False)))
self.tokens.append(CFGToken(r"[ \t\r]", CFGToken.NoToken))
self.tokens.append(CFGToken(r"\\\n^", CFGToken.NoToken))
self.tokens.append(CFGToken(r"^[ \t]*#[^\n]*\n",
self._comment_line_handler, True))
self.tokens.append(CFGToken(r"#[^\n]*",
comment.data_wrapper( Comment )))
self.nonterminals = ( goal, cfg_file, section,
code, code_line, code_lines, code_bits,
naming, regexes, products, productions,
regex, production ) = (
CFGNonterminal.create(
"goal", "cfg_file", "section",
"code", "code_line", "code_lines", "code_bits",
"naming", "regexes", "products", "productions",
"regex", "production"
) )
for t in self.nonterminals:
setattr( self, t.identifier, t )
make_list = lambda *args: list(args)
def append( l, i ): l.append(i); return l
def append_tuple( l, *a ): l.append( tuple(a) ); return l
first = lambda a, *args: a
second = lambda a, b, *args: b
third = lambda a, b, c, *args: c
fourth = lambda a, b, c, d, *args: d
goal.production( first, cfg_file, eof )
cfg_file.production( File.append, cfg_file, section, code_lines )
cfg_file.production( File, code_lines )
code_toks = [ operator, identifier, number, literal, colon,
tokens_id, productions_id ]
for t in code_toks:
code_bits.production( append, code_bits, t )
code_bits.production( make_list, t )
code_line.production( append, code_bits, newline )
code_line.production( lambda i: [], newline )
code_lines.production( Code.append, code_lines, code_line )
code_lines.production( Code.add_block,
code_lines, indent, code_lines, dedent )
code_lines.production( lambda: Code() )
code.production( lambda _, c, __: Code(c), colon, code_bits, newline )
code.production( fourth, colon, newline, indent, code_lines, dedent )
code.production( lambda n: Code(), newline )
section.production( TokenSection.create,
start_section, tokens_id, regexes, end_section )
section.production( ProductionSection.create,
start_section, productions_id, productions, end_section )
naming.production( second, start_name, identifier, end_name )
naming.production( lambda: Identifier("", None, None) )
regex.production( Regex, literal, naming, code )
regexes.production( append, regexes, regex )
for white in [ newline, indent, dedent ]:
regexes.production( first, regexes, white )
regexes.production( lambda: [] )
products.production( append_tuple, products, identifier, naming )
products.production( lambda: [] )
production.production( Production.create,
identifier, produces, products, code )
productions.production( append, productions, production )
for white in [ newline, indent, dedent ]:
productions.production( first, productions, white )
productions.production( lambda: [] )