def parse(seq): """Returns the AST of the given token sequence.""" global depth unarg = lambda f: lambda x: f(*x) tokval = lambda x: x.value # returns the value of a token toktype = lambda t: some(lambda x: x.type == t) >> tokval # checks type of token paren = lambda s: a(Token('Parentheses', s)) >> tokval # return the value if token is Op paren_ = lambda s: skip(paren(s)) # checks if token is Op and ignores it def application(z, list): return reduce(lambda s, x: Application(s, x), list, z) depth = 0 variable = lambda x: Variable(str(x)+":"+str(depth)) def abstraction(x): global depth abst = Abstraction(str(x[0])+":"+str(depth), x[1]) depth += 1 return abst variable = toktype('Name') >> variable term = variable | with_forward_decls(lambda: paren_('(') + exp + paren_(')')) | \ with_forward_decls(lambda: skip(toktype('Lambda')) + toktype('Name') + \ skip(toktype('Dot')) + exp >> abstraction) exp = term + many(term) >> unarg(application) return exp.parse(seq)
def parse(seq): """Returns the AST of the given token sequence.""" def eval_expr(z, list): return reduce(lambda s, (f, x): f(s, x), list, z) unarg = lambda f: lambda x: f(*x) const = lambda x: lambda _: x # like ^^^ in Scala tokval = lambda x: x.value # returns the value of a token op = lambda s: a(Token('Op', s)) >> tokval # return the value if token is Op op_ = lambda s: skip(op(s)) # checks if token is Op and ignores it toktype = lambda t: some(lambda x: x.type == t) >> tokval # checks type of token def lst(h,t): return [h,] + t makeop = lambda s, f: op(s) >> const(f) or_op = makeop('|', Or) char = with_forward_decls(lambda: toktype('Char') >> Char | op_('(') + exp + op_(')')) star = char + op_('*') >> Star | char lst2_exp = star + many(star) >> unarg(lst) lst_exp = lst2_exp >> Lst exp = lst_exp + many(or_op + lst_exp) >> unarg(eval_expr) return exp.parse(seq)
def parse(seq): """ Parses the list of tokens and generates an AST. """ def eval_expr(z, list): return reduce(lambda s, (f, x): f(s, x), list, z) unarg = lambda f: lambda x: f(*x) tokval = lambda x: x.value # returns the value of a token toktype = lambda t: some(lambda x: x.type == t) >> tokval # checks type of token const = lambda x: lambda _: x # like ^^^ in Scala op = lambda s: a(Token('Op', s)) >> tokval # return the value if token is Op op_ = lambda s: skip(op(s)) # checks if token is Op and ignores it lst = lambda x: [x[0],] + x[1] tup = lambda x: (x[0], x[1]) makeop = lambda s, f: op(s) >> const(f) add = makeop('+', Add) sub = makeop('-', Sub) mul = makeop('*', Mul) div = makeop('/', Div) lt = makeop('<', Lt) gt = makeop('>', Gt) eq = makeop('=', Eq) operation = add | sub | mul | div | lt | gt | eq decl = with_forward_decls(lambda:toktype('Var') + op_('=') + (exp | fun) >> tup) decls = decl + many(skip(toktype('Semicolon')) + decl) >> lst variable = toktype('Var') >> Variable variables = variable + many(skip(toktype('Comma')) + variable) >> lst fun = with_forward_decls(lambda: skip(toktype('Fun')) + variables + skip(toktype('Arrow')) + exp + skip(toktype('End'))) >> unarg(Fun) parameters = with_forward_decls(lambda: exp + many(skip(toktype('Comma')) + exp) >> lst) call = skip(toktype('Call')) + (fun | variable) + skip(toktype('Lp')) + parameters + skip(toktype('Rp')) >> unarg(Call) ex = with_forward_decls(lambda:variable | toktype('Number') >> (lambda x: Const(int(x))) |\ toktype('True') >> (lambda x: Const(True)) | toktype('False') >> (lambda x: Const(False)) |\ skip(toktype('Let')) + decls + skip(toktype('In')) + exp + skip(toktype('End')) >> unarg(Let) |\ skip(toktype('If')) + exp + skip(toktype('Then')) + exp + maybe(skip(toktype('Else')) + exp) + skip(toktype('Fi')) >> unarg(If) |\ fun | call) exp = ex + many(operation + ex) >> unarg(eval_expr) prog = skip(toktype('Prog')) + exp >> Prog return prog.parse(seq)
def parse(seq): """Returns the AST of the given token sequence.""" def eval_expr(z, list): return reduce(lambda s, (f, x): f(s, x), list, z) unarg = lambda f: lambda x: f(*x) const = lambda x: lambda _: x # like ^^^ in Scala tokval = lambda x: x.value # returns the value of a token op = lambda s: a(Token('Op', s)) >> tokval # return the value if token is Op op_ = lambda s: skip(op(s)) # checks if token is Op and ignores it toktype = lambda t: some(lambda x: x.type == t) >> tokval # checks type of token def lst(h,t): return [h,] + t call = lambda x: Call(x[0], x[1]) makeop = lambda s, f: op(s) >> const(f) add = makeop('+', Plus) sub = makeop('-', Minus) mul = makeop('*', Times) div = makeop('/', Div) def make_const(i): return const(int(i)) number = toktype('Number') >> Const mul_op = mul | div add_op = add | sub factor = with_forward_decls(lambda: number | op_('(') + exp + op_(')') | call) term = factor + many(mul_op + factor) >> unarg(eval_expr) exp = term + many(add_op + term) >> unarg(eval_expr) exp_lst = with_forward_decls(lambda: exp + many(op_(',') + exp) >> unarg(lst)) call = toktype('Name') + op_('(') + exp_lst + op_(')') >> call return exp.parse(seq)
def parse(tokens): ## building blocks kw_priority = some(toktype("kw_priority")) kw_probability = some(toktype("kw_probability")) kw_reaction = some(toktype("kw_reaction")) kw_exists = some(toktype("kw_exists")) kw_as = some(toktype("kw_as")) op_tilde = some(toktype("op_tilde")) op_priority_maximal = some(toktype("op_priority_maximal")) op_production = some(toktype("op_production")) atom = some(toktype("name")) number = some(toktype("number")) dissolve = some(toktype("op_dissolve")) osmose = some(toktype("op_osmose")) osmose_location = some(toktype("op_osmose_location")) env_open = some(toktype("env_open")) env_close = some(toktype("env_close")) membrane_open = some(toktype("membrane_open")) membrane_close = some(toktype("membrane_close")) ## grammar from the bottom up name = atom | number symbol = atom | (dissolve + maybe(name)) | (osmose + name + maybe(osmose_location + name)) priority = kw_priority + op_tilde + name + op_priority_maximal + name reaction = (kw_reaction + maybe(kw_as + name) + op_tilde + oneplus(name) + op_production + many(symbol)) exists = kw_exists + op_tilde + oneplus(name) expr = (exists | reaction | priority) statement = with_forward_decls(lambda: membrane | expr) >> Statement body = maybe(name) + many(statement) membrane = (skip(membrane_open) + body + skip(membrane_close)) >> Membrane env = (skip(env_open) + body + skip(env_close)) >> Environment program = many(env) + skip(finished) >> Program return program.parse(tokens)
def parse(tokens): ## building blocks kw_priority = some(toktype("kw_priority")) kw_probability = some(toktype("kw_probability")) kw_reaction = some(toktype("kw_reaction")) kw_exists = some(toktype("kw_exists")) kw_as = some(toktype("kw_as")) op_tilde = some(toktype("op_tilde")) op_priority_maximal = some(toktype("op_priority_maximal")) op_production = some(toktype("op_production")) atom = some(toktype("name")) number = some(toktype("number")) dissolve = some(toktype("op_dissolve")) osmose = some(toktype("op_osmose")) osmose_location = some(toktype("op_osmose_location")) env_open = some(toktype("env_open")) env_close = some(toktype("env_close")) membrane_open = some(toktype("membrane_open")) membrane_close = some(toktype("membrane_close")) ## grammar from the bottom up name = atom | number symbol = atom | (dissolve + maybe(name)) | (osmose + name + maybe(osmose_location + name)) priority = kw_priority + op_tilde + name + op_priority_maximal + name reaction = kw_reaction + maybe(kw_as + name) + op_tilde + oneplus(name) + op_production + many(symbol) exists = kw_exists + op_tilde + oneplus(name) expr = exists | reaction | priority statement = with_forward_decls(lambda: membrane | expr) >> Statement body = maybe(name) + many(statement) membrane = (skip(membrane_open) + body + skip(membrane_close)) >> Membrane env = (skip(env_open) + body + skip(env_close)) >> Environment program = many(env) + skip(finished) >> Program return program.parse(tokens)
def parse(tokens): var = some(toktype("name")) | some(toktype("number")) open_form = some(toktype("form_open")) close_form = some(toktype("form_close")) op_lambda = some(toktype("op_lambda")) op_map = some(toktype("op_map")) prim_bind = some(toktype("kw_bind")) prim_halt = some(toktype("kw_halt")) exp = with_forward_decls(lambda: lam | var | prim_exp | exprn) >> Expression lam = open_form + op_lambda + many(var) + op_map + oneplus(exp) + close_form >> Lambda bind_exp = open_form + prim_bind + var + lam + close_form halt_exp = open_form + prim_halt + exp + close_form prim_exp = bind_exp | halt_exp exprn = open_form + oneplus(exp) + close_form >> Form prog = many(exp) + skip(finished) >> Program return prog.parse(tokens)
#@+node:peckj.20140124085532.4003: *3* grammar posnumber = (some(lambda tok: tok.type == 'NUMBER') >> tokval >> make_number) add = makeop('+', operator.add) sub = makeop('-', operator.sub) mul = makeop('*', operator.mul) div = makeop('/', operator.div) pow = makeop('**', operator.pow) negnumber = (sub + posnumber) >> negate number = posnumber | negnumber mul_op = mul | div add_op = add | sub primary = with_forward_decls(lambda: number | (op_('(') + expr + op_(')'))) factor = primary + many(pow + primary) >> f term = factor + many(mul_op + factor) >> f expr = term + many(add_op + term) >> f endmark = a(Token(token.ENDMARKER, '')) end = skip(endmark + finished) toplevel = maybe(expr) + end #@+node:peckj.20140124085532.4012: *3* parse def parse(tokens): return toplevel.parse(tokens) parse_and_run = lambda x: parse(tokenize(x))
return item larg, fun, rarg = item, rest[0], rest[1] return Function(fun, [larg, rarg]) # # Parser. # lparen = some(lambda tok: tok == "(") rparen = some(lambda tok: tok == ")") op = some(lambda tok: tok in "+-*^&|") eof = some(lambda tok: tok == EOF) number = some(lambda tok: tok.isdigit()) >> make_number paren_expr = with_forward_decls( lambda: lparen + expr + rparen ) # *Mark here are not really required, but if you are going to do # anything complex that requires that you discern between different # parsing paths, marks are often give you least hassle. expr = with_forward_decls( lambda: (number + pure(NumberMark) + expr_rest | paren_expr + pure(ParenMark) + expr_rest) >> make_expr) # This one allows us to add more complex expressions like function # application and ternary operators to the above definition with ease. # Otherwise terms such as `apply = expr lparen many(expr) rpanen` # would be impossible to add, always leading to infinite left recursion. expr_rest = maybe(op + expr)
Return a parser that tries to parse p, and raises a CustomParseError when it fails. """ @Parser def _try_p(tokens, s): try: return p.run(tokens, s) except NoParseError as err: raise CustomParseError(msg, err) return _try_p # Grammar starts here test = with_forward_decls( lambda: choice([or_test + maybe(IF + or_test + ELSE + test), lambdef])) not_test = with_forward_decls(lambda: (NOT + not_test) | comparison) and_test = sep_by(not_test, AND) or_test = sep_by(and_test, OR) test_nocond = with_forward_decls(lambda: or_test | lambdef_nocond) testlist = with_forward_decls(lambda: sep_by(test)) atom = with_forward_decls(lambda: choice([ between(yield_expr | testlist_comp, empty=True), between(testlist_comp, op_('['), op_(']'), empty=True), between(dictorsetmaker, op_('{'), op_('}'), empty=True), IDENTIFIER, NUMBER, STRING + many(STRING), ELLIPSIS, NONE, TRUE, FALSE ])) sliceop = COLON + maybe(test) subscript = test | (maybe(test) + COLON + maybe(test) + maybe(sliceop))
Return a parser that tries to parse p, and raises a CustomParseError when it fails. """ @Parser def _try_p(tokens, s): try: return p.run(tokens, s) except NoParseError as err: raise CustomParseError(msg, err) return _try_p # Grammar starts here test = with_forward_decls(lambda: choice([ or_test + maybe(IF + or_test + ELSE + test), lambdef])) not_test = with_forward_decls(lambda: (NOT + not_test) | comparison) and_test = sep_by(not_test, AND) or_test = sep_by(and_test, OR) test_nocond = with_forward_decls(lambda: or_test | lambdef_nocond) testlist = with_forward_decls(lambda: sep_by(test)) atom = with_forward_decls(lambda: choice([ between(yield_expr | testlist_comp, empty=True), between(testlist_comp, op_('['), op_(']'), empty=True), between(dictorsetmaker, op_('{'), op_('}'), empty=True), IDENTIFIER, NUMBER, STRING + many(STRING),
def parse(seq): """ Parses the list of tokens and generates an AST. """ def eval_expr(z, list): return reduce(lambda s, (f, x): f(s, x), list, z) unarg = lambda f: lambda x: f(*x) tokval = lambda x: x.value # returns the value of a token toktype = lambda t: some(lambda x: x.type == t) >> tokval # checks type of token const = lambda x: lambda _: x # like ^^^ in Scala eval_cond = lambda x: x[1](x[0], x[2]) op = lambda s: a(Token('Op', s)) >> tokval # return the value if token is Op op_ = lambda s: skip(op(s)) # checks if token is Op and ignores it ident = lambda s: a(Token('Ident', s)) >> tokval # return the value if token is Op ident_ = lambda s: skip(ident(s)) # checks if token is Op and ignores it lst = lambda x: [x[0],] + x[1] makeop = lambda s, f: op(s) >> const(f) add = makeop('+', Add) sub = makeop('-', Sub) mul = makeop('*', Mul) div = makeop('/', Div) mod = makeop('%', Mod) lt = makeop('<', Lt) gt = makeop('>', Gt) eq = toktype('Eq') >> const(Eq) orop = toktype('Or') >> const(Or) andop = toktype('And') >> const(And) neq = toktype('Neq') >> const(Neq) notop = toktype('Not') >> const(Not) le = toktype('Le') >> const(Le) ge = toktype('Ge') >> const(Ge) point_op = mul | div | mod line_op = add | sub comp_op = lt | gt | eq | neq | notop | le | ge combinator = orop | andop empty_fun = lambda x: () heap_assign = with_forward_decls(lambda: pointer + skip(toktype('Assign')) + exp >> unarg(HeapAssign)) assign = with_forward_decls(lambda: toktype('Ident') + skip(toktype('Assign')) + exp >> unarg(Assignment)) ifexp = with_forward_decls(lambda: ident_('if') + cond + cmd + \ maybe(ident_('else') + cmd) >> unarg(IfThenElse)) whileexp = with_forward_decls(lambda: ident_('while') + cond + cmd >> unarg(While)) printexp = with_forward_decls(lambda: ident_('print') + exp >> Print) vardef = with_forward_decls(lambda: ident_('var') + toktype('Ident') + op_('=') + exp \ >> unarg(Declaration)) arglst = with_forward_decls(lambda: exp + many(skip(toktype('Comma')) + exp) >> lst) returnexp = with_forward_decls(lambda: ident_('return') + exp) call = with_forward_decls(lambda: dot + skip(toktype('Lp') + toktype('Rp')) >> Call | \ dot + skip(toktype('Lp')) + arglst + skip(toktype('Rp')) >> unarg(Call)) args = toktype('Ident') + many(skip(toktype('Comma')) + toktype('Ident')) >> lst decl = with_forward_decls(lambda: toktype('Ident') + op_('=') + exp >> unarg(Declaration)) decls = decl + many(skip(toktype('Semicolon')) + decl) >> lst cmd = with_forward_decls(lambda: call | returnexp | assign | ifexp | whileexp | printexp | vardef | heap_assign | \ skip(toktype('Lb')) + cmd_list + skip(toktype('Rb'))) cmd_list = (cmd + many(skip(toktype('Semicolon')) + cmd) >> lst) >> CmdList variable = toktype('Ident') >> Variable dotop = toktype('Dot') >> const(Dot) dot = variable + many(dotop + toktype('Ident')) >> unarg(eval_expr) pointer = with_forward_decls(lambda: op_('*') + exp >> Pointer) constexp = toktype('Number') >> Const string = toktype('String') >> String factor = with_forward_decls(lambda: dot | constexp | pointer | string | \ skip(toktype('Lp')) + exp + skip(toktype('Rp'))) summand = factor + many(point_op + factor) >> unarg(eval_expr) function = ident_('function') + (skip(toktype('Lp')) + skip(toktype('Rp')) >> empty_fun | \ skip(toktype('Lp')) + args + skip(toktype('Rp'))) + skip(toktype('Lb')) + cmd_list \ + skip(toktype('Rb')) >> unarg(Function) objectexp = ident_('object') + skip(toktype('Lb')) + decls + skip(toktype('Rb')) >> Object exp = with_forward_decls(lambda: objectexp | function | call | \ summand + many(line_op + summand) >> unarg(eval_expr) | cond) cond = (exp + comp_op + exp >> eval_cond) + many(combinator + (exp + comp_op + exp >> eval_cond)) >> unarg(eval_expr) return cmd.parse(seq)