def _recurse(tree, **kw): if type(tree) is Compare and type(tree.ops[0]) is In: return q((ast(tree.left)).in_(ast(tree.comparators[0]))) elif type(tree) is GeneratorExp: aliases = map(f(_.target), tree.generators) tables = map(f(_.iter), tree.generators) aliased_tables = map(lambda x: q((ast(x)).alias().c), tables) elt = tree.elt if type(elt) is Tuple: sel = q(ast_list(elt.elts)) else: sel = q([ast(elt)]) out = q(select(ast(sel))) for gen in tree.generators: for cond in gen.ifs: out = q(ast(out).where(ast(cond))) out = q((lambda x: ast(out))()) out.func.args.args = aliases out.args = aliased_tables return out
def test_name_collision(self): sym0 = 1 sym1 = 2 func1 = f(_ + sym0) assert func1(10) == 11 func2 = f(_ + sym0 + _ + sym1) assert func2(10, 10) == 23
def reduce_chain(chain): chain = list(reversed(chain)) o_dict = { "+": f(_+_), "-": f(_-_), "*": f(_*_), "/": f(_/_), } while len(chain) > 1: a, [o, b] = chain.pop(), chain.pop() chain.append(o_dict[o](a, b)) return chain[0]
def reduce_chain(chain): chain = list(reversed(chain)) o_dict = { "+": f(_ + _), "-": f(_ - _), "*": f(_ * _), "/": f(_ / _), } while len(chain) > 1: a, [o, b] = chain.pop(), chain.pop() chain.append(o_dict[o](a, b)) return chain[0]
def test_block(self): with peg: parse1 = ("Hello World", "!".rep1) // f(_[1]) parse2 = parse1 // len with require: parse1.parse_string("Hello World!!!").output == ['!', '!', '!'] parse1.parse_string("Hello World").index == 11 parse2.parse_string("Hello World!!!").output == 3
def test_conversion(self): parse1 = peg((("Hello World", "!".rep1) // f(_[1]))) with require: parse1.parse_string("Hello World!!!").output == ['!', '!', '!'] parse1.parse_string("Hello World").index == 11 parse2 = parse1 // len with require: parse2.parse_string("Hello World!!!").output == 3
def _PegWalker(tree, ctx, stop, collect, **kw): if type(tree) is Str: stop() return q(Parser.Raw(ast(tree))) if type(tree) is BinOp and type(tree.op) is RShift: tree.left, b_left = _PegWalker.recurse_real(tree.left) tree.right = q(lambda bindings: ast(tree.right)) tree.right.args.args = map(f(Name(id=_)), flatten(b_left)) stop() return tree if type(tree) is BinOp and type(tree.op) is FloorDiv: tree.left, b_left = _PegWalker.recurse_real(tree.left) stop() collect(b_left) return tree if type(tree) is Tuple: result = q(Parser.Seq([])) result.args[0].elts = tree.elts all_bindings = [] for i, elt in enumerate(tree.elts): result.args[0].elts[i], bindings = _PegWalker.recurse_real( tree.elts[i]) all_bindings.append(bindings) stop() collect(all_bindings) return result if type(tree) is Compare and type(tree.ops[0]) is Is: left_tree, bindings = _PegWalker.recurse_real(tree.left) new_tree = q(ast(left_tree).bind_to(u(tree.comparators[0].id))) stop() collect(bindings + [tree.comparators[0].id]) return new_tree
def test_arithmetic(self): """ PEG grammar from Wikipedia Op <- "+" / "-" / "*" / "/" Value <- [0-9]+ / '(' Expr ')' Expr <- Value (Op Value)* simplified it to remove operator precedence """ def reduce_chain(chain): chain = list(reversed(chain)) o_dict = { "+": f(_ + _), "-": f(_ - _), "*": f(_ * _), "/": f(_ / _), } while len(chain) > 1: a, [o, b] = chain.pop(), chain.pop() chain.append(o_dict[o](a, b)) return chain[0] with peg: value = '[0-9]+'.r // int | ('(', expr, ')') // (f(_[1])) op = '+' | '-' | '*' | '/' expr = (value is first, (op, value).rep is rest) >> reduce_chain([first] + rest) with require: expr.parse_string("123").output == 123 expr.parse_string("((123))").output == 123 expr.parse_string("(123+456+789)").output == 1368 expr.parse_string("(6/2)").output == 3 expr.parse_string("(1+2+3)+2").output == 8 expr.parse_string( "(((((((11)))))+22+33)*(4+5+((6))))/12*(17+5)").output == 1804
def test_arithmetic(self): """ PEG grammar from Wikipedia Op <- "+" / "-" / "*" / "/" Value <- [0-9]+ / '(' Expr ')' Expr <- Value (Op Value)* simplified it to remove operator precedence """ def reduce_chain(chain): chain = list(reversed(chain)) o_dict = { "+": f(_+_), "-": f(_-_), "*": f(_*_), "/": f(_/_), } while len(chain) > 1: a, [o, b] = chain.pop(), chain.pop() chain.append(o_dict[o](a, b)) return chain[0] with peg: value = '[0-9]+'.r // int | ('(', expr, ')') // (f(_[1])) op = '+' | '-' | '*' | '/' expr = (value is first, (op, value).rep is rest) >> reduce_chain([first] + rest) with require: expr.parse_string("123").output == 123 expr.parse_string("((123))").output == 123 expr.parse_string("(123+456+789)").output == 1368 expr.parse_string("(6/2)").output == 3 expr.parse_string("(1+2+3)+2").output == 8 expr.parse_string("(((((((11)))))+22+33)*(4+5+((6))))/12*(17+5)").output == 1804
def _PegWalker(tree, ctx, stop, collect, **kw): if type(tree) is Str: stop() return q(Parser.Raw(ast(tree))) if type(tree) is BinOp and type(tree.op) is RShift: tree.left, b_left = _PegWalker.recurse_real(tree.left) tree.right = q(lambda bindings: ast(tree.right)) tree.right.args.args = map(f(Name(id = _)), flatten(b_left)) stop() return tree if type(tree) is BinOp and type(tree.op) is FloorDiv: tree.left, b_left = _PegWalker.recurse_real(tree.left) stop() collect(b_left) return tree if type(tree) is Tuple: result = q(Parser.Seq([])) result.args[0].elts = tree.elts all_bindings = [] for i, elt in enumerate(tree.elts): result.args[0].elts[i], bindings = _PegWalker.recurse_real(tree.elts[i]) all_bindings.append(bindings) stop() collect(all_bindings) return result if type(tree) is Compare and type(tree.ops[0]) is Is: left_tree, bindings = _PegWalker.recurse_real(tree.left) new_tree = q(ast(left_tree).bind_to(u(tree.comparators[0].id))) stop() collect(bindings + [tree.comparators[0].id]) return new_tree
def call(*args): if args not in cache: f.cache[args] = f(*args) return f.cache[args]
def test_attribute(self): assert map(f(_.split(' ')[0]), ["i am cow", "hear me moo"]) == ["i", "hear"]
def test_no_args(self): from random import random thunk = f(random()) assert thunk() != thunk()
def test_basic(self): assert map(f(_ - 1), [1, 2, 3]) == [0, 1, 2] assert reduce(f(_ + _), [1, 2, 3]) == 6
def test_partial(self): basetwo = f(int(_, base=2)) assert basetwo('10010') == 18
class Tests(unittest.TestCase): def test_basic(self): parse1 = peg("Hello World") with require: parse1.parse_string("Hello World").output == 'Hello World' parse1.parse_string("Hello, World").index == 0 parse2 = peg(("Hello World", (".").r)) with require: parse2.parse_string("Hello World").index == 11 parse2.parse_string("Hello World1").output == ['Hello World', '1'] parse2.parse_string("Hello World ").output == ['Hello World', ' '] def test_operators(self): parse1 = peg("Hello World") parse2 = peg((parse1, "!".rep1)) with require: parse2.parse_string("Hello World!!!").output == [ 'Hello World', ['!', '!', '!'] ] parse2.parse_string("Hello World!").output == [ 'Hello World', ['!'] ] parse2.parse_string("Hello World").index == 11 parse3 = peg((parse1, ("!" | "?"))) with require: parse3.parse_string("Hello World!").output == ['Hello World', '!'] parse3.parse_string("Hello World?").output == ['Hello World', '?'] parse3.parse_string("Hello World%").index == 11 parse4 = peg((parse1, "!".rep & "!!!")) with require: parse4.parse_string("Hello World!!!").output == [ 'Hello World', ['!', '!', '!'] ] parse4.parse_string("Hello World!!").index == 11 parse4 = peg((parse1, "!".rep & "!!!")) with require: parse4.parse_string("Hello World!!!").output == [ "Hello World", ["!", "!", "!"] ] parse5 = peg((parse1, "!".rep & - "!!!")) with require: parse5.parse_string("Hello World!!").output == [ "Hello World", ['!', '!'] ] parse5.parse_string("Hello World!!!").index == 11 parse6 = peg((parse1, "!" * 3)) with require: parse6.parse_string("Hello World!").index == 12 parse6.parse_string("Hello World!!").index == 13 parse6.parse_string("Hello World!!!").output == [ "Hello World", ['!', '!', '!'] ] parse6.parse_string("Hello World!!!!").index == 14 def test_conversion(self): parse1 = peg((("Hello World", "!".rep1) // f(_[1]))) with require: parse1.parse_string("Hello World!!!").output == ['!', '!', '!'] parse1.parse_string("Hello World").index == 11 parse2 = parse1 // len with require: parse2.parse_string("Hello World!!!").output == 3 def test_block(self): with peg: parse1 = ("Hello World", "!".rep1) // f(_[1]) parse2 = parse1 // len with require: parse1.parse_string("Hello World!!!").output == ['!', '!', '!'] parse1.parse_string("Hello World").index == 11 parse2.parse_string("Hello World!!!").output == 3 def test_recursive(self): with peg: expr = ("(", expr, ")").rep | "" with require: expr.parse_string("()").output expr.parse_string("(()())").output expr.parse_partial("(((()))))").output expr.parse_partial("((()))))").output expr.parse_string("((()))))").index == 6 expr.parse_partial(")((()()))(").output == [] expr.parse_string(")((()()))(").index == 0 expr.parse_partial(")()").output == [] expr.parse_string(")()").index == 0 def test_bindings(self): with peg: short = ("omg" is wtf) >> wtf * 2 medium = ("omg" is o, " ", "wtf" is w, " ", "bb+q".r is b) >> o + w + b seq1 = ("l", ("ol".rep1) is xxx) >> xxx seq2 = ("l", ("ol" is xxx).rep1) >> xxx seq3 = ("l", ("ol" is xxx).rep1) >> sum(map(len, xxx)) with require: short.parse_string('omg').output == 'omgomg' short.parse_string('omgg').index == 3 short.parse_string('cow').index == 0 medium.parse_string('omg wtf bbq').output == 'omgwtfbbq' medium.parse_string('omg wtf bbbbbq').output == 'omgwtfbbbbbq' medium.parse_string('omg wtf bbqq').index == 11 seq3.parse_string("lolololol").output == 8 for x in ["lol", "lolol", "ol", "'"]: if type(seq1.parse_string(x)) is Success: require( seq1.parse_string(x).output == seq2.parse_string(x).output) else: require( seq1.parse_string(x).index == seq2.parse_string(x).index) def test_arithmetic(self): """ PEG grammar from Wikipedia Op <- "+" / "-" / "*" / "/" Value <- [0-9]+ / '(' Expr ')' Expr <- Value (Op Value)* simplified it to remove operator precedence """ def reduce_chain(chain): chain = list(reversed(chain)) o_dict = { "+": f(_ + _), "-": f(_ - _), "*": f(_ * _), "/": f(_ / _), } while len(chain) > 1: a, [o, b] = chain.pop(), chain.pop() chain.append(o_dict[o](a, b)) return chain[0] with peg: value = '[0-9]+'.r // int | ('(', expr, ')') // (f(_[1])) op = '+' | '-' | '*' | '/' expr = (value is first, (op, value).rep is rest) >> reduce_chain([first] + rest) with require: expr.parse_string("123").output == 123 expr.parse_string("((123))").output == 123 expr.parse_string("(123+456+789)").output == 1368 expr.parse_string("(6/2)").output == 3 expr.parse_string("(1+2+3)+2").output == 8 expr.parse_string( "(((((((11)))))+22+33)*(4+5+((6))))/12*(17+5)").output == 1804 def test_cut(self): with peg: expr1 = ("1", cut, "2", "3") | ("1", "b", "c") expr2 = ("1", "2", "3") | ("1", "b", "c") with require: expr1.parse_string("1bc").index == 1 expr2.parse_string("1bc").output == ['1', 'b', 'c'] def test_bindings_json(self): def test(parser, string): import json try: parser.parse_string(string).output == json.loads(string) except Exception, e: print(parser.parse_string(string)) print(json.loads(string)) raise e """ JSON <- S? ( Object / Array / String / True / False / Null / Number ) S? Object <- "{" ( String ":" JSON ( "," String ":" JSON )* / S? ) "}" Array <- "[" ( JSON ( "," JSON )* / S? ) "]" String <- S? ["] ( [^ " \ U+0000-U+001F ] / Escape )* ["] S? Escape <- [\] ( [ " / \ b f n r t ] / UnicodeEscape ) UnicodeEscape <- "u" [0-9A-Fa-f]{4} True <- "true" False <- "false" Null <- "null" Number <- Minus? IntegralPart FractionalPart? ExponentPart? Minus <- "-" IntegralPart <- "0" / [1-9] [0-9]* FractionalPart <- "." [0-9]+ ExponentPart <- ( "e" / "E" ) ( "+" / "-" )? [0-9]+ S <- [ U+0009 U+000A U+000D U+0020 ]+ """ with peg: json_exp = (space.opt, (obj | array | string | true | false | null | number) is exp, space.opt) >> exp pair = (string is k, ':', cut, json_exp is v) >> (k, v) obj = ('{', cut, pair is first, (',', pair is rest).rep, space.opt, '}') >> dict([first] + rest) array = ('[', cut, json_exp is first, (',', json_exp is rest).rep, space.opt, ']') >> [first] + rest string = (space.opt, '"', ('[^"]'.r | escape).rep // ("".join) is body, '"') >> "".join(body) escape = '\\', ('"' | '/' | '\\' | 'b' | 'f' | 'n' | 'r' | 't' | unicode_escape) unicode_escape = 'u', '[0-9A-Fa-f]'.r * 4 true = 'true' >> True false = 'false' >> False null = 'null' >> None number = (minus.opt, integral, fractional.opt, exponent.opt) // f( float("".join(_))) minus = '-' integral = '0' | '[1-9][0-9]*'.r fractional = ('.', '[0-9]+'.r) // "".join exponent = (('e' | 'E'), ('+' | '-').opt, "[0-9]+".r) // "".join space = '\s+'.r # test Success test(number, "12031.33123E-2") test(string, '"i am a cow lol omfg"') test(array, '[1, 2, "omg", ["wtf", "bbq", 42]]') test(obj, '{"omg": "123", "wtf": 456, "bbq": "789"}') test(json_exp, '{"omg": 1, "wtf": 12.4123} ') test( json_exp, """ { "firstName": "John", "lastName": "Smith", "age": 25, "address": { "streetAddress": "21 2nd Street", "city": "New York", "state": "NY", "postalCode": 10021 }, "phoneNumbers": [ { "type": "home", "number": "212 555-1234" }, { "type": "fax", "number": "646 555-4567" } ] } """) # test Failure with self.assertRaises(ParseError) as e: json_exp.parse('{ : 1, "wtf": 12.4123}') assert e.exception.message ==\ """ index: 5, line: 1, col: 6 json_exp / obj / pair / string { : 1, "wtf": 12.4123} ^ """.strip() with self.assertRaises(ParseError) as e: json_exp.parse('{"omg": "123", "wtf": , "bbq": "789"}') assert e.exception.message ==\ """ index: 22, line: 1, col: 23 json_exp / obj / pair / json_exp {"omg": "123", "wtf": , "bbq": "789"} ^ """.strip() with self.assertRaises(ParseError) as e: json_exp.parse("""{ "firstName": "John", "lastName": "Smith", "age": 25, "address": { "streetAddress": "21 2nd Street", "city": "New York", "state": "NY", "postalCode": 10021 }, "phoneNumbers": [ { "type": "home", "number": "212 555-1234" }, { "type": "fax", "number": 646 555-4567" } ] } """) assert e.exception.message == \ """ index: 655, line: 18, col: 43 json_exp / obj / pair / json_exp / array / json_exp / obj "number": 646 555-4567" ^ """.strip()