Exemple #1
0
def _recurse(tree, **kw):
    if type(tree) is Compare and type(tree.ops[0]) is In:
        return q((ast(tree.left)).in_(ast(tree.comparators[0])))

    elif type(tree) is GeneratorExp:

        aliases = map(f(_.target), tree.generators)
        tables = map(f(_.iter), tree.generators)

        aliased_tables = map(lambda x: q((ast(x)).alias().c), tables)

        elt = tree.elt
        if type(elt) is Tuple:
            sel = q(ast_list(elt.elts))
        else:
            sel = q([ast(elt)])

        out = q(select(ast(sel)))

        for gen in tree.generators:
            for cond in gen.ifs:
                out = q(ast(out).where(ast(cond)))


        out = q((lambda x: ast(out))())
        out.func.args.args = aliases
        out.args = aliased_tables
        return out
Exemple #2
0
 def test_name_collision(self):
     sym0 = 1
     sym1 = 2
     func1 = f(_ + sym0)
     assert func1(10) == 11
     func2 = f(_ + sym0 + _ + sym1)
     assert func2(10, 10) == 23
Exemple #3
0
 def reduce_chain(chain):
     chain = list(reversed(chain))
     o_dict = {
         "+": f(_+_),
         "-": f(_-_),
         "*": f(_*_),
         "/": f(_/_),
     }
     while len(chain) > 1:
         a, [o, b] = chain.pop(), chain.pop()
         chain.append(o_dict[o](a, b))
     return chain[0]
Exemple #4
0
 def reduce_chain(chain):
     chain = list(reversed(chain))
     o_dict = {
         "+": f(_ + _),
         "-": f(_ - _),
         "*": f(_ * _),
         "/": f(_ / _),
     }
     while len(chain) > 1:
         a, [o, b] = chain.pop(), chain.pop()
         chain.append(o_dict[o](a, b))
     return chain[0]
Exemple #5
0
    def test_block(self):
        with peg:
            parse1 = ("Hello World", "!".rep1) // f(_[1])
            parse2 = parse1 // len

        with require:
            parse1.parse_string("Hello World!!!").output == ['!', '!', '!']
            parse1.parse_string("Hello World").index == 11
            parse2.parse_string("Hello World!!!").output == 3
Exemple #6
0
    def test_block(self):
        with peg:
            parse1 = ("Hello World", "!".rep1) // f(_[1])
            parse2 = parse1 // len

        with require:
            parse1.parse_string("Hello World!!!").output == ['!', '!', '!']
            parse1.parse_string("Hello World").index == 11
            parse2.parse_string("Hello World!!!").output == 3
Exemple #7
0
    def test_conversion(self):
        parse1 = peg((("Hello World", "!".rep1) // f(_[1])))

        with require:
            parse1.parse_string("Hello World!!!").output == ['!', '!', '!']
            parse1.parse_string("Hello World").index == 11

        parse2 = parse1 // len

        with require:
            parse2.parse_string("Hello World!!!").output == 3
Exemple #8
0
    def test_conversion(self):
        parse1 = peg((("Hello World", "!".rep1) // f(_[1])))

        with require:
            parse1.parse_string("Hello World!!!").output == ['!', '!', '!']
            parse1.parse_string("Hello World").index == 11

        parse2 = parse1 // len

        with require:
            parse2.parse_string("Hello World!!!").output == 3
Exemple #9
0
def _PegWalker(tree, ctx, stop, collect, **kw):
    if type(tree) is Str:
        stop()
        return q(Parser.Raw(ast(tree)))

    if type(tree) is BinOp and type(tree.op) is RShift:
        tree.left, b_left = _PegWalker.recurse_real(tree.left)
        tree.right = q(lambda bindings: ast(tree.right))
        tree.right.args.args = map(f(Name(id=_)), flatten(b_left))
        stop()
        return tree

    if type(tree) is BinOp and type(tree.op) is FloorDiv:
        tree.left, b_left = _PegWalker.recurse_real(tree.left)
        stop()
        collect(b_left)
        return tree

    if type(tree) is Tuple:
        result = q(Parser.Seq([]))

        result.args[0].elts = tree.elts
        all_bindings = []
        for i, elt in enumerate(tree.elts):
            result.args[0].elts[i], bindings = _PegWalker.recurse_real(
                tree.elts[i])
            all_bindings.append(bindings)
        stop()
        collect(all_bindings)
        return result

    if type(tree) is Compare and type(tree.ops[0]) is Is:
        left_tree, bindings = _PegWalker.recurse_real(tree.left)
        new_tree = q(ast(left_tree).bind_to(u(tree.comparators[0].id)))
        stop()
        collect(bindings + [tree.comparators[0].id])
        return new_tree
Exemple #10
0
    def test_arithmetic(self):
        """
        PEG grammar from Wikipedia

        Op      <- "+" / "-" / "*" / "/"
        Value   <- [0-9]+ / '(' Expr ')'
        Expr <- Value (Op Value)*

        simplified it to remove operator precedence
        """
        def reduce_chain(chain):
            chain = list(reversed(chain))
            o_dict = {
                "+": f(_ + _),
                "-": f(_ - _),
                "*": f(_ * _),
                "/": f(_ / _),
            }
            while len(chain) > 1:
                a, [o, b] = chain.pop(), chain.pop()
                chain.append(o_dict[o](a, b))
            return chain[0]

        with peg:
            value = '[0-9]+'.r // int | ('(', expr, ')') // (f(_[1]))
            op = '+' | '-' | '*' | '/'
            expr = (value is first,
                    (op, value).rep is rest) >> reduce_chain([first] + rest)

        with require:
            expr.parse_string("123").output == 123
            expr.parse_string("((123))").output == 123
            expr.parse_string("(123+456+789)").output == 1368
            expr.parse_string("(6/2)").output == 3
            expr.parse_string("(1+2+3)+2").output == 8
            expr.parse_string(
                "(((((((11)))))+22+33)*(4+5+((6))))/12*(17+5)").output == 1804
Exemple #11
0
    def test_arithmetic(self):
        """
        PEG grammar from Wikipedia

        Op      <- "+" / "-" / "*" / "/"
        Value   <- [0-9]+ / '(' Expr ')'
        Expr <- Value (Op Value)*

        simplified it to remove operator precedence
        """

        def reduce_chain(chain):
            chain = list(reversed(chain))
            o_dict = {
                "+": f(_+_),
                "-": f(_-_),
                "*": f(_*_),
                "/": f(_/_),
            }
            while len(chain) > 1:
                a, [o, b] = chain.pop(), chain.pop()
                chain.append(o_dict[o](a, b))
            return chain[0]

        with peg:
            value = '[0-9]+'.r // int | ('(', expr, ')') // (f(_[1]))
            op = '+' | '-' | '*' | '/'
            expr = (value is first, (op, value).rep is rest) >> reduce_chain([first] + rest)

        with require:
            expr.parse_string("123").output == 123
            expr.parse_string("((123))").output == 123
            expr.parse_string("(123+456+789)").output == 1368
            expr.parse_string("(6/2)").output == 3
            expr.parse_string("(1+2+3)+2").output == 8
            expr.parse_string("(((((((11)))))+22+33)*(4+5+((6))))/12*(17+5)").output == 1804
Exemple #12
0
def _PegWalker(tree, ctx, stop, collect, **kw):
    if type(tree) is Str:
        stop()
        return q(Parser.Raw(ast(tree)))

    if type(tree) is BinOp and type(tree.op) is RShift:
        tree.left, b_left = _PegWalker.recurse_real(tree.left)
        tree.right = q(lambda bindings: ast(tree.right))
        tree.right.args.args = map(f(Name(id = _)), flatten(b_left))
        stop()
        return tree

    if type(tree) is BinOp and type(tree.op) is FloorDiv:
        tree.left, b_left = _PegWalker.recurse_real(tree.left)
        stop()
        collect(b_left)
        return tree

    if type(tree) is Tuple:
        result = q(Parser.Seq([]))

        result.args[0].elts = tree.elts
        all_bindings = []
        for i, elt in enumerate(tree.elts):
            result.args[0].elts[i], bindings = _PegWalker.recurse_real(tree.elts[i])
            all_bindings.append(bindings)
        stop()
        collect(all_bindings)
        return result

    if type(tree) is Compare and type(tree.ops[0]) is Is:
        left_tree, bindings = _PegWalker.recurse_real(tree.left)
        new_tree = q(ast(left_tree).bind_to(u(tree.comparators[0].id)))
        stop()
        collect(bindings + [tree.comparators[0].id])
        return new_tree
Exemple #13
0
 def call(*args):
     if args not in cache:
         f.cache[args] = f(*args)
     return f.cache[args]
Exemple #14
0
 def test_attribute(self):
     assert map(f(_.split(' ')[0]), ["i am cow", "hear me moo"]) == ["i", "hear"]
Exemple #15
0
 def test_no_args(self):
     from random import random
     thunk = f(random())
     assert thunk() != thunk()
Exemple #16
0
 def test_basic(self):
     assert map(f(_ - 1), [1, 2, 3]) == [0, 1, 2]
     assert reduce(f(_ + _), [1, 2, 3]) == 6
Exemple #17
0
 def test_partial(self):
     basetwo = f(int(_, base=2))
     assert basetwo('10010') == 18
Exemple #18
0
class Tests(unittest.TestCase):
    def test_basic(self):
        parse1 = peg("Hello World")
        with require:
            parse1.parse_string("Hello World").output == 'Hello World'
            parse1.parse_string("Hello, World").index == 0

        parse2 = peg(("Hello World", (".").r))
        with require:
            parse2.parse_string("Hello World").index == 11
            parse2.parse_string("Hello World1").output == ['Hello World', '1']
            parse2.parse_string("Hello World ").output == ['Hello World', ' ']

    def test_operators(self):
        parse1 = peg("Hello World")

        parse2 = peg((parse1, "!".rep1))
        with require:
            parse2.parse_string("Hello World!!!").output == [
                'Hello World', ['!', '!', '!']
            ]
            parse2.parse_string("Hello World!").output == [
                'Hello World', ['!']
            ]
            parse2.parse_string("Hello World").index == 11

        parse3 = peg((parse1, ("!" | "?")))

        with require:
            parse3.parse_string("Hello World!").output == ['Hello World', '!']
            parse3.parse_string("Hello World?").output == ['Hello World', '?']
            parse3.parse_string("Hello World%").index == 11

        parse4 = peg((parse1, "!".rep & "!!!"))

        with require:
            parse4.parse_string("Hello World!!!").output == [
                'Hello World', ['!', '!', '!']
            ]
            parse4.parse_string("Hello World!!").index == 11

        parse4 = peg((parse1, "!".rep & "!!!"))

        with require:
            parse4.parse_string("Hello World!!!").output == [
                "Hello World", ["!", "!", "!"]
            ]

        parse5 = peg((parse1, "!".rep & - "!!!"))
        with require:
            parse5.parse_string("Hello World!!").output == [
                "Hello World", ['!', '!']
            ]
            parse5.parse_string("Hello World!!!").index == 11

        parse6 = peg((parse1, "!" * 3))
        with require:
            parse6.parse_string("Hello World!").index == 12
            parse6.parse_string("Hello World!!").index == 13
            parse6.parse_string("Hello World!!!").output == [
                "Hello World", ['!', '!', '!']
            ]
            parse6.parse_string("Hello World!!!!").index == 14

    def test_conversion(self):
        parse1 = peg((("Hello World", "!".rep1) // f(_[1])))

        with require:
            parse1.parse_string("Hello World!!!").output == ['!', '!', '!']
            parse1.parse_string("Hello World").index == 11

        parse2 = parse1 // len

        with require:
            parse2.parse_string("Hello World!!!").output == 3

    def test_block(self):
        with peg:
            parse1 = ("Hello World", "!".rep1) // f(_[1])
            parse2 = parse1 // len

        with require:
            parse1.parse_string("Hello World!!!").output == ['!', '!', '!']
            parse1.parse_string("Hello World").index == 11
            parse2.parse_string("Hello World!!!").output == 3

    def test_recursive(self):
        with peg:
            expr = ("(", expr, ")").rep | ""

        with require:
            expr.parse_string("()").output
            expr.parse_string("(()())").output
            expr.parse_partial("(((()))))").output

            expr.parse_partial("((()))))").output
            expr.parse_string("((()))))").index == 6
            expr.parse_partial(")((()()))(").output == []
            expr.parse_string(")((()()))(").index == 0
            expr.parse_partial(")()").output == []
            expr.parse_string(")()").index == 0

    def test_bindings(self):
        with peg:
            short = ("omg" is wtf) >> wtf * 2
            medium = ("omg" is o, " ", "wtf" is w, " ",
                      "bb+q".r is b) >> o + w + b
            seq1 = ("l", ("ol".rep1) is xxx) >> xxx
            seq2 = ("l", ("ol" is xxx).rep1) >> xxx
            seq3 = ("l", ("ol" is xxx).rep1) >> sum(map(len, xxx))
        with require:
            short.parse_string('omg').output == 'omgomg'
            short.parse_string('omgg').index == 3
            short.parse_string('cow').index == 0
            medium.parse_string('omg wtf bbq').output == 'omgwtfbbq'
            medium.parse_string('omg wtf bbbbbq').output == 'omgwtfbbbbbq'
            medium.parse_string('omg wtf bbqq').index == 11
            seq3.parse_string("lolololol").output == 8

        for x in ["lol", "lolol", "ol", "'"]:
            if type(seq1.parse_string(x)) is Success:

                require(
                    seq1.parse_string(x).output == seq2.parse_string(x).output)
            else:

                require(
                    seq1.parse_string(x).index == seq2.parse_string(x).index)

    def test_arithmetic(self):
        """
        PEG grammar from Wikipedia

        Op      <- "+" / "-" / "*" / "/"
        Value   <- [0-9]+ / '(' Expr ')'
        Expr <- Value (Op Value)*

        simplified it to remove operator precedence
        """
        def reduce_chain(chain):
            chain = list(reversed(chain))
            o_dict = {
                "+": f(_ + _),
                "-": f(_ - _),
                "*": f(_ * _),
                "/": f(_ / _),
            }
            while len(chain) > 1:
                a, [o, b] = chain.pop(), chain.pop()
                chain.append(o_dict[o](a, b))
            return chain[0]

        with peg:
            value = '[0-9]+'.r // int | ('(', expr, ')') // (f(_[1]))
            op = '+' | '-' | '*' | '/'
            expr = (value is first,
                    (op, value).rep is rest) >> reduce_chain([first] + rest)

        with require:
            expr.parse_string("123").output == 123
            expr.parse_string("((123))").output == 123
            expr.parse_string("(123+456+789)").output == 1368
            expr.parse_string("(6/2)").output == 3
            expr.parse_string("(1+2+3)+2").output == 8
            expr.parse_string(
                "(((((((11)))))+22+33)*(4+5+((6))))/12*(17+5)").output == 1804

    def test_cut(self):
        with peg:
            expr1 = ("1", cut, "2", "3") | ("1", "b", "c")
            expr2 = ("1", "2", "3") | ("1", "b", "c")

        with require:
            expr1.parse_string("1bc").index == 1
            expr2.parse_string("1bc").output == ['1', 'b', 'c']

    def test_bindings_json(self):
        def test(parser, string):
            import json
            try:
                parser.parse_string(string).output == json.loads(string)
            except Exception, e:
                print(parser.parse_string(string))
                print(json.loads(string))
                raise e

        """
        JSON <- S? ( Object / Array / String / True / False / Null / Number ) S?

        Object <- "{"
                     ( String ":" JSON ( "," String ":" JSON )*
                     / S? )
                 "}"

        Array <- "["
                    ( JSON ( "," JSON )*
                    / S? )
                "]"

        String <- S? ["] ( [^ " \ U+0000-U+001F ] / Escape )* ["] S?

        Escape <- [\] ( [ " / \ b f n r t ] / UnicodeEscape )

        UnicodeEscape <- "u" [0-9A-Fa-f]{4}

        True <- "true"
        False <- "false"
        Null <- "null"

        Number <- Minus? IntegralPart FractionalPart? ExponentPart?

        Minus <- "-"
        IntegralPart <- "0" / [1-9] [0-9]*
        FractionalPart <- "." [0-9]+
        ExponentPart <- ( "e" / "E" ) ( "+" / "-" )? [0-9]+
        S <- [ U+0009 U+000A U+000D U+0020 ]+

        """
        with peg:
            json_exp = (space.opt,
                        (obj | array | string | true | false | null | number)
                        is exp, space.opt) >> exp

            pair = (string is k, ':', cut, json_exp is v) >> (k, v)
            obj = ('{', cut, pair is first, (',', pair is rest).rep, space.opt,
                   '}') >> dict([first] + rest)
            array = ('[', cut, json_exp is first, (',', json_exp is rest).rep,
                     space.opt, ']') >> [first] + rest

            string = (space.opt, '"', ('[^"]'.r | escape).rep //
                      ("".join) is body, '"') >> "".join(body)
            escape = '\\', ('"' | '/' | '\\' | 'b' | 'f' | 'n' | 'r' | 't'
                            | unicode_escape)
            unicode_escape = 'u', '[0-9A-Fa-f]'.r * 4

            true = 'true' >> True
            false = 'false' >> False
            null = 'null' >> None

            number = (minus.opt, integral, fractional.opt, exponent.opt) // f(
                float("".join(_)))
            minus = '-'
            integral = '0' | '[1-9][0-9]*'.r
            fractional = ('.', '[0-9]+'.r) // "".join
            exponent = (('e' | 'E'), ('+' | '-').opt, "[0-9]+".r) // "".join

            space = '\s+'.r

        # test Success
        test(number, "12031.33123E-2")
        test(string, '"i am a cow lol omfg"')
        test(array, '[1, 2, "omg", ["wtf", "bbq", 42]]')
        test(obj, '{"omg": "123", "wtf": 456, "bbq": "789"}')
        test(json_exp, '{"omg": 1, "wtf": 12.4123}  ')
        test(
            json_exp, """
            {
                "firstName": "John",
                "lastName": "Smith",
                "age": 25,
                "address": {
                    "streetAddress": "21 2nd Street",
                    "city": "New York",
                    "state": "NY",
                    "postalCode": 10021
                },
                "phoneNumbers": [
                    {
                        "type": "home",
                        "number": "212 555-1234"
                    },
                    {
                        "type": "fax",
                        "number": "646 555-4567"
                    }
                ]
            }
        """)

        # test Failure
        with self.assertRaises(ParseError) as e:
            json_exp.parse('{    : 1, "wtf": 12.4123}')

        assert e.exception.message ==\
"""
index: 5, line: 1, col: 6
json_exp / obj / pair / string
{    : 1, "wtf": 12.4123}
     ^
""".strip()

        with self.assertRaises(ParseError) as e:
            json_exp.parse('{"omg": "123", "wtf": , "bbq": "789"}')

        assert e.exception.message ==\
"""
index: 22, line: 1, col: 23
json_exp / obj / pair / json_exp
{"omg": "123", "wtf": , "bbq": "789"}
                      ^
""".strip()

        with self.assertRaises(ParseError) as e:
            json_exp.parse("""{
                    "firstName": "John",
                    "lastName": "Smith",
                    "age": 25,
                    "address": {
                        "streetAddress": "21 2nd Street",
                        "city": "New York",
                        "state": "NY",
                        "postalCode": 10021
                    },
                    "phoneNumbers": [
                        {
                            "type": "home",
                            "number": "212 555-1234"
                        },
                        {
                            "type": "fax",
                            "number": 646 555-4567"
                        }
                    ]
                }
            """)

        assert e.exception.message == \
"""
index: 655, line: 18, col: 43
json_exp / obj / pair / json_exp / array / json_exp / obj
                         "number": 646 555-4567"
                                       ^
""".strip()