def seq_comp_test(self, node_type, ends):
     gen = ast.comprehension(ast.Name('x', ast.Store()), ast.Name('y',
         ast.Load()), [ast.Num(2)])
     listcomp = node_type(ast.Name('w', ast.Load()), [gen])
     self.verify(listcomp, '{}w for x in y if 2{}'.format(*ends))
     gen2 = ast.comprehension(ast.Name('a', ast.Store()), ast.Name('b',
         ast.Load()), [])
     listcomp.generators.append(gen2)
     self.verify(listcomp, '{}w for x in y if 2 for a in b{}'.format(*ends))
     return listcomp
Example #2
0
 def seq_comp_test(self, node_type, ends):
     gen = ast.comprehension(ast.Name('x', ast.Store()),
                             ast.Name('y', ast.Load()), [ast.Num(2)])
     listcomp = node_type(ast.Name('w', ast.Load()), [gen])
     self.verify(listcomp, '{}w for x in y if 2{}'.format(*ends))
     gen2 = ast.comprehension(ast.Name('a', ast.Store()),
                              ast.Name('b', ast.Load()), [])
     listcomp.generators.append(gen2)
     self.verify(listcomp, '{}w for x in y if 2 for a in b{}'.format(*ends))
     return listcomp
Example #3
0
 def test_DictComp(self):
     gen = ast.comprehension(ast.Name('x', ast.Store()),
                             ast.Name('y', ast.Load()), [ast.Num(2)])
     dictcomp = ast.DictComp(ast.Name('v', ast.Load()),
                             ast.Name('w', ast.Load()), [gen])
     self.verify(dictcomp, '{v:w for x in y if 2}')
     gen2 = ast.comprehension(ast.Name('a', ast.Store()),
                              ast.Name('b', ast.Load()), [])
     dictcomp.generators.append(gen2)
     self.verify(dictcomp, '{v:w for x in y if 2 for a in b}')
 def test_DictComp(self):
     gen = ast.comprehension(ast.Name('x', ast.Store()), ast.Name('y',
         ast.Load()), [ast.Num(2)])
     dictcomp = ast.DictComp(ast.Name('v', ast.Load()), ast.Name('w',
         ast.Load()), [gen])
     self.verify(dictcomp, '{v:w for x in y if 2}')
     gen2 = ast.comprehension(ast.Name('a', ast.Store()), ast.Name('b',
         ast.Load()), [])
     dictcomp.generators.append(gen2)
     self.verify(dictcomp, '{v:w for x in y if 2 for a in b}')
Example #5
0
 def _check_comprehension(self, fac):
     self.expr(fac([]), "comprehension with no generators")
     g = ast.comprehension(ast.Name("x", ast.Load()), ast.Name("x", ast.Load()), [])
     self.expr(fac([g]), "must have Store context")
     g = ast.comprehension(ast.Name("x", ast.Store()), ast.Name("x", ast.Store()), [])
     self.expr(fac([g]), "must have Load context")
     x = ast.Name("x", ast.Store())
     y = ast.Name("y", ast.Load())
     g = ast.comprehension(x, y, [None])
     self.expr(fac([g]), "None disallowed")
     g = ast.comprehension(x, y, [ast.Name("x", ast.Store())])
     self.expr(fac([g]), "must have Load context")
Example #6
0
 def _check_comprehension(self, fac):
     self.expr(fac([]), "comprehension with no generators")
     g = ast.comprehension(ast.Name("x", ast.Load()),
                           ast.Name("x", ast.Load()), [], 0)
     self.expr(fac([g]), "must have Store context")
     g = ast.comprehension(ast.Name("x", ast.Store()),
                           ast.Name("x", ast.Store()), [], 0)
     self.expr(fac([g]), "must have Load context")
     x = ast.Name("x", ast.Store())
     y = ast.Name("y", ast.Load())
     g = ast.comprehension(x, y, [None], 0)
     self.expr(fac([g]), "None disallowed")
     g = ast.comprehension(x, y, [ast.Name("x", ast.Store())], 0)
     self.expr(fac([g]), "must have Load context")
Example #7
0
def _peval_comprehension_generators(state, generators, ctx):
    if len(generators) == 0:
        return state, []

    generator = generators[0]
    next_generators = generators[1:]

    state, iter_result = _peval_expression(state, generator.iter, ctx)

    masked_bindings = _get_masked_bindings(generator.target, ctx.bindings)
    masked_ctx = ctx.set('bindings', masked_bindings)

    state, ifs_result = _peval_comprehension_ifs(state, generator.ifs, masked_ctx)

    if is_known_value(ifs_result):
        success, bool_value = try_call(bool, args=(ifs_result.value,))
        if success and bool_value:
            ifs_result = []

    state, new_generator_kwds = map_reify(
        state, dict(target=generator.target, iter=iter_result, ifs=ifs_result))
    new_generator = ast.comprehension(**new_generator_kwds)

    state, new_generators = _peval_comprehension_generators(state, next_generators, ctx)

    return state, [new_generator] + new_generators
Example #8
0
def _generate_comprehension(max_depth=None):
    target = generate_variable(max_depth=max_depth - 1)
    iterable = generate_expression(max_depth=max_depth - 1)

    ifs = [generate_expression(max_depth=max_depth - 1)]

    return ast.comprehension(target=target, iter=iterable, ifs=ifs)
Example #9
0
    def parse(self):
        '''Convert the parsed tokens into a list of expressions
        Then join them'''
        steps = []
        self.stream = tokenise(self.source)
        for token in self.stream:
            code = self._token_to_code(token)
            if code is not None:
                steps.append(code)

        # result = [str(x) for x in steps]
        return ast.Module(
            body=[ast.Assign(
                targets=[ast.Name(id='result', ctx=ast.Store())],
                value=ast.ListComp(
                    elt=ast.Call(
                        func=ast.Name(id='str', ctx=ast.Load()),
                        args=[
                            ast.Name(id='x', ctx=ast.Load()),
                        ],
                        keywords=[],
                    ),
                    generators=[
                        ast.comprehension(
                            target=ast.Name(id='x', ctx=ast.Store()),
                            iter=ast.List(elts=steps, ctx=ast.Load()),
                            ifs=[]
                        )
                    ]
                )
            )
        ])
Example #10
0
File: compiler.py Project: koo5/hy
    def compile_list_comprehension(self, expr):
        # (list-comp expr (target iter) cond?)
        expr.pop(0)
        expression = expr.pop(0)
        tar_it = iter(expr.pop(0))
        targets = zip(tar_it, tar_it)

        cond = self.compile(expr.pop(0)) if expr != [] else None

        ret = ast.ListComp(
            lineno=expr.start_line,
            col_offset=expr.start_column,
            elt=self.compile(expression),
            generators=[])

        for target, iterable in targets:
            ret.generators.append(ast.comprehension(
                target=self._storeize(self.compile(target)),
                iter=self.compile(iterable),
                ifs=[]))

        if cond:
            ret.generators[-1].ifs.append(cond)

        return ret
Example #11
0
def For(var_in, body):
    gen = [ast.comprehension(Store(n.id), s, []) for (n, s) in var_in]
    ## Nested for-loops are elided together.
    if isinstance(body, ast.GeneratorExp):
        gen.extend(body.generators)
        body = body.elt
    return ast.GeneratorExp(body, gen)
Example #12
0
    def parse(self):
        """
        Convert the parsed tokens into a list of expressions then join them
        """
        self.stream = tokenise(self.source)
        steps = []
        for token in self.stream:
            code = self._token_to_code(token)
            if code:
                steps.append(code)

        # result = [str(x) for x in steps]
        return ast.Expression(
            body=ast.ListComp(
                elt=build_call(
                    ast.Name(id='str', ctx=ast.Load()),
                    args=[
                        ast.Name(id='x', ctx=ast.Load()),
                    ],
                ),
                generators=[
                    ast.comprehension(
                        target=ast.Name(id='x', ctx=ast.Store()),
                        iter=ast.List(elts=steps, ctx=ast.Load()),
                        ifs=[]
                    )
                ]
            )
        )
Example #13
0
 def _init_non_primitive_nested_class(node_assign, object_, prop):
     """
     If the nested list is non-primitive, initialise sub-classes in a list comp
     If the nest is primitive, we can simply get it
     Marshmallow will do the type marshalling
     """
     return ast.ListComp(
         elt=ast.Call(
             func=ast.Name(id=ObjectGenerator._nesting_class(node_assign)),
             args=[ast.Name(id="el")],
             keywords=[],
         ),
         generators=[
             ast.comprehension(
                 target=ast.Name(id="el"),
                 iter=ast.Call(
                     func=ast.Attribute(value=ast.Name(id=object_),
                                        attr="get"),
                     args=[ast.Str(s=prop),
                           ast.Dict(keys=[], values=[])],
                     keywords=[],
                 ),
                 ifs=[],
                 is_async=0,
             )
         ],
     )
Example #14
0
def For(var_in, body):
    gen = [ast.comprehension(Store(n.id), s, []) for (n, s) in var_in]
    ## Nested for-loops are elided together.
    if isinstance(body, ast.GeneratorExp):
        gen.extend(body.generators)
        body = body.elt
    return ast.GeneratorExp(body, gen)
Example #15
0
 def visit_Call(self, node):
     if node.func.id not in GRP_FUNCS:
         return node
     else:
         self.generic_visit(node)
         return ast.Call(
             func=node.func,
             args=[
                 ast.ListComp(
                     elt=node.args[0],
                     generators=[
                         ast.comprehension(
                             target=ast.Name(id="datarow", ctx=ast.Store(), lineno=0, col_offset=0),
                             iter=ast.Name(id="data", ctx=ast.Load(), lineno=0, col_offset=0),
                             ifs=[],
                             lineno=0,
                             col_offset=0
                         )
                     ],
                     lineno=0,
                     col_offset=0,
                 )
             ],
             keywords=[],
             ctx=ast.Load(),
             lineno=0,
             col_offset=0,
         )
Example #16
0
 def p_comp_async_for1(self, p):
     ''' comp_for : ASYNC FOR exprlist IN or_test '''
     target = p[3]
     self.set_context(target, Store, p)
     p[0] = [
         ast.comprehension(target=target, iter=p[5], ifs=[], is_async=1)
     ]
Example #17
0
 def test_async_comprehension(self):
     tree = ast.comprehension(target=ast.Name(id='a', ctx=ast.Store()),
                              iter=ast.Name(id='b', ctx=ast.Load()),
                              ifs=[],
                              is_async=True)
     expected = 'async for a in b'
     result = to_source(tree)
     self.assertEqual(expected, result)
Example #18
0
 def _simple_comp(self, fac):
     g = ast.comprehension(ast.Name("x", ast.Store()),
                           ast.Name("x", ast.Load()), [])
     self.expr(fac(ast.Name("x", ast.Store()), [g]),
               "must have Load context")
     def wrap(gens):
         return fac(ast.Name("x", ast.Store()), gens)
     self._check_comprehension(wrap)
 def handleComprehensions(self, comps, lineno, offset, *args):
     ccomps = []
 
     for c in comps:
         ccomps.append(ast.comprehension(target=self.dispatch(c.target, *args),
                                         iter=retic_ast.UseCheck(value=self.dispatch(c.iter), type=retic_ast.Subscriptable(), lineno=lineno, col_offset=offset),
                                         ifs=self.reduce(c.ifs, *args)))
     return ccomps, []
Example #20
0
 def _simple_comp(self, fac):
     g = ast.comprehension(ast.Name("x", ast.Store()),
                           ast.Name("x", ast.Load()), [], 0)
     self.expr(fac(ast.Name("x", ast.Store()), [g]),
               "must have Load context")
     def wrap(gens):
         return fac(ast.Name("x", ast.Store()), gens)
     self._check_comprehension(wrap)
def make_comprehension():
    """comprehension = (expr target, expr iter, expr* ifs, int is_async)"""

    return ast.comprehension(
        target=make_expression(ctx=ast.Store()),
        iter=ast.Name(id="iterable", ctx=ast.Load()),
        ifs=[],
        is_async=0,
    )
Example #22
0
    def visit_comp_for(self, values, ctx):
        """comp_for: 'for' exprlist 'in' or_test [comp_iter]"""
        if len(values) > 4:
            raise NotImplementedError("comp_for with more than 4 arguments not supported")

        node = ast.comprehension()
        node.target = self.visit(values[1], ast.Store())
        node.iter = self.visit(values[3], ctx)
        node.ifs = []
        
        return node
Example #23
0
    def generate(self, element:Element, GC:GenerationContext):


        acode = element.code

        if len(acode) is 2 and is_form(acode[1], "for"):

            for_form = acode[1].code

            # list comprehension

            # («[]» (for (in i lst) (f i))) # list compr

            in_el = for_form[1]
            in_el_code = in_el.code

            #with GC.let(domain=ExDom):

            assert is_identifier(in_el, "in")

            target_element = in_el_code[1]
            iter_element = in_el_code[2]

            with GC.let(domain=LVDom):
                target_code = GC.generate(target_element)

            with GC.let(domain=ExDom):
                iter_code = GC.generate(iter_element)



            generators = [ ast.comprehension(target=target_code,
                                             iter=iter_code,
                                             ifs=[]) ]


            to_evaluate_element = for_form[2]

            with GC.let(domain=ExDom):

                to_evaluate_code = GC.generate(to_evaluate_element)


            return ast.ListComp(to_evaluate_code, generators)


        else:

            els = self.generate_as_expressions(GC, *acode[1:])

            if GC.domain == LVDom:
                return ast.List(els, ast.Store())
            return expr_wrap(ast.List(els, ast.Load()), GC)
Example #24
0
 def handleComprehensions(self, comps, varchecks, *args):
     generators = []
     for comp in comps:
         iter = self.dispatch(comp.iter, varchecks, *args)
         target = self.dispatch(comp.target, varchecks, *args)
         
         vars = scope.WriteTargetFinder().preorder(target)
         varchecks = set.union(vars, varchecks)
         
         ifs = self.dispatch(comp.ifs, varchecks, *args)
         generators.append(ast.comprehension(target=target, iter=iter, ifs=ifs))
     return generators, varchecks
Example #25
0
    def test_dictcomp(self):
        g = ast.comprehension(ast.Name("y", ast.Store()), ast.Name("p", ast.Load()), [])
        c = ast.DictComp(ast.Name("x", ast.Store()), ast.Name("y", ast.Load()), [g])
        self.expr(c, "must have Load context")
        c = ast.DictComp(ast.Name("x", ast.Load()), ast.Name("y", ast.Store()), [g])
        self.expr(c, "must have Load context")

        def factory(comps):
            k = ast.Name("x", ast.Load())
            v = ast.Name("y", ast.Load())
            return ast.DictComp(k, v, comps)

        self._check_comprehension(factory)
    def handleComprehensions(self, comps, lineno, offset, *args):
        ccomps = []

        for c in comps:
            ccomps.append(
                ast.comprehension(target=self.dispatch(c.target, *args),
                                  iter=retic_ast.UseCheck(
                                      value=self.dispatch(c.iter),
                                      type=retic_ast.Subscriptable(),
                                      lineno=lineno,
                                      col_offset=offset),
                                  ifs=self.reduce(c.ifs, *args)))
        return ccomps, []
Example #27
0
 def p_comp_async_for2(self, p):
     ''' comp_for : ASYNC FOR exprlist IN or_test comp_iter '''
     target = p[3]
     self.set_context(target, Store, p)
     gens = []
     gens.append(ast.comprehension(target=target, iter=p[5], ifs=[],
                                   is_async=1))
     for item in p[6]:
         if isinstance(item, ast.comprehension):
             gens.append(item)
         else:
             gens[-1].ifs.append(item)
     p[0] = gens
    def visit_comprehension(self, node: comprehension, *args,
                            **kwargs) -> C.comprehension:
        target = self.visit(node.target, *args, **kwargs)
        iter = self.visit(node.iter, *args, **kwargs)
        ifs = self.visit(node.ifs, *args, **kwargs)
        is_async = self.visit(node.is_async, *args, **kwargs)

        return C.comprehension(
            target=target,
            iter=iter,
            ifs=ifs,
            is_async=is_async,
        )
Example #29
0
    def handleComprehensions(self, comps, varchecks, *args):
        generators = []
        for comp in comps:
            iter = self.dispatch(comp.iter, varchecks, *args)
            target = self.dispatch(comp.target, varchecks, *args)

            vars = scope.WriteTargetFinder().preorder(target)
            varchecks = set.union(vars, varchecks)

            ifs = self.dispatch(comp.ifs, varchecks, *args)
            generators.append(
                ast.comprehension(target=target, iter=iter, ifs=ifs))
        return generators, varchecks
Example #30
0
 def test_dictcomp(self):
     g = ast.comprehension(ast.Name("y", ast.Store()),
                           ast.Name("p", ast.Load()), [], 0)
     c = ast.DictComp(ast.Name("x", ast.Store()),
                      ast.Name("y", ast.Load()), [g])
     self.expr(c, "must have Load context")
     c = ast.DictComp(ast.Name("x", ast.Load()),
                      ast.Name("y", ast.Store()), [g])
     self.expr(c, "must have Load context")
     def factory(comps):
         k = ast.Name("x", ast.Load())
         v = ast.Name("y", ast.Load())
         return ast.DictComp(k, v, comps)
     self._check_comprehension(factory)
Example #31
0
 def init(vars):
     node = Name(id=Log.log_dict, ctx=Store())
     var_strs = [Str(s=var.name) for var in vars]
     void = Call(func=Name(id='Void', ctx=Load()))
     return [
         Assign(targets=[node],
                value=DictComp(key=Name(id='__key', ctx=Load()),
                               value=Log.__fromkeys(var_strs, void),
                               generators=[
                                   comprehension(target=Name(id='__key',
                                                             ctx=Store()),
                                                 iter=Log.__fromkeys(
                                                     Log.sections))
                               ]))
     ]
 def compile_expr(self, target_type):
     assert target_type is not None
     dims = self._compile_dims(target_type)
     ltype = self._lambda_type(target_type)
     
     #def array(func,dims):
     #    return [func(*d) for d in itertools.product(*(map(range,dims))]
     elt_expr = _pyast.Call(self.genexpr.compile_expr(ltype), [], [], _pyast.Name('_d', _pyast.Load()), None)                  # func(*d)
     # elt_expr = _pyast.Call(_pyast.Name('tuple', _pyast.Load()), [_pyast.Name('_d', _pyast.Load()), elt_expr], [], None, None) # tuple(d, func(*d))
     pdt_expr = _pyast.Attribute(_pyast.Name('_pyitertools', _pyast.Load()), 'product', _pyast.Load())                            # itertools.product
     itr_expr = _pyast.Call(_pyast.Name('map', _pyast.Load()), [_pyast.Name('range', _pyast.Load()), dims], [], None, None)    # map(range,dims)
     itr_expr = _pyast.Call(pdt_expr, [], [], itr_expr, None)                                                                  # itertools.product(*(map(range,dims)))
     return _pyast.ListComp(
         elt_expr,
         [_pyast.comprehension(_pyast.Name('_d', _pyast.Store()), itr_expr, [])])
Example #33
0
def genProcessCollectionToList(
        funcName: str,
        parentName: str,
        iterFuncName: str,
        processorFunc: ast.Attribute,
        firstArgName: str = "parsed",
        iterVarName: str = "f",
        returnType: None = None) -> typing.Iterator[ast.FunctionDef]:
    """Generates a function transforming a collection (`min`) or a macro around it into a python `list` of nodes"""

    o = ast.Name(id=firstArgName, ctx=ast.Load())
    yield ast.FunctionDef(
        name=funcName,
        args=ast.arguments(
            posonlyargs=[],
            args=[
                astSelfArg,
                ast.arg(arg=firstArgName, annotation=None, type_comment=None),
            ],
            vararg=None,
            kwonlyargs=[],
            kw_defaults=[],
            kwarg=None,
            defaults=[],
        ),
        body=[
            ast.Return(value=ast.ListComp(
                elt=ast.Call(
                    func=processorFunc,
                    args=[ast.Name(id=iterVarName)],
                    keywords=[],
                ),
                generators=[
                    ast.comprehension(
                        target=ast.Name(id=iterVarName),
                        iter=ast.Call(func=ast.Attribute(
                            value=ASTSelf, attr=iterFuncName, ctx=ast.Load()),
                                      args=[o],
                                      keywords=[]),
                        ifs=[],
                        is_async=0)
                ],
            ))
        ],
        decorator_list=[],
        returns=(genTypingIterable(returnType) if returnType else None),
        type_comment=None,
    )
Example #34
0
    def visit_comp(self, node):
        """ Find all functions that are called multiple times with the same
         arguments as we will replace them with one variable
        """
        call_visitor = DuplicateCallFinder()
        call_visitor.visit(node)

        # Keep track of what calls we need to replace using a stack so we
        # support nested comprehensions
        self.calls_to_replace_stack.append(call_visitor.duplicate_calls)

        # Visit children of this list comprehension and replace calls
        self.generic_visit(node)

        # Gather the existing if statements as we need to move them to the
        # last comprehension generator (or there will be issues looking up
        # identifiers)
        existing_ifs = []
        for generator in node.generators:
            existing_ifs += generator.ifs
            generator.ifs = []

        # Create a new for loop for each function call result that we want
        # to alias and add it to the list comprehension
        for call in call_visitor.duplicate_calls:
            new_comprehension = comprehension(
                # Notice that we're storing (Store) the result of the call
                # instead of loading it (Load)
                target=Name(
                    id=OptimizeComprehensions._identifier_from_Call(call),
                    ctx=Store()),
                iter=List(elts=[call], ctx=Load()),
                ifs=[],
                is_async=0,
            )
            # Add linenos and other things the compile needs to node
            fix_missing_locations(new_comprehension)
            node.generators.append(new_comprehension)

        node.generators[-1].ifs = existing_ifs

        # Make sure we clear the calls to replace so we don't replace other
        # calls outside of the scope of this current list comprehension
        self.calls_to_replace_stack.pop()
        return node
Example #35
0
File: astx.py Project: cyrus-/typy
def expr_Raise_Exception_string(message):
    # hack: (_ for _ in ()).throw(Exception(message))
    return ast.Call(
        func=ast.Attribute(
            value=ast.GeneratorExp(
                elt=ast.Name(id='_', ctx=ast.Load()),
                generators=[
                    ast.comprehension(
                        target=ast.Name(id='_', ctx=ast.Store()),
                        iter=ast.Tuple(
                            elts=[], 
                            ctx=ast.Load()), 
                        ifs=[])]), 
            attr='throw', ctx=ast.Load()),
        args=[builtin_call('Exception', [ast.Str(s=message)])],
        keywords=[],
        starargs=None,
        kwargs=None)
Example #36
0
def handle_str(node):
    """Replaces an ast.Constant containing a string, with an ascii representation joined together

    Args:
        **node (:obj: `ast.Constant`)**: The node to replace with the ast.Call node

    Returns:
        A new obscured node with updated locations
    """
    if type(node.parent) != ast.FormattedValue and type(
            node.parent.parent) != ast.FormattedValue and type(
                node.parent) != ast.JoinedStr and type(
                    node.parent.parent) != ast.JoinedStr:
        new_node = ast.Call(
            func=ast.Attribute(
                value=ast.Constant(value='', kind=None),
                attr='join',
                ctx=ast.Load(),
            ),
            args=[
                ast.ListComp(
                    elt=ast.Call(func=ast.Name(id='chr', ctx=ast.Load()),
                                 args=[ast.Name(id='x', ctx=ast.Load())],
                                 keywords=[]),
                    generators=[
                        ast.comprehension(
                            target=ast.Name(id='x', ctx=ast.Store()),
                            iter=ast.List(
                                elts=[
                                    ast.Constant(value=ord(x), kind=None)
                                    for x in node.value
                                ],
                                ctx=ast.Load(),
                            ),
                            ifs=[],
                            is_async=0,
                        ),
                    ],
                ),
            ],
            keywords=[])
        return ast.copy_location(new_node, node)
    else:
        return node
Example #37
0
def makeGroup(name, base_score, useNumPy=None):
    groupTemp = ast.Subscript(value=treesAccumulator,
                              slice=ast.Index(value=ast.Name(name)))
    code = []
    res = ast.Name(id="res")
    baseScore = astNum(n=base_score)
    treeCall = ast.Call(func=treeTemp, args=[vector], keywords=[])
    iterArgDict = {
        "target": treeTemp,
        "iter": groupTemp,
    }

    if not useNumPy:
        code.append(ast.Assign(targets=[res], value=baseScore))
        code.append(
            ast.For(
                body=[ast.AugAssign(target=res, op=ast.Add(), value=treeCall)],
                orelse=[],
                **iterArgDict,
            ))
    else:
        res = ast.BinOp(
            left=baseScore,
            op=ast.Add(),
            right=ast.Call(
                func=numpySum,
                args=[
                    ast.ListComp(
                        elt=treeCall,
                        generators=[
                            ast.comprehension(
                                ifs=[],
                                **iterArgDict,
                            )
                        ],
                    )
                ],
                keywords=[],
            ),
        )
    code.append(ast.Return(res))
    return wrapWithFunc(code, name)
Example #38
0
 def enums_helper(self, enums):
     generators = []
     
     for enum in enums:
         
         if dka.hasnodetype(enum, dha.EnumFor):
             target = self.visit(enum.target)
             iter = self.visit(enum.iter)
             generators.append(ast.comprehension(target, iter, []))
         
         elif dka.hasnodetype(enum, dha.EnumIf):
             assert(len(generators) > 0)
             target = self.visit(enum.target)
             iter = self.visit(enum.iter)
             generators[-1].ifs.append(
                 ast.Compare(target, [ast.In()], [iter]))
         
         else: assert()
     
     return generators
Example #39
0
def _mutual_equality_for(a, b, equal):
  if len(a.elts) > len(b.elts):
    a, b = b, a
  if equal:
    op = ast.Eq()
  else:
    op = ast.NotEq()
  return ast.For(
      target=ast.Name(id='__x', ctx=_STORE),
      iter=a,
      body=[
        ast.Expr(value=ast.Compare(
            left=ast.Call(
                func=ast.Name(id='sum', ctx=_LOAD),
                args=[
                  ast.GeneratorExp(
                      elt=ast.Subscript(
                          value=ast.Name(id='__x', ctx=_LOAD),
                          slice=ast.Index(
                              value=ast.Name(id='__y', ctx=_LOAD),
                          ),
                          ctx=_LOAD,
                      ),
                      generators=[
                        ast.comprehension(
                            target=ast.Name(id='__y', ctx=_STORE),
                            iter=b,
                            ifs=[],
                            is_async=0,
                        ),
                      ],
                  ),
                ],
                keywords=[],
            ),
            ops=[op],
            comparators=[ast.Num(n=1)],
        )),
      ],
      orelse=[],
  )
Example #40
0
    def compile_list_comprehension(self, expr):
        # (list-comp expr (target iter) cond?)
        expr.pop(0)
        expression = expr.pop(0)
        tar_it = iter(expr.pop(0))
        targets = zip(tar_it, tar_it)

        cond = self.compile(expr.pop(0)) if expr != [] else None

        ret = ast.ListComp(lineno=expr.start_line,
                           col_offset=expr.start_column,
                           elt=self.compile(expression),
                           generators=[])

        for target, iterable in targets:
            ret.generators.append(
                ast.comprehension(target=self._storeize(self.compile(target)),
                                  iter=self.compile(iterable),
                                  ifs=[]))

        if cond:
            ret.generators[-1].ifs.append(cond)

        return ret
Example #41
0
    def generate(self, element:Element, GC:GenerationContext):


        acode = element.code

        head = acode[0].code
        assert isinstance(head, Identifier)
        headtext = head.full_name


        if len(acode) is 2 and is_form(acode[1].code, "for"):


            ccode = acode[1].code
            assert len(ccode) == 3

            target_iter_element = ccode[1]
            expr_element = ccode[2]

            with GC.let(domain=LVDom):
                target_code = GC.generate(target_iter_element.code[1])

            with GC.let(domain=ExDom):
                iter_code = GC.generate(target_iter_element.code[2])

                comp_code = ast.comprehension(target=target_code,
                                              iter=iter_code,
                                              ifs=[])

                if is_form(expr_element.code, "="):
                    # dict comp
                    key_code = GC.generate(expr_element.code[1])
                    value_code = GC.generate(expr_element.code[2])

                    return ast.DictComp(key=key_code, value=value_code,
                                        generators=[comp_code])

                else:
                    # set comp
                    elt_code = GC.generate(expr_element)

                    return ast.SetComp(elt=elt_code, generators=[comp_code])


        else:

            if len(acode) is 1:

                return ast.Dict([], [])


            if all([is_form(i.code, "=") for i in acode[1:]]):

                #dict

                keys = []
                values = []

                with GC.let(domain=ExDom):

                    for kvpair_el in acode[1:]:

                        kvpair = kvpair_el.code

                        key_code = GC.generate(kvpair[1])
                        value_code = GC.generate(kvpair[2])

                        keys.append(key_code)
                        values.append(value_code)

                return expr_wrap(ast.Dict(keys, values), GC)

            else:

                #set

                el_codes = self.generate_as_expressions(GC, *acode[1:])

                return expr_wrap(ast.Set(el_codes), GC)
Example #42
0
        def visit_FunctionDef(self, node: FunctionDef):
            nonlocal function_name, context
            if node is not function_head:
                return node
            function_body = []
            function_name = node.name
            function_args = [arg.arg for arg in node.args.args]
            inlined_start = 1

            if inspect.ismethod(method):
                inlined_start += 1

            iterator_name = function_args[inlined_start - 1]
            function_args[:inlined_start] = []
            arity = len(function_args)

            try:
                vararg = as_arg(node.args.vararg.arg)
            except Exception:
                if arity != len(inline_args):
                    raise ArgumentCountMismatch
            else:
                context[vararg] = inline_args[arity:]

            for name, value in zip(function_args, inline_args):
                targets = [Name(id=as_var(name), ctx=Store())]
                if isinstance(value, PassAsConstant):
                    context[as_var(name)] = value.value
                    continue
                if isinstance(value, (int, str, bytes)):
                    context[as_var(name)] = value
                    continue
                context[as_arg(name)] = value
                function_body.append(
                    Assign(targets=targets,
                           value=Call(func=Name(id='next', ctx=Load()),
                                      args=[Name(id=as_arg(name), ctx=Load())],
                                      keywords=[])))

            if node.args.vararg:
                name = node.args.vararg.arg
                function_body.append(
                    Assign(targets=[Name(id=as_var(name), ctx=Store())],
                           value=Call(
                               func=Name(id='tuple', ctx=Load()),
                               args=[
                                   GeneratorExp(
                                       elt=Call(func=Name(id='next',
                                                          ctx=Load()),
                                                args=[
                                                    Name(id=as_tmp(name),
                                                         ctx=Load())
                                                ],
                                                keywords=[]),
                                       generators=[
                                           comprehension(
                                               is_async=0,
                                               target=Name(id=as_tmp(name),
                                                           ctx=Store()),
                                               iter=Name(id=as_arg(name),
                                                         ctx=Load()),
                                               ifs=[])
                                       ])
                               ],
                               keywords=[])))

            function_body.extend(node.body)
            context[as_arg(iterator_name)] = iterator
            function_body = [
                For(target=Name(id=as_var(iterator_name), ctx=Store()),
                    iter=Name(id=as_arg(iterator_name), ctx=Load()),
                    body=function_body,
                    orelse=[])
            ]

            node.body = function_body
            node.args.args = [arg(arg=as_var('self'))]
            node.args.vararg = None
            node.decorator_list = []
            return node
Example #43
0
 def comprehension(target, iter, ifs, is_async=0):
     return _ast.comprehension(target, iter, ifs, is_async)
    def visit_Call(self, call_node):
        func_node = call_node.func
        if isinstance(func_node, ast.Name) and func_node.id == "isinstance":
            # original code was:
            # assert isinstance(obj, cls_or_tuple)
            #
            # generated code is:
            #
            # @contexts_assertion_var1 = obj
            # @contexts_assertion_var2 = cls_or_tuple
            # @contexts_assertion_var3 = obj.__class__
            # @contexts_assertion_var4 = (tuple(@x.__name__ for @x in @contexts_assertion_var2)
            #                             if isinstance(@contexts_assertion_var2, tuple)
            #                             else @contexts_assertion_var2.__name__)
            # assert isinstance(@contexts_assertion_var1, @contexts_assertion_var2), 'Asserted isinstance({0}, {1}) but found it to be a {2}'.format(@contexts_assertion_var1, repr(@contexts_assertion_var4).replace("'", ""), @contexts_assertion_var3)
            if sys.version_info < (3, 6):
                tupleAstArgs = ast.comprehension(ast.Name('@x', ast.Store()), self.load('@contexts_assertion_var2'), [])
            else:
                tupleAstArgs = ast.comprehension(ast.Name('@x', ast.Store()), self.load('@contexts_assertion_var2'), [], False)

            return [
                self.assign('@contexts_assertion_var1', call_node.args[0]),
                self.assign('@contexts_assertion_var2', call_node.args[1]),
                self.assign('@contexts_assertion_var3', self.clsname(self.getattr(self.load('@contexts_assertion_var1'), '__class__'))),
                self.assign('@contexts_assertion_var4', ast.IfExp(
                    ast.Call(func=self.load('isinstance'), args=[
                        self.load('@contexts_assertion_var2'),
                        self.load('tuple'),
                    ], keywords=[]),

                    ast.Call(func=self.load('tuple'), args=[
                        ast.GeneratorExp(self.clsname(self.load('@x')), [
                            tupleAstArgs,
                        ]),
                    ], keywords=[]),
                    self.clsname(self.load('@contexts_assertion_var2'))
                )),
                ast.Assert(
                    ast.Call(func=ast.Name(id='isinstance', ctx=ast.Load()), args=[
                        self.load('@contexts_assertion_var1'),
                        self.load('@contexts_assertion_var2'),
                    ], keywords=[]),
                    self.format('Asserted isinstance({0}, {1}) but found it to be a {2}', [
                        self.repr(self.load('@contexts_assertion_var1')),
                        ast.Call(func=self.getattr(self.repr(self.load('@contexts_assertion_var4')), 'replace'), args=[ast.Str("'"), ast.Str("")], keywords=[]),
                        self.load('@contexts_assertion_var3'),
                    ]))
            ]
        if isinstance(func_node, ast.Name) and func_node.id == "all":
            # original code was:
            #
            # assert all(iterable)
            #
            # generated code is:
            #
            # @contexts_assertion_var1 = iterable
            # for @contexts_assertion_var_ix, @contexts_assertion_var_elem in enumerate(@contexts_assertion_var1):
            #     assert x, "Not all elements of {} were truthy. First falsy element: {} at position {}".format(@contexts_assertion_var1, @contexts_assertion_var_ix, @contexts_assertion_var_elem)
            return [
                self.assign('@contexts_assertion_var1', call_node.args[0]),
                ast.For(ast.Tuple([ast.Name("@contexts_assertion_var_ix", ast.Store()),
                                   ast.Name("@contexts_assertion_var_elem", ast.Store())], ast.Store()), ast.Call(func=self.load("enumerate"), args=[self.load('@contexts_assertion_var1')], keywords=[]), [
                    ast.Assert(
                        self.load('@contexts_assertion_var_elem'),
                        self.format("Not all elements of {0} were truthy. First falsy element: {1} at position {2}", [
                            self.repr(self.load('@contexts_assertion_var1')),
                            self.repr(self.load('@contexts_assertion_var_elem')),
                            self.load('@contexts_assertion_var_ix'),
                        ]))
                ], [])
            ]
Example #45
0
 def test_comprehension(self):
     comp = ast.comprehension(ast.Name('x', ast.Store()),
                              ast.Name('y', ast.Load()), [])
     self.verify(comp, 'for x in y')
     comp.ifs = [ast.Num(2), ast.Num(3)]
     self.verify(comp, 'for x in y if 2 if 3')
Example #46
0
def makeRegressor(model: Xgboost,
                  useSigmoids: bool = False,
                  useNumPy: bool = False) -> ast.Module:
    """Converts a regression XGBoost decision tree model into a python module of if-else trees"""
    #code:typing.List[ast.stmt]=[]
    code = []
    numpyModuleName = "numpy"
    if useSigmoids:
        code.extend(
            generateFermiSplitFunction(
                sigmoidSplitFuncName,
                tanhModuleName=(numpyModuleName if useNumPy else "math")))

    if useNumPy:
        code.append(
            ast.ImportFrom(module=numpyModuleName,
                           names=[ast.alias(name=numpySumName, asname=None)],
                           level=0))

    features = int(model.param.num_feature)
    trees = int(model.gbm_.param.num_trees)
    groups = int(model.gbm_.param.num_output_group)
    if groups == 0:
        groups = 1

    treesInAGroup = trees // groups

    groupFunctions = []
    groupFunctionsNames = []
    elts = []
    for j in range(groups):
        group = []
        for i in range(treesInAGroup):
            t = model.gbm_.trees[j * treesInAGroup + i]
            name = "g" + str(j) + "_t" + str(i)
            code.extend(treesToFuncs(t, name, useSigmoids))
            group.append(ast.Name(id=name))
        elts.append(ast.Tuple(elts=group))
        groupFuncName = "g" + str(j)
        groupFunctionsNames.append(ast.Name(groupFuncName))
        groupFunctions.append(
            makeGroup(groupFuncName, model.param.base_score,
                      useNumPy=useNumPy))

    code.extend(groupFunctions)
    code.append(
        ast.Assign(targets=[treesAccumulator],
                   value=ast.Dict(keys=groupFunctionsNames, values=elts)))

    code.append(
        wrapWithFunc([
            ast.Expr(value=astStr(s=genModelSummary(model))),
            ast.Assert(test=ast.Compare(left=ast.Call(
                func=lenFunc, args=[vector], keywords=[]),
                                        ops=[ast.Eq()],
                                        comparators=[astNum(n=features)]),
                       msg=ast.BinOp(left=ast.BinOp(
                           left=astStr(s="This model requires exactly " +
                                       str(features) + " feature, but "),
                           op=ast.Add(),
                           right=ast.Call(func=strFunc,
                                          args=[
                                              ast.Call(func=lenFunc,
                                                       args=[vector],
                                                       keywords=[])
                                          ],
                                          keywords=[])),
                                     op=ast.Add(),
                                     right=astStr(s=" were given"))),
            ast.Assign(
                targets=[res],
                value=ast.ListComp(
                    elt=ast.Call(func=groupTemp, args=[vector], keywords=[]),
                    generators=[
                        ast.comprehension(
                            target=groupTemp, iter=treesAccumulator, ifs=[])
                    ])),
            ast.Return(value=res)
        ], regressorFunctionName))

    return ast.Module(body=code)
Example #47
0
 def p_comp_async_for1(self, p):
     ''' comp_for : ASYNC FOR exprlist IN or_test '''
     target = p[3]
     self.set_context(target, Store, p)
     p[0] = [ast.comprehension(target=target, iter=p[5], ifs=[],
                               is_async=1)]
Example #48
0
def kompile(src, debug=False):
    '''
    Creates a new class based on the supplied template, and returnsit.

    class Template(object):
        def __call__(self, context):
            return ''.join(str(x) for x in self._root(context))

        def _root(self, context):
            yield ''
            yield ...
            yield from self.head(context)

    Blocks create new methods, and add a 'yield from self.{block}(context)' to
    the current function

    '''

    parser = Parser(src)
    parser.load_library('knights.tags')
    parser.load_library('knights.helpers')

    # Define the __call__ method
    # return ''.join(str(x) for x in self._root(context))
    func = ast.FunctionDef(
        name='__call__',
        args=ast.arguments(
            args=[
                ast.arg(arg='self', annotation=None),
                ast.arg(arg='context', annotation=None),
            ],
            vararg=None,
            kwonlyargs=[],
            kwarg=None,
            defaults=[],
            kw_defaults=[],
        ),
        body=[
            ast.Return(
                value=ast.Call(
                    func=ast.Attribute(
                        value=ast.Str(s=''),
                        attr='join',
                        ctx=ast.Load()
                    ),
                    args=[
                        ast.GeneratorExp(
                            elt=ast.Call(
                                func=ast.Name(id='str', ctx=ast.Load()),
                                args=[
                                    ast.Name(id='x', ctx=ast.Load()),
                                ],
                                keywords=[], starargs=None, kwargs=None
                            ),
                            generators=[
                                ast.comprehension(
                                    target=ast.Name(id='x', ctx=ast.Store()),
                                    iter=ast.Call(
                                        func=ast.Attribute(
                                            value=ast.Name(id='self', ctx=ast.Load()),
                                            attr='_root',
                                            ctx=ast.Load()
                                        ),
                                        args=[
                                            ast.Name(id='context', ctx=ast.Load()),
                                        ], keywords=[], starargs=None, kwargs=None
                                    ),
                                    ifs=[]
                                ),
                            ]
                        ),
                    ],
                    keywords=[], starargs=None, kwargs=None
                )
            ),
        ],
        decorator_list=[],
    )

    parser.methods.append(func)
    parser.build_method('_root')

    if parser.parent:
        # Remove _root from the method list
        parser.methods = [
            method for method in parser.methods if method.name != '_root'
        ]
    klass = parser.build_class()

    # Wrap it in a module
    inst = ast.Module(body=[klass])

    ast.fix_missing_locations(inst)

    if debug:
        import astpp
        print(astpp.dump(inst))
    # Compile code to create class
    code = compile(inst, filename='<compiler>', mode='exec', optimize=2)

    # Execute it and return the instance
    g = {
        'helpers': parser.helpers,
        'parent': parser.parent,
        'ContextScope': ContextScope,
    }
    eval(code, g)

    return g['Template']
                [ast.ImportFrom("__future__", names=[ast.alias(name="lol")])]),
            "'lol' is not defined",
        ),
        (
            ast.Assign(targets=[ast.Starred(), ast.Starred()]),
            "More then one starred expression",
        ),
        (
            ast.Assign(targets=([ast.Name("i")] * 500 + [ast.Starred()])),
            "Too many expressions",
        ),
        (
            ast.Assign(targets=([ast.Name("i")] * 500 + [ast.Starred()])),
            "Too many expressions",
        ),
        (ast.comprehension(is_async=1), "inside of a coroutine"),
        (ast.Yield(), "inside of a function"),
        (
            ast.AsyncFunctionDef(body=[ast.YieldFrom()]),
            "can't be used in a coroutine",
        ),
        (ast.Await(), "inside of a function"),
        (ast.FunctionDef(body=[ast.Await()]), "inside of a coroutine"),
    ],
)
def test_simple_ast_validator(node, message):
    validator = ContextualASTValidator()
    with pytest.raises(SyntaxError) as cm:
        validator.validate(ast.fix_missing_locations(node))
    assert cm.match(message)
Example #50
0
    def test_empty_init(self):
        # Jython 2.5.0 did not allow empty constructors for many ast node types
        # but CPython ast nodes do allow this.  For the moment, I don't see a
        # reason to allow construction of the super types (like ast.AST and
        # ast.stmt) as well as the op types that are implemented as enums in
        # Jython (like boolop), but I've left them in but commented out for
        # now.  We may need them in the future since CPython allows this, but
        # it may fall under implementation detail.

        #ast.AST()
        ast.Add()
        ast.And()
        ast.Assert()
        ast.Assign()
        ast.Attribute()
        ast.AugAssign()
        ast.AugLoad()
        ast.AugStore()
        ast.BinOp()
        ast.BitAnd()
        ast.BitOr()
        ast.BitXor()
        ast.BoolOp()
        ast.Break()
        ast.Call()
        ast.ClassDef()
        ast.Compare()
        ast.Continue()
        ast.Del()
        ast.Delete()
        ast.Dict()
        ast.Div()
        ast.Ellipsis()
        ast.Eq()
        ast.Exec()
        ast.Expr()
        ast.Expression()
        ast.ExtSlice()
        ast.FloorDiv()
        ast.For()
        ast.FunctionDef()
        ast.GeneratorExp()
        ast.Global()
        ast.Gt()
        ast.GtE()
        ast.If()
        ast.IfExp()
        ast.Import()
        ast.ImportFrom()
        ast.In()
        ast.Index()
        ast.Interactive()
        ast.Invert()
        ast.Is()
        ast.IsNot()
        ast.LShift()
        ast.Lambda()
        ast.List()
        ast.ListComp()
        ast.Load()
        ast.Lt()
        ast.LtE()
        ast.Mod()
        ast.Module()
        ast.Mult()
        ast.Name()
        ast.Not()
        ast.NotEq()
        ast.NotIn()
        ast.Num()
        ast.Or()
        ast.Param()
        ast.Pass()
        ast.Pow()
        ast.Print()
        ast.RShift()
        ast.Raise()
        ast.Repr()
        ast.Return()
        ast.Slice()
        ast.Store()
        ast.Str()
        ast.Sub()
        ast.Subscript()
        ast.Suite()
        ast.TryExcept()
        ast.TryFinally()
        ast.Tuple()
        ast.UAdd()
        ast.USub()
        ast.UnaryOp()
        ast.While()
        ast.With()
        ast.Yield()
        ast.alias()
        ast.arguments()
        #ast.boolop()
        #ast.cmpop()
        ast.comprehension()
        #ast.excepthandler()
        #ast.expr()
        #ast.expr_context()
        ast.keyword()
Example #51
0
 def make_base_class_dict_test_stmts(
     self, bases: List[TAst], instance_fields: Set[str]
 ) -> stmt:
     slots_stmt = self.make_slots_stmt(instance_fields)
     # if there are non-names in the bases of the class, give up and just create slots
     if not all(isinstance(b, ast.Name) for b in bases):
         return slots_stmt
     # if __dict__ is not added to instance fields (no loose slots), just slotify
     if "__dict__" not in instance_fields:
         return slots_stmt
     # generate code that decide whether __dict__ should be included
     # if any('__dict__' in getattr(_t, '__dict__', ()) for b in <bases> for _t in b.mro()):
     #   __slots__ = <slots without __dict__>
     # else:
     #   __slots__ = <slots with __dict__>
     names = [lineinfo(ast.Name(cast(ast.Name, n).id, ast.Load())) for n in bases]
     condition = lineinfo(
         ast.Call(
             lineinfo(ast.Name("any", ast.Load())),
             [
                 lineinfo(
                     ast.GeneratorExp(
                         lineinfo(
                             ast.Compare(
                                 lineinfo(Constant("__dict__")),
                                 [lineinfo(ast.In())],
                                 [
                                     lineinfo(
                                         ast.Call(
                                             lineinfo(
                                                 ast.Name("getattr", ast.Load())
                                             ),
                                             [
                                                 lineinfo(
                                                     ast.Name("_t", ast.Load())
                                                 ),
                                                 lineinfo(Constant("__dict__")),
                                                 lineinfo(ast.Tuple([], ast.Load())),
                                             ],
                                             [],
                                         )
                                     )
                                 ],
                             )
                         ),
                         [
                             lineinfo(
                                 ast.comprehension(
                                     lineinfo(ast.Name("b", ast.Store())),
                                     lineinfo(ast.List(names, ast.Load())),
                                     [],
                                     0,
                                 )
                             ),
                             lineinfo(
                                 ast.comprehension(
                                     lineinfo(ast.Name("_t", ast.Store())),
                                     lineinfo(
                                         ast.Call(
                                             lineinfo(
                                                 ast.Attribute(
                                                     lineinfo(
                                                         ast.Name("b", ast.Load())
                                                     ),
                                                     "mro",
                                                     ast.Load(),
                                                 )
                                             ),
                                             [],
                                             [],
                                         )
                                     ),
                                     [],
                                     0,
                                 )
                             ),
                         ],
                     )
                 )
             ],
             [],
         )
     )
     slots_stmt_without_dict = self.make_slots_stmt(instance_fields - {"__dict__"})
     return lineinfo(ast.If(condition, [slots_stmt_without_dict], [slots_stmt]))
Example #52
0
	def init(vars):
		node = Name(id=Log.log_dict, ctx=Store())
		var_strs = [Str(s=var.name) for var in vars]
		void = Call(func=Name(id='Void', ctx=Load()))
		return [Assign(targets=[node], value=DictComp(key=Name(id='__key', ctx=Load()), value=Log.__fromkeys(var_strs,void), generators=[comprehension(target=Name(id='__key', ctx=Store()), iter=Log.__fromkeys(Log.sections))]))]
Example #53
0
 def visitcomprehension(self, n, *args):
     iter = self.dispatch(n.iter, *args)
     ifs = self.reduce(n.ifs, *args)
     target = self.dispatch(n.target, *args)
     return ast.comprehension(target=target, iter=iter, ifs=ifs)
Example #54
0
    def test_operators(self):
        boolop0 = ast.BoolOp()
        boolop1 = ast.BoolOp(ast.And(), 
                          [ ast.Name('True', ast.Load()), ast.Name('False', ast.Load()), ast.Name('a',ast.Load())])
        boolop2 = ast.BoolOp(ast.And(), 
                          [ ast.Name('True', ast.Load()), ast.Name('False', ast.Load()), ast.Name('a',ast.Load())],
                          0, 0)
        binop0 = ast.BinOp()
        binop1 = ast.BinOp(ast.Str('xy'), ast.Mult(), ast.Num(3))
        binop2 = ast.BinOp(ast.Str('xy'), ast.Mult(), ast.Num(3), 0, 0)

        unaryop0 = ast.UnaryOp()
        unaryop1 = ast.UnaryOp(ast.Not(), ast.Name('True',ast.Load())) 
        unaryop2 = ast.UnaryOp(ast.Not(), ast.Name('True',ast.Load()), 0, 0)

        lambda0 = ast.Lambda()
        lambda1 = ast.Lambda(ast.arguments([ast.Name('x', ast.Param())], None, None, []), ast.Name('x', ast.Load()))
        
        ifexp0 = ast.IfExp()
        ifexp1 = ast.IfExp(ast.Name('True',ast.Load()), ast.Num(1), ast.Num(0))
        ifexp2 = ast.IfExp(ast.Name('True',ast.Load()), ast.Num(1), ast.Num(0), 0, 0)

        dict0 = ast.Dict()
        dict1 = ast.Dict([ast.Num(1), ast.Num(2)], [ast.Str('a'), ast.Str('b')])
        dict2 = ast.Dict([ast.Num(1), ast.Num(2)], [ast.Str('a'), ast.Str('b')], 0, 0)

        set0 = ast.Set()
        set1 = ast.Set([ast.Num(1), ast.Num(2)])
        set2 = ast.Set([ast.Num(1), ast.Num(2)], 0, 0)

        lc0 = ast.ListComp()
        lc1 = ast.ListComp( ast.Name('x',ast.Load()), 
                   [ast.comprehension(ast.Name('x', ast.Store()), 
                                      ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])])
        lc2 = ast.ListComp( ast.Name('x',ast.Load()), 
                   [ast.comprehension(ast.Name('x', ast.Store()), 
                                      ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])], 0, 0)


        setcomp0 = ast.SetComp()
        setcomp1 = ast.SetComp(ast.Name('x', ast.Load()), 
                   [ast.comprehension(ast.Name('x', ast.Store()), ast.Str('abracadabra'), 
                                      [ast.Compare(ast.Name('x', ast.Load()), [ast.NotIn()], 
                                                   [ast.Str('abc')])])])


        comprehension0 = ast.comprehension()
        comprehension1 = ast.comprehension(ast.Name('x', ast.Store()), 
                                           ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])


        # "{i : chr(65+i) for i in (1,2)}")
        dictcomp0 = ast.DictComp()
        dictcomp1 = ast.DictComp(ast.Name('i', ast.Load()), 
                                 ast.Call(ast.Name('chr', ast.Load()), 
                                          [ast.BinOp(ast.Num(65), ast.Add(), ast.Name('i', ast.Load()))],
                                          [], None, None), 
                                 [ast.comprehension(ast.Name('i', ast.Store()), 
                                                    ast.Tuple([ast.Num(1), ast.Num(n=2)], ast.Load()), [])])
        dictcomp2 = ast.DictComp(ast.Name('i', ast.Load()), 
                                 ast.Call(ast.Name('chr', ast.Load()), 
                                          [ast.BinOp(ast.Num(65), ast.Add(), ast.Name('i', ast.Load()))],
                                          [], None, None), 
                                 [ast.comprehension(ast.Name('i', ast.Store()), 
                                                    ast.Tuple([ast.Num(1), ast.Num(n=2)], ast.Load()), [])],0,0)

        # (x for x in (1,2))
        genexp0 = ast.GeneratorExp()
        genexp1 = ast.GeneratorExp(ast.Name('x', ast.Load()), 
                                   [ast.comprehension(ast.Name('x', ast.Store()), 
                                                      ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])])
        genexp2 = ast.GeneratorExp(ast.Name('x', ast.Load()), 
                                   [ast.comprehension(ast.Name('x', ast.Store()), 
                                                      ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])],0,0)

        # yield 2
        yield0 = ast.Yield()
        yield1 = ast.Yield(ast.Num(2))
        yield2 = ast.Yield(ast.Num(2),0,0)
        yield20 = ast.Yield(lineno=0, col_offset=0)

        # a>0
        compare0 = ast.Compare()
        compare1 = ast.Compare(ast.Name('a', ast.Load()), [ast.Gt()], [ast.Num(0)])
        compare2 = ast.Compare(ast.Name('a', ast.Load()), [ast.Gt()], [ast.Num(0)],0,0)

        # chr(65)
        call0 = ast.Call()
        call1 = ast.Call(ast.Name('chr', ast.Load()), [ast.Num(65)], [], None, None)
        call2 = ast.Call(ast.Name('chr', ast.Load()), [ast.Num(65)], [], None, None, 0, 0)
        call20 = ast.Call(ast.Name('f', ast.Load()), [ast.Num(0)], [])
        call21 = ast.Call(ast.Name('f', ast.Load()), [ast.Num(0)], [], lineno=0, col_offset=0)

        # 0
        num0 = ast.Num()
        num1 = ast.Num(0)
        num2 = ast.Num(0,0,0)

        # "foo"
        str0 = ast.Str()
        str1 = ast.Str("foo")
        str2 = ast.Str("foo",0,0)

        # TODO: come back
        repr0 = ast.Repr()
        repr1 = ast.Repr(ast.Num(0))
        repr2 = ast.Repr(ast.Num(0),0,0)

        # foo.bar
        attr0 = ast.Attribute()
        attr1 = ast.Attribute(ast.Name('foo', ast.Load()), 'bar', ast.Load())
        attr2 = ast.Attribute(ast.Name('foo', ast.Load()), 'bar', ast.Load(), 0,0)

        # a[1:2]
        subscript0 = ast.Subscript()
        subscript1 = ast.Subscript(ast.Name('a', ast.Load()), ast.Slice(ast.Num(1), ast.Num(2)), ast.Load())
        subscript2 = ast.Subscript(ast.Name('a', ast.Load()), ast.ExtSlice([ast.Num(1), ast.Num(2)]), ast.Load(), 0, 0)

        # name
        name0 = ast.Name()
        name1 = ast.Name("name", ast.Load())
        name2 = ast.Name("name", ast.Load(),0,0)

        # [1,2]
        list0 = ast.List()
        list1 = ast.List([ast.Num(1), ast.Num(2)], ast.Load())
        list2 = ast.List([ast.Num(1), ast.Num(2)], ast.Load(),0,0)

        # (1,2)
        tuple0 = ast.Tuple()
        tuple1 = ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load())
        tuple2 = ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load(), 0, 0)
Example #55
0
def as_ast(dct):
    """See https://docs.python.org/2/library/ast.html"""
    if dct['ast_type'] == "Module":
        return ast.Module(dct["body"])
    elif dct['ast_type'] == "Interactive":
        return ast.Interactive(dct["body"])
    elif dct['ast_type'] == "Expression":
        return ast.Expression(dct["body"])
    elif dct['ast_type'] == "Suite":
        return ast.Suite(dct["body"])
    elif dct['ast_type'] == "FunctionDef":
        return ast.FunctionDef(dct["name"], dct["args"], dct["body"],
                               dct["decorator_list"])
    elif dct['ast_type'] == "ClassDef":
        return ast.ClassDef(dct["name"], dct["bases"], dct["body"],
                            dct["decorator_list"])
    elif dct['ast_type'] == "Return":
        return ast.Return(dct["value"])
    elif dct['ast_type'] == "Delete":
        return ast.Delete(dct["targets"])
    elif dct['ast_type'] == "Assign":
        return ast.Assign(dct["targets"], dct["value"])
    elif dct['ast_type'] == "AugAssign":
        return ast.AugAssign(dct["target"], dct["op"], dct["value"])
    elif dct['ast_type'] == "Print":
        return ast.Print(dct["dest"], dct["values"], dct["nl"])
    elif dct['ast_type'] == "For":
        return ast.For(dct["target"], dct["iter"], dct["body"], dct["orelse"])
    elif dct['ast_type'] == "While":
        return ast.While(dct["test"], dct["body"], dct["orelse"])
    elif dct['ast_type'] == "If":
        return ast.If(dct["test"], dct["body"], dct["orelse"])
    elif dct['ast_type'] == "With":
        return ast.With(dct["context_expr"], dct["optional_vars"], dct["body"])
    elif dct['ast_type'] == "Raise":
        return ast.Raise(dct["type"], dct["inst"], dct["tback"])
    elif dct['ast_type'] == "TryExcept":
        return ast.TryExcept(dct["body"], dct["handlers"], dct["orelse"])
    elif dct['ast_type'] == "TryFinally":
        return ast.TryFinally(dct["body"], dct["finalbody"])
    elif dct['ast_type'] == "Assert":
        return ast.Assert(dct["test"], dct["msg"])
    elif dct['ast_type'] == "Import":
        return ast.Import(dct["names"])
    elif dct['ast_type'] == "ImportFrom":
        return ast.ImportFrom(dct["module"], dct["names"], dct["level"])
    elif dct['ast_type'] == "Exec":
        return ast.Exec(dct["body"], dct["globals"], dct["locals"])
    elif dct['ast_type'] == "Global":
        return ast.Global(dct["names"])
    elif dct['ast_type'] == "Expr":
        return ast.Expr(dct["value"])
    elif dct['ast_type'] == "Pass":
        return ast.Pass()
    elif dct['ast_type'] == "Break":
        return ast.Break()
    elif dct['ast_type'] == "Continue":
        return ast.Continue()
    elif dct['ast_type'] == "BoolOp":
        return ast.BoolOp(dct["op"], dct["values"])
    elif dct['ast_type'] == "BinOp":
        return ast.BinOp(dct["left"], dct["op"], dct["right"])
    elif dct['ast_type'] == "UnaryOp":
        return ast.UnaryOp(dct["op"], dct["operand"])
    elif dct['ast_type'] == "Lambda":
        return ast.Lambda(dct["args"], dct["body"])
    elif dct['ast_type'] == "IfExp":
        return ast.IfExp(dct["test"], dct["body"], dct["orelse"])
    elif dct['ast_type'] == "Dict":
        return ast.Dict(dct["keys"], dct["values"])
    elif dct['ast_type'] == "Set":
        return ast.Set(dct["elts"])
    elif dct['ast_type'] == "ListComp":
        return ast.ListComp(dct["elt"], dct["generators"])
    elif dct['ast_type'] == "SetComp":
        return ast.SetComp(dct["elt"], dct["generators"])
    elif dct['ast_type'] == "DictComp":
        return ast.DictComp(dct["key"], dct["value"], dct["generators"])
    elif dct['ast_type'] == "GeneratorExp":
        return ast.GeneratorExp(dct["elt"], dct["generators"])
    elif dct['ast_type'] == "Yield":
        return ast.Yield(dct["value"])
    elif dct['ast_type'] == "Compare":
        return ast.Compare(dct["left"], dct["ops"], dct["comparators"])
    elif dct['ast_type'] == "Call":
        return ast.Call(dct["func"], dct["args"], dct["keywords"],
                        dct["starargs"], dct["kwargs"])
    elif dct['ast_type'] == "Repr":
        return ast.Repr(dct["value"])
    elif dct['ast_type'] == "Num":
        return ast.Num(dct["n"])
    elif dct['ast_type'] == "Str":
        # Converting to ASCII
        return ast.Str(dct["s"].encode('ascii', 'ignore'))
    elif dct['ast_type'] == "Attribute":
        return ast.Attribute(dct["value"], dct["attr"], dct["ctx"])
    elif dct['ast_type'] == "Subscript":
        return ast.Subscript(dct["value"], dct["slice"], dct["ctx"])
    elif dct['ast_type'] == "Name":
        return ast.Name(dct["id"], dct["ctx"])
    elif dct['ast_type'] == "List":
        return ast.List(dct["elts"], dct["ctx"])
    elif dct['ast_type'] == "Tuple":
        return ast.Tuple(dct["elts"], dct["ctx"])
    elif dct['ast_type'] == "Load":
        return ast.Load()
    elif dct['ast_type'] == "Store":
        return ast.Store()
    elif dct['ast_type'] == "Del":
        return ast.Del()
    elif dct['ast_type'] == "AugLoad":
        return ast.AugLoad()
    elif dct['ast_type'] == "AugStore":
        return ast.AugStore()
    elif dct['ast_type'] == "Param":
        return ast.Param()
    elif dct['ast_type'] == "Ellipsis":
        return ast.Ellipsis()
    elif dct['ast_type'] == "Slice":
        return ast.Slice(dct["lower"], dct["upper"], dct["step"])
    elif dct['ast_type'] == "ExtSlice":
        return ast.ExtSlice(dct["dims"])
    elif dct['ast_type'] == "Index":
        return ast.Index(dct["value"])
    elif dct['ast_type'] == "And":
        return ast.And()
    elif dct['ast_type'] == "Or":
        return ast.Or()
    elif dct['ast_type'] == "Add":
        return ast.Add()
    elif dct['ast_type'] == "Sub":
        return ast.Sub()
    elif dct['ast_type'] == "Mult":
        return ast.Mult()
    elif dct['ast_type'] == "Div":
        return ast.Div()
    elif dct['ast_type'] == "Mod":
        return ast.Mod()
    elif dct['ast_type'] == "Pow":
        return ast.Pow()
    elif dct['ast_type'] == "LShift":
        return ast.LShift()
    elif dct['ast_type'] == "RShift":
        return ast.RShift()
    elif dct['ast_type'] == "BitOr":
        return ast.BitOr()
    elif dct['ast_type'] == "BitXor":
        return ast.BitXor()
    elif dct['ast_type'] == "BitAnd":
        return ast.BitAnd()
    elif dct['ast_type'] == "FloorDiv":
        return ast.FloorDiv()
    elif dct['ast_type'] == "Invert":
        return ast.Invert()
    elif dct['ast_type'] == "Not":
        return ast.Not()
    elif dct['ast_type'] == "UAdd":
        return ast.UAdd()
    elif dct['ast_type'] == "USub":
        return ast.USub()
    elif dct['ast_type'] == "Eq":
        return ast.Eq()
    elif dct['ast_type'] == "NotEq":
        return ast.NotEq()
    elif dct['ast_type'] == "Lt":
        return ast.Lt()
    elif dct['ast_type'] == "LtE":
        return ast.LtE()
    elif dct['ast_type'] == "Gt":
        return ast.Gt()
    elif dct['ast_type'] == "GtE":
        return ast.GtE()
    elif dct['ast_type'] == "Is":
        return ast.Is()
    elif dct['ast_type'] == "IsNot":
        return ast.IsNot()
    elif dct['ast_type'] == "In":
        return ast.In()
    elif dct['ast_type'] == "NotIn":
        return ast.NotIn()
    elif dct['ast_type'] == "comprehension":
        return ast.comprehension(dct["target"], dct["iter"], dct["ifs"])
    elif dct['ast_type'] == "ExceptHandler":
        return ast.ExceptHandler(dct["type"], dct["name"], dct["body"])
    elif dct['ast_type'] == "arguments":
        return ast.arguments(dct["args"], dct["vararg"], dct["kwarg"],
                             dct["defaults"])
    elif dct['ast_type'] == "keyword":
        return ast.keyword(dct["arg"], dct["value"])
    elif dct['ast_type'] == "alias":
        return ast.alias(dct["name"], dct["asname"])
    else:
        return dct
 def test_comprehension(self):
     comp = ast.comprehension(ast.Name('x', ast.Store()), ast.Name('y',
                                 ast.Load()), [])
     self.verify(comp, 'for x in y')
     comp.ifs = [ast.Num(2), ast.Num(3)]
     self.verify(comp, 'for x in y if 2 if 3')