def parse_layer_from_yaml(ast_state, yaml_data, layer_name, output_fn, fn_name_fn): matchers = [] for yaml_datum in yaml_data['filters']: matcher = create_matcher(yaml_datum, output_fn) matchers.append(matcher) stmts = [] for matcher in matchers: # columns in the query should be those needed by the rule, union those # needed by the output, minus any which are synthetic and local to the # query function. stmts.append(matcher.as_ast(ast_state)) prepend_statements = [] if ast_state.has_way_area: way_area_stmt = make_way_area_assignment() prepend_statements.append(way_area_stmt) if ast_state.has_volume: volume_stmt = make_volume_assignment() prepend_statements.append(volume_stmt) if ast_state.has_zoom: zoom_stmt = make_zoom_assignment() prepend_statements.append(zoom_stmt) stmts = prepend_statements + stmts func = ast.FunctionDef( fn_name_fn(layer_name), ast.arguments([ ast.Name('shape', ast.Param()), ast.Name('props', ast.Param()), ast.Name('fid', ast.Param()), ast.Name('meta', ast.Param()), ], None, None, []), stmts, []) return func
def visit_arguments(self, node): """Get surrogate name nodes from function arguments. This method only handles the *args and **kwargs variable names. In PY2 the rest of the children for this node names. In PY3+ they are args and handled by another visitor. """ # PY2 vararg and kwarg are raw strings which represent the tokens of # the variable name chosen. These have become arg nodes in PY33. if pycompat.PY2 or (pycompat.PY3 and pycompat.VERSION.minor < 4): if node.vararg is not None: yield references.copy_location( ast.Name( id=node.vararg, ctx=ast.Param(), ), node, ) if node.kwarg is not None: yield references.copy_location( ast.Name( id=node.kwarg, ctx=ast.Param(), ), node, )
def python_ast(self): decorators = [] iseq = [] args = [] if self.method: if self.name == "Thavron": self.name = "__init__" args.append( ast.Name("self", ast.Param(), lineno=self.line, col_offset=self.column)) if self.bound: if self.method: decorators.append(ast.Name('classmethod', ast.Load())) else: pass #TODO: Raise Semantic exception! if self.instruction_sequence: iseq = self.instruction_sequence.python_ast() return ast.FunctionDef(self.name, ast.arguments( args + [ ast.Name(name, ast.Param(), lineno=self.line, col_offset=self.column) for name in self.arguments ], None, None, []), iseq, decorators, lineno=self.line, col_offset=self.column)
def p_params(p): '''params : type ID | params COMMA type ID''' if len(p) == 3: p[0] = [ast.Param(p[1], p[2]).at(loc(p, 2))] else: p[0] = p[1] + [ast.Param(p[3], p[4]).at(loc(p, 4))]
def visit_arguments(self, node): # missing locations for vararg and kwarg set at function level if node.vararg: vararg = ast.Name(node.vararg, ast.Param()) else: vararg = None if node.kwarg: kwarg = ast.Name(node.kwarg, ast.Param()) else: kwarg = None if node.vararg: vararg = ast.Name(node.vararg, ast.Param()) else: vararg = None new_node = gast.arguments( self._visit(node.args), [], # posonlyargs self._visit(vararg), [], # kwonlyargs [], # kw_defaults self._visit(kwarg), self._visit(node.defaults), ) return new_node
def visit_FunctionDef(self, node): self.IN_INIT = False if self.CONVERT and node.name == '__init__': new_args = (node.args.args[:2] + [ ast.Name(id='new1', ctx=ast.Param()), ast.Name(id='new2', ctx=ast.Param()) ]) node.args.args = new_args self.IN_INIT = True self.generic_visit(node) return node
def parse(self, func): """Function decorator, returning a correct method from a pseudo-Python one""" # Get the function AST parsed = ast.parse(inspect.getsource(func)) fc_ast = parsed.body[0] argument_names = [get_arg_name(name) for name in fc_ast.args.args] # Init local cache self._local_ctx = {} # Translate (blocks[0][0] is the current instr) blocks, body = self._parse_body(fc_ast.body, argument_names) # Build the new function fc_ast.args.args[0:0] = [ gen_arg('ir', ast.Param()), gen_arg('instr', ast.Param()) ] cur_instr = blocks[0][0] if len(blocks[-1][0]) == 0: ## Last block can be empty blocks.pop() other_blocks = blocks[1:] body.append( ast.Return(value=ast.Tuple(elts=[ ast.List(elts=cur_instr, ctx=ast.Load()), ast.List(elts=other_blocks, ctx=ast.Load()) ], ctx=ast.Load()))) ret = ast.parse('') ret.body = [ ast.FunctionDef(name=fc_ast.name, args=fc_ast.args, body=body, decorator_list=[]) ] # To display the generated function, use codegen.to_source # codegen: https://github.com/andreif/codegen # Compile according to the context fixed = ast.fix_missing_locations(ret) codeobj = compile(fixed, '<string>', 'exec') ctx = self._ctx.copy() eval(codeobj, ctx) # Get the function back self._functions[fc_ast.name] = ctx[fc_ast.name] return ctx[fc_ast.name]
def _transform_function_arguments(left): if type(left) is ast.Name: names = [left] else: names = left.elts # Python3 if hasattr(_ast, 'arg'): args = [ _ast.arg(annotation=None, arg=name.id, col_offset=name.col_offset, lineno=name.lineno) for name in names ] return ast.arguments(args=args, defaults=[], kwonlyargs=[], kw_defaults=[]) # Python 2 arguments = ast.arguments(args=names, defaults=[]) for argument in arguments.args: argument.ctx = ast.Param() return arguments
def formal_parameters(self): """ formal_parameters : ID (COMMA ID)* COLON type_spec procedure's params, just like Foo(a, b : int) current process is a, b : int """ param_nodes = [] param_tokens = [self.current_token] self.eat(TokenType.ID) # while contain ',' while self.current_token.type == TokenType.COMMA: self.eat(TokenType.COMMA) # add all ID param_tokens.append(self.current_token) self.eat(TokenType.ID) # eat ':' self.eat(TokenType.COLON) # start params type type_node = self.type_spec() for param_token in param_tokens: var = ast.Var(param_token) param_node = ast.Param(var, type_node) param_nodes.append(param_node) return param_nodes
def visit_GeneratorExp(self, node): if node in self.optimizable_comprehension: self.generic_visit(node) iters = [self.make_Iterator(gen) for gen in node.generators] variables = [ast.Name(gen.target.id, ast.Param()) for gen in node.generators] # If dim = 1, product is useless if len(iters) == 1: iterAST = iters[0] varAST = ast.arguments([variables[0]], None, None, []) else: prodName = ast.Attribute( value=ast.Name(id='itertools', ctx=ast.Load()), attr='product', ctx=ast.Load()) iterAST = ast.Call(prodName, iters, [], None, None) varAST = ast.arguments([ast.Tuple(variables, ast.Store())], None, None, []) imapName = ast.Attribute( value=ast.Name(id='itertools', ctx=ast.Load()), attr='imap', ctx=ast.Load()) ldBodyimap = node.elt ldimap = ast.Lambda(varAST, ldBodyimap) return ast.Call(imapName, [ldimap, iterAST], [], None, None) else: return self.generic_visit(node)
def visit_GeneratorExp(self, node): node.elt = self.visit(node.elt) name = "generator_expression{0}".format(self.count) self.count += 1 args = self.passmanager.gather(ImportedIds, node, self.ctx) self.count_iter = 0 body = reduce(self.nest_reducer, reversed(node.generators), ast.Expr(ast.Yield(node.elt)) ) sargs = sorted(ast.Name(arg, ast.Param()) for arg in args) fd = ast.FunctionDef(name, ast.arguments(sargs, None, None, []), [body], []) self.ctx.module.body.append(fd) return ast.Call( ast.Name(name, ast.Load()), [ast.Name(arg.id, ast.Load()) for arg in sargs], [], None, None ) # no sharing !
def wrap_callable(self, node): return ast.Lambda( args=ast.arguments( args=[ast.Name(id=CVAR_NAME, ctx=ast.Param())], vararg=None, kwarg=None, defaults=[] ), body=node, )
def resolveNameRef(self, visitor, callinfo): # assert False # visitor.logger.debug('LibFuncMatch.resolveNameRef', self, visitor, callinfo) callinfo.getOwnerFunc().info.dispatched = True argcount = len(callinfo.args) assert argcount == 2 caseblock = callinfo.args[-1] assert isinstance(caseblock, ast.CaseBlock) if argcount == 2: caseblock.matchVar = callinfo.args[0] for entry in caseblock.entries: # visitor.logger.debug('match resolve entry', callinfo.caller, caseblock.matchVar, entry.pattern, callinfo.getOwnerFunc()) assert hasattr(entry.pattern, 'name'), ('match resolve entry', callinfo.caller, caseblock.matchVar, entry.pattern, callinfo.getOwnerFunc()) assert hasattr(caseblock.matchVar, 'name'), ('match resolve matchVar', callinfo.caller, caseblock.matchVar, entry.pattern, callinfo.getOwnerFunc()) casevar = ast.CaseVarSpec( ast.Param(caseblock.matchVar.name, ast.UserType([entry.pattern.name]))) entry.addSymbol(casevar.variable) casevar.setOwner(entry) visitor.visitNewItem(casevar)
def visit_Lambda(self, node): if modules['functools'] not in self.global_declarations.values(): import_ = ast.Import([ast.alias('functools', None)]) self.imports.append(import_) self.global_declarations['functools'] = modules['functools'] self.generic_visit(node) forged_name = "{0}_lambda{1}".format(self.prefix, len(self.lambda_functions)) ii = self.passmanager.gather(ImportedIds, node, self.ctx) ii.difference_update(self.lambda_functions) # remove current lambdas binded_args = [ast.Name(iin, ast.Load()) for iin in sorted(ii)] former_nbargs = len(node.args.args) node.args.args = ([ast.Name(iin, ast.Param()) for iin in sorted(ii)] + node.args.args) forged_fdef = ast.FunctionDef(forged_name, copy(node.args), [ast.Return(node.body)], []) self.lambda_functions.append(forged_fdef) proxy_call = ast.Name(forged_name, ast.Load()) if binded_args: return ast.Call( ast.Attribute(ast.Name('functools', ast.Load()), "partial", ast.Load()), [proxy_call] + binded_args, [], None, None) else: return proxy_call
def _translateFuncToPyAst(self, func): assert isinstance(func, CFunc) base = FuncEnv(globalScope=self.globalScope) assert func.name is not None base.func = func base.astNode.name = func.name base.pushScope(base.astNode.body) for arg in func.args: name = base.registerNewVar(arg.name, arg) assert name is not None base.astNode.args.args.append(ast.Name(id=name, ctx=ast.Param())) if func.body is None: # TODO: search in other C files # Hack for now: ignore :) print "XXX:", func.name, "is not loaded yet" else: cCodeToPyAstList(base, func.body) base.popScope() if isSameType(self._cStateWrapper, func.type, CVoidType()): returnValueAst = NoneAstNode else: returnTypeAst = getAstNodeForVarType(self, func.type) returnValueAst = makeAstNodeCall(returnTypeAst) base.astNode.body.append(ast.Return(value=returnValueAst)) return base
def __call__(self, selection): if not self.is_initialized: self._initialize() try: parse_result = self.expression.parseString(selection, parseAll=True) except ParseException as e: msg = str(e) lines = ["%s: %s" % (msg, selection), " " * (12 + len("%s: " % msg) + (e.loc)) + "^^^"] raise ValueError('\n'.join(lines)) # Change __ATOM__ in function bodies. It must bind to the arg # name specified below (i.e. 'atom') astnode = self.transformer.visit(deepcopy(parse_result[0].ast())) if PY2: args = [ast.Name(id='atom', ctx=ast.Param())] signature = ast.arguments(args=args, vararg=None, kwarg=None, defaults=[]) else: args = [ast.arg(arg='atom', annotation=None)] signature = ast.arguments(args=args, vararg=None, kwarg=None, kwonlyargs=[], defaults=[], kw_defaults=[]) func = ast.Expression(body=ast.Lambda(signature, astnode)) source = codegen.to_source(astnode) expr = eval( compile(ast.fix_missing_locations(func), '<string>', mode='eval'), SELECTION_GLOBALS) return _ParsedSelection(expr, source, astnode)
def _make_fn(name, chain_fn, args, defaults): args_with_self = ['_self'] + list(args) arguments = [_ast.Name(id=arg, ctx=_ast.Load()) for arg in args_with_self] defs = [_ast.Name(id='_def{0}'.format(idx), ctx=_ast.Load()) for idx, _ in enumerate(defaults)] if _PY2: parameters = _ast.arguments(args=[_ast.Name(id=arg, ctx=_ast.Param()) for arg in args_with_self], defaults=defs) else: parameters = _ast.arguments(args=[_ast.arg(arg=arg) for arg in args_with_self], kwonlyargs=[], defaults=defs, kw_defaults=[]) module_node = _ast.Module(body=[_ast.FunctionDef(name=name, args=parameters, body=[_ast.Return(value=_ast.Call(func=_ast.Name(id='_chain', ctx=_ast.Load()), args=arguments, keywords=[]))], decorator_list=[])]) module_node = _ast.fix_missing_locations(module_node) # compile the ast code = compile(module_node, '<string>', 'exec') # and eval it in the right context globals_ = {'_chain': chain_fn} locals_ = dict(('_def{0}'.format(idx), value) for idx, value in enumerate(defaults)) eval(code, globals_, locals_) # extract our function from the newly created module return locals_[name]
def visit_AnyComp(self, node, comp_type, *path): node.elt = self.visit(node.elt) name = "{0}_comprehension{1}".format(comp_type, self.count) self.count += 1 args = self.passmanager.gather(ImportedIds, node, self.ctx) self.count_iter = 0 starget = "__target" body = reduce( self.nest_reducer, reversed(node.generators), ast.Expr( ast.Call( reduce(lambda x, y: ast.Attribute(x, y, ast.Load()), path[1:], ast.Name(path[0], ast.Load())), [ast.Name(starget, ast.Load()), node.elt], [], None, None))) # add extra metadata to this node metadata.add(body, metadata.Comprehension(starget)) init = ast.Assign([ast.Name(starget, ast.Store())], ast.Call( ast.Attribute( ast.Name('__builtin__', ast.Load()), comp_type, ast.Load()), [], [], None, None)) result = ast.Return(ast.Name(starget, ast.Load())) sargs = sorted(ast.Name(arg, ast.Param()) for arg in args) fd = ast.FunctionDef(name, ast.arguments(sargs, None, None, []), [init, body, result], []) self.ctx.module.body.append(fd) return ast.Call(ast.Name(name, ast.Load()), [ast.Name(arg.id, ast.Load()) for arg in sargs], [], None, None) # no sharing !
def visitName(self, node, ctype=None): if isinstance(node.ctx, ast.Param): if node.id not in self.argtypes: raise CTypeError( node.id, 'function %s() requires argument %r' % (self.func_name, node.id)) ctype = self.argtypes[node.id] return cast.CName(node.id, ast.Param(), ctype, **n(node)) elif isinstance(node.ctx, ast.Load): try: ctype = self.scope(node.id) except NameError as err: raise cast.CError(node, NameError, err.args[0]) return cast.CName(node.id, ast.Load(), ctype, **n(node)) elif isinstance(node.ctx, ast.Store): assert type is not None if node.id in self.locls: ectype = self.locls[node.id] try: greatest_common_type(ctype, ectype) except: # Raise a custom exception if the types are not compatible raise ctype = ectype self.locls[node.id] = ctype return cast.CName(node.id, ast.Store(), ctype, **n(node)) else: assert False
def new_class_def(): node = ast.ClassDef( name='Conf', bases=[ast.Name(id='object', ctx=ast.Load())], body=[ ast.FunctionDef( name='__init__', args=ast.arguments(args=[ast.Name(id='self', ctx=ast.Param())], vararg=None, kwarg=None, defaults=[]), body=[ ast.Assign(targets=[ ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()), attr='AA', ctx=ast.Store()) ], value=ast.List(elts=[], ctx=ast.Load())), ast.Assign(targets=[ ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()), attr='BB', ctx=ast.Store()) ], value=ast.Num(n=2)) ], decorator_list=[]) ], decorator_list=[]) return node
def compile_func(arg_names, statements, name='_the_func', debug=False): """Compile a list of statements as the body of a function and return the resulting Python function. If `debug`, then print out the bytecode of the compiled function. """ func_def = ast.FunctionDef( name.encode('utf8'), ast.arguments( [ast.Name(n, ast.Param()) for n in arg_names], None, None, [ex_literal(None) for _ in arg_names], ), statements, [], ) mod = ast.Module([func_def]) ast.fix_missing_locations(mod) prog = compile(mod, b'<generated>', b'exec') # Debug: show bytecode. if debug: dis.dis(prog) for const in prog.co_consts: if isinstance(const, types.CodeType): dis.dis(const) the_locals = {} exec prog in {}, the_locals return the_locals[name]
def assign_namespace(self, root): setup_def = [ func for func in root.body if isinstance(func, ast.FunctionDef) and func.name == 'setup' ] setup_def = None if len(setup_def) == 0 else setup_def[0] if setup_def is None: body = [self.get_assign_node(self.namespace)] if self.app_metadata: body.append( self.get_call_append_metadata_node(self.app_metadata)) setup_node = ast.FunctionDef( name='setup', args=ast.arguments(args=[ ast.arg(arg='self', annotation=None) if six.PY3 else ast.Name(id='self', ctx=ast.Param()) ], vararg=None, kwarg=None, defaults=[]), body=body, decorator_list=[]) root.body.append(setup_node) else: assign_node = [ assign for assign in setup_def.body if isinstance(assign, ast.Assign) and self.has_target(assign.targets, "__pd_gateway_namespace__") ] if len(assign_node) == 0: setup_def.body.append(self.get_assign_node(self.namespace)) if self.app_metadata: setup_def.body.append( self.get_call_append_metadata_node(self.app_metadata))
def _build_arg(name): try: # Python 3 return ast.arg(arg=name) except AttributeError: # Python 2 return ast.Name(id=name, ctx=ast.Param())
def visit_Expr(self, node): if type(node.value) is ast.Call: call = node.value if self.is_concurrent_call(call): self.encounter_call(call) return node elif any([self.is_concurrent_call(arg) for arg in call.args]): conc_args = [(i, arg) for i, arg in enumerate(call.args) if self.is_concurrent_call(arg)] if len(conc_args) > 1: raise self.not_implemented_error(call, "Functions with multiple @concurrent parameters are unsupported") conc_call = conc_args[0][1] if isinstance(call.func, ast.Attribute): self.arguments.add(SchedulerRewriter.top_level_name(call.func.value)) self.encounter_call(conc_call) call.args[conc_args[0][0]] = ast.Name("__value__", ast.Load()) if sys.version_info >= (3, 0): args = [ast.arg("__value__", None)] else: args = [ast.Name("__value__", ast.Param())] call_lambda = ast.Lambda(ast.arguments(args = args, defaults = [], kwonlyargs = [], kw_defaults = []), call) copy_location_kwargs = { "func": ast.Attribute(conc_call.func, 'call', ast.Load()), "args": [call_lambda] + conc_call.args, "keywords": conc_call.keywords } if(sys.version_info < (3, 0)): copy_location_kwargs["kwargs"] = conc_call.kwargs return copy_location(ast.Expr(ast.Call(**copy_location_kwargs)), node) return self.generic_visit(node)
def gen_unpack_wrapper(self, node, target, ctx='store'): """Helper function to protect tuple unpacks. node: used to copy the locations for the new nodes. target: is the tuple which must be protected. ctx: Defines the context of the returned temporary node. It returns a tuple with two element. Element 1: Is a temporary name node which must be used to replace the target. The context (store, param) is defined by the 'ctx' parameter.. Element 2: Is a try .. finally where the body performs the protected tuple unpack of the temporary variable into the original target. """ # Generate a tmp name to replace the tuple with. tnam = self.tmpName # Generates an expressions which protects the unpack. # converter looks like 'wrapper(tnam)'. # 'wrapper' takes care to protect sequence unpacking with __rl_getiter__. converter = self.protect_unpack_sequence(target, ast.Name(tnam, ast.Load())) # Assign the expression to the original names. # Cleanup the temporary variable. # Generates: # try: # # converter is 'wrapper(tnam)' # arg = converter # finally: # del tmp_arg try_body = [ast.Assign(targets=[target], value=converter)] finalbody = [self.gen_del_stmt(tnam)] cleanup = ast.Try(body=try_body, finalbody=finalbody, handlers=[], orelse=[]) if ctx == 'store': ctx = ast.Store() elif ctx == 'param': ctx = ast.Param() else: # pragma: no cover # Only store and param are defined ctx. raise NotImplementedError('bad ctx "%s"' % type(ctx)) # This node is used to catch the tuple in a tmp variable. tmp_target = ast.Name(tnam, ctx) copy_locations(tmp_target, node) copy_locations(cleanup, node) return (tmp_target, cleanup)
def visitLambda(self, node): body = self.visit(node.body) args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) for var_id in sorted(self.locls.keys()): args.args.append(ast.Name(var_id, ast.Param(), **n(node))) return ast.Lambda(args, body, **n(node))
def cont_as_function(self, name, k, ck): formals = ast.arguments() if k.name and k.name != '_': #formals.args = [ast.Name (k.name, ast.Param())] formals.args = [ast.arg(k.name, ast.Param())] else: formals.args = [] return FunctionDef(name, True, formals, [], k.exp, dead_cont(ck))
def build_class(): """ Constructs a :class:`ast.ClassDef` node that wraps the entire template file. The class will have an entry function ``root`` with: .. function:: root(context) Starts the template parsing with the given context. :returns: Returns a generator of strings that can be joined to the rendered template. :returns: a 2-tuple with the class and the entry function """ args = {} if PY3: args.update({ 'args': [ ast.arg(arg='self', annotation=None), ast.arg(arg='context', annotation=None), ], 'kwonlyargs': [], 'kw_defaults': [], }) else: args['args'] = [ ast.Name(id='self', ctx=ast.Param()), ast.Name(id='context', ctx=ast.Param()) ] root_func = ast.FunctionDef( name='root', args=ast.arguments(vararg=None, kwarg=None, defaults=[], **args), body=[ # we add an empty string to guarantee for a string and generator on # root level build_yield(ast.Str(s='')) ], decorator_list=[]) klass = ast.ClassDef(name='Template', bases=[ast.Name(id='object', ctx=ast.Load())], keywords=[], starargs=None, kwargs=None, body=[root_func], decorator_list=[]) return klass, root_func
def _arg_name(name): """ :param str name: :return: """ if sys.version_info[0] == 2: return ast.Name(id=name, ctx=ast.Param()) return ast.arg(arg=name, annotation=None)
def new_param(self): id = self.expect('id') name = stdast.Name() name.lineno = self.i + 1 name.id = id name.ctx = stdast.Param() name.ctx.lineno = self.i + 1 return name