def stop_iteration(node): cmp_break = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.BREAK)]) cmp_return = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.RETURN)]) test = ast.BoolOp(op=ast.Or(), values=[cmp_break, cmp_return]) break_stmt = ast.Break() ifstmt = ast.If(test=test, body=[break_stmt], orelse=[]) node.body.append(ifstmt)
def import_call_expr(self, name, fromlist=None, level=None): name_expr = mk_str(name) if name else mk_None() if fromlist: from_expr = mk_tuple(mk_str(item) for item in fromlist) else: from_expr = mk_None() args = [name_expr, mk_call("globals"), mk_None(), from_expr] if level: args.append(mk_num(level)) return mk_call("__import__", args)
def visit_FunctionDef(self, node): args = node.args if not args.defaults: return node orig_id = node.name node.name = self.id_factory(orig_id) parameters = mk_tuple(mk_str(arg.id) for arg in args.args) defaults = mk_tuple(args.defaults) parameters_id = self.id_factory("tuple") parameters_stmt = mk_assign(parameters_id, parameters) parameters = mk_name(parameters_id) defaults_id = self.id_factory("tuple") default_stmt = mk_assign(defaults_id, defaults) defaults = mk_name(defaults_id) args.defaults = [] func_expr = mk_call("__pydron_defaults__", [mk_name(node.name), parameters, defaults]) assign_stmt = mk_assign(orig_id, func_expr) return [node, parameters_stmt, default_stmt, assign_stmt]
def visit_Call(self, node): node = self.generic_visit(node) if not isinstance(node.func, ast.Name): # only direct calls, we cannot detect them otherwise return node if node.func.id_scope != Scope.GLOBAL: # `locals` is a python built-in. return node if node.func.id != "locals": return node block = self._stack[-1] def condition(var, scope): if naming.decode_id(var)[1]: # no variables we introduced return False elif var.startswith("__pydron"): return False elif scope == Scope.LOCAL or scope == Scope.SHARED: # local variables are welcome return True elif not isinstance(block, ast.ClassDef): # free variables too... return scope == Scope.FREE else: # ... unless we are in a class return False localvars = set([var for var,scope in block.scopes.iteritems() if condition(var, scope)]) args = mk_dict((mk_str(var), mk_call('__pydron_unbound_unchecked__', [mk_name(var)])) for var in localvars) return mk_call('__pydron_locals__', [args])
def visit_FunctionDef(self, node): self.generic_visit(node) scopes = node.scopes shared = [var for var,scope in scopes.iteritems() if scope == scoping.Scope.SHARED] free = [var for var,scope in scopes.iteritems() if scope == scoping.Scope.FREE] # create cells first thing in body shared_stmts = [mk_assign(var, mk_call("__pydron_new_cell__", [mk_str(var)])) for var in shared] node.body = shared_stmts + node.body if free: # free vars become arguments free_args = [ast.Name(id=var, ctx=ast.Param()) for var in free] node.args.args = free_args + node.args.args # rename the function orig_name = node.name tmp_name = self.id_factory(node.name) node.name = tmp_name # wrap it if self._inside_class(): wrap_args = [mk_name(naming.passthrough_var(var)) for var in free] else: wrap_args = [mk_name(var) for var in free] wrap_call = mk_call("__pydron_wrap_closure__", [mk_name(tmp_name), mk_tuple(wrap_args)]) wrap_stmt = mk_assign(orig_name, wrap_call) return [node, wrap_stmt] else: return node
def visit_Return(self, node): node = self.generic_visit(node) if node.value: retval = mk_assign(self.retval_id, node.value) else: retval = mk_assign(self.retval_id, mk_name("None")) flag = mk_assign(self.flag_id, mk_str(self.RETURN)) return [retval, flag], {"return"}, {"return"}
def visit_Name(self, node): if self.inside_module(): return node if not self.is_global_name(node): return node if isinstance(node.ctx, ast.Load): return mk_call("__pydron_read_global__", args=[mk_str(node.id)]) else: raise ValueError("Unhandled assignment to a global variable: %s." % node.id)
def unpack_star(self, module_id): tmp = self._unique_modattr_id() dir_expr = mk_call("dir", args=[mk_name(module_id)]) all_expr = mk_call("getattr", args=[mk_name(module_id), mk_str('__all__'), dir_expr]) value_expr = mk_call("getattr", args=[mk_name(module_id), mk_name(tmp)]) body_stmt = mk_subscript_assign(mk_call("globals"), mk_name(tmp), value_expr) return ast.For(target=mk_name(tmp), iter=all_expr, body=[body_stmt], orelse=[])
def visit_Delete(self, node): if self.inside_module(): return node statements = [] for target in node.targets: if isinstance(target, ast.Name) and self.is_global_name(target): stmt = mk_call("__pydron_delete_global__", args=[mk_str(target.id)]) statements.append(ast.Expr(stmt)) else: statements.append(ast.Delete(targets=[target])) return statements
def visit_ClassDef(self, node): node = self.generic_visit(node) # free variables of the class block become parameters of the function scopes = getattr(node, 'scopes', {}) free = [var for var,scope in scopes.iteritems() if scope == scoping.Scope.FREE] free_param = [ast.Name(id=var, ctx=ast.Param()) for var in free] free_args = [ast.Name(id=var, ctx=ast.Load()) for var in free] # free variables of sub-blocks. Those need name mangling to avoid # collisions with variables local to the class block. passthrough = self.find_passthrough_vars(node) pt_param = [ast.Name(id=naming.passthrough_var(var), ctx=ast.Param()) for var in passthrough] pt_args = [ast.Name(id=var, ctx=ast.Load()) for var in passthrough] # function to execute the class body and collect the attributes func = ast.FunctionDef() func.name = self.id_factory("class_" + node.name) func.args = ast.arguments(args=free_param + pt_param, vararg=None, kwarg=None, defaults=[]) func.body = node.body + [ast.Return(value=mk_name('__pydron_members__')) ] func.decorator_list = [] # replicate name mangling of `LocalizeFreeVariables` all_args = free_args + pt_args if self._inside_class(): for arg in all_args: arg.id = naming.passthrough_var(arg.id) # create the class typefunc = mk_call('__pydron_read_global__', [mk_str('type')]) class_expr = mk_call_expr(typefunc, [mk_str(node.name), mk_tuple(node.bases), mk_call(func.name, all_args)]) for decorator in reversed(node.decorator_list): class_expr = mk_call_expr(decorator, [class_expr]) stmt = mk_assign(node.name, class_expr) return [func, stmt]
def visit_ClassDef(self, node): node = self.generic_visit(node) # free variables of the class block become parameters of the function scopes = getattr(node, "scopes", {}) free = [var for var, scope in scopes.iteritems() if scope == scoping.Scope.FREE] free_param = [ast.Name(id=var, ctx=ast.Param()) for var in free] free_args = [ast.Name(id=var, ctx=ast.Load()) for var in free] # free variables of sub-blocks. Those need name mangling to avoid # collisions with variables local to the class block. passthrough = self.find_passthrough_vars(node) pt_param = [ast.Name(id=naming.passthrough_var(var), ctx=ast.Param()) for var in passthrough] pt_args = [ast.Name(id=var, ctx=ast.Load()) for var in passthrough] # function to execute the class body and collect the attributes func = ast.FunctionDef() func.name = self.id_factory("class_" + node.name) func.args = ast.arguments(args=free_param + pt_param, vararg=None, kwarg=None, defaults=[]) func.body = node.body + [ast.Return(value=mk_name("__pydron_members__"))] func.decorator_list = [] # replicate name mangling of `LocalizeFreeVariables` all_args = free_args + pt_args if self._inside_class(): for arg in all_args: arg.id = naming.passthrough_var(arg.id) # create the class typefunc = mk_call("__pydron_read_global__", [mk_str("type")]) class_expr = mk_call_expr(typefunc, [mk_str(node.name), mk_tuple(node.bases), mk_call(func.name, all_args)]) for decorator in reversed(node.decorator_list): class_expr = mk_call_expr(decorator, [class_expr]) stmt = mk_assign(node.name, class_expr) return [func, stmt]
def process_body(self, stmts): """ Processes a sequence of statements. The returned code contains no interrupt statements and acts the same way the original code would, assuming that the first statement is executed (we don't handle interrupts that happen before). Returns a list of statements and a set with all interrupts that might be triggered within those statements. """ stmt_iter = iter(stmts) body = [] for stmt in stmt_iter: stmt, candidates, guaranteed = self.visit_with_interrupts(stmt) # If we know for sure that an interrupt will trigger, we # can remove the reminder of the statements. if guaranteed: self._merge(body, stmt) return body, candidates, guaranteed # If the previous statement might have interrupted, # we wrap all the remaining statements in an `if` # statement. elif candidates: # initialize the flag varible, so that it is defined # even if the interrupt does not trigger. # This happens before the statement that might trigger. init = mk_assign(self.flag_id, mk_str(self.NONE)) body.insert(-1, init) self._merge(body, stmt) ifstmts, more_candidates, _ = self.wrap_with_if(stmt_iter, candidates) body.extend(ifstmts) # The rest of the body might trigger other # interrupts as well. return body, candidates | more_candidates, set() else: self._merge(body, stmt) return body, set(), set()
def process_body(self, stmts): """ Processes a sequence of statements. The returned code contains no interrupt statements and acts the same way the original code would, assuming that the first statement is executed (we don't handle interrupts that happen before). Returns a list of statements and a set with all interrupts that might be triggered within those statements. """ stmt_iter = iter(stmts) body = [] for stmt in stmt_iter: stmt, candidates, guaranteed = self.visit_with_interrupts(stmt) # If we know for sure that an interrupt will trigger, we # can remove the reminder of the statements. if guaranteed: self._merge(body, stmt) return body, candidates, guaranteed # If the previous statement might have interrupted, # we wrap all the remaining statements in an `if` # statement. elif candidates: # initialize the flag varible, so that it is defined # even if the interrupt does not trigger. # This happens before the statement that might trigger. init = mk_assign(self.flag_id, mk_str(self.NONE)) body.insert(-1, init) self._merge(body, stmt) ifstmts, more_candidates, _ = self.wrap_with_if( stmt_iter, candidates) body.extend(ifstmts) # The rest of the body might trigger other # interrupts as well. return body, candidates | more_candidates, set() else: self._merge(body, stmt) return body, set(), set()
def visit_FunctionDef(self, node): self.stack.append(node) node = self.generic_visit(node) self.stack.pop() if self.inside_module(): return node if self.is_global(node, "name"): orig_name = node.name new_name = self.id_factory(orig_name) node.name = new_name stmt = mk_call("__pydron_assign_global__", args=[mk_str(orig_name), mk_name(new_name)]) return [node, ast.Expr(stmt)] else: return node
def visit_ExceptHandler(self, node): if self.inside_module(): return node if isinstance(node.name, ast.Name) and self.is_global_name(node.name): var = node.name.id stmt = mk_call("__pydron_assign_global__", args=[mk_str(var), mk_name(var)]) node.body = [ast.Expr(stmt)] + node.body elif node.name: node.name = self.visit(node.name) if node.type: node.type = self.visit(node.type) return node
def visit_FunctionDef(self, node): self.generic_visit(node) scopes = node.scopes shared = [ var for var, scope in scopes.iteritems() if scope == scoping.Scope.SHARED ] free = [ var for var, scope in scopes.iteritems() if scope == scoping.Scope.FREE ] # create cells first thing in body shared_stmts = [ mk_assign(var, mk_call("__pydron_new_cell__", [mk_str(var)])) for var in shared ] node.body = shared_stmts + node.body if free: # free vars become arguments free_args = [ast.Name(id=var, ctx=ast.Param()) for var in free] node.args.args = free_args + node.args.args # rename the function orig_name = node.name tmp_name = self.id_factory(node.name) node.name = tmp_name # wrap it if self._inside_class(): wrap_args = [ mk_name(naming.passthrough_var(var)) for var in free ] else: wrap_args = [mk_name(var) for var in free] wrap_call = mk_call( "__pydron_wrap_closure__", [mk_name(tmp_name), mk_tuple(wrap_args)]) wrap_stmt = mk_assign(orig_name, wrap_call) return [node, wrap_stmt] else: return node
def unpack_star(self, module_id): tmp = self._unique_modattr_id() dir_expr = mk_call("dir", args=[mk_name(module_id)]) all_expr = mk_call( "getattr", args=[mk_name(module_id), mk_str('__all__'), dir_expr]) value_expr = mk_call("getattr", args=[mk_name(module_id), mk_name(tmp)]) body_stmt = mk_subscript_assign(mk_call("globals"), mk_name(tmp), value_expr) return ast.For(target=mk_name(tmp), iter=all_expr, body=[body_stmt], orelse=[])
def visit_Assign(self, node): node.value = self.generic_visit(node.value) if self.inside_module(): return node statements = [] if len(node.targets) == 1: target = node.targets[0] if isinstance(target, ast.Name) and self.is_global_name(target): stmt = mk_call("__pydron_assign_global__", args=[mk_str(target.id), node.value]) statements.append(ast.Expr(value=stmt)) else: statements.append(node) else: raise ValueError("not supported") return statements
def wrap_with_if(self, stmt_iter, candidates): """ Wraps statements in an `if` statement so that they only execute if no interrupts have triggered. Returns the `if` statement and a set of interrupts that might trigger inside the body. """ ifbody, additional_interrupts, _ = self.process_body(stmt_iter) if ifbody: comparison = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.NONE)]) ifstmt = ast.If(test=comparison, body=ifbody, orelse=[]) return [ifstmt], additional_interrupts, set() else: # there are no statements, so we don't have to put an `if` around them. return [], additional_interrupts, set()
def _visit_loop(self, node, stop_iteration): body, body_interrupts, body_guranteed = self.process_body(node.body) orelse, orelse_interrupts, orelse_guranteed = self.process_body(node.orelse) # All interrupts of `orelse` propagate. Of `body` both `break` and `continue` are consumed # and only `return` goes on. for_interrupts = orelse_interrupts if "return" in body_interrupts: for_interrupts.add("return") if "break" not in body_interrupts and "return" not in body_interrupts: for_guranteed = orelse_guranteed else: for_guranteed = set() statements = [] # Clear the 'continue' at the end of the body if "continue" in body_interrupts: # if flag == "continue": # flag == None clearflag = mk_assign(self.flag_id, mk_str(self.NONE)) if "continue" not in body_guranteed: comparison = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.CONTINUE)]) ifstmt = ast.If(test=comparison, body=[clearflag], orelse=[]) body.append(ifstmt) else: body.append(clearflag) # init the flags for "break" for the case when there isn't a single iteration. # If we 'pass on' some interrupts then body_visit will take care of this. if "break" in body_interrupts and "return" not in for_interrupts: statements.append(mk_assign(self.flag_id, mk_str(self.NONE))) # Assemble modified loop. node.body = body node.orelse = [] # make the loop stop on 'break' and 'return' if "break" in body_interrupts or "return" in body_interrupts: stop_iteration(node) # Add the loop to the result statements.append(node) # `orelse` if "break" in body_interrupts or "return" in body_interrupts: # the `orelse` section may or may not execute if "break" in body_interrupts: # if flag == "break": # flag == None clearflag = mk_assign(self.flag_id, mk_str(self.NONE)) comparison = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.BREAK)]) clear_break = [ast.If(test=comparison, body=[clearflag], orelse=[])] else: clear_break = [] if not orelse: statements.extend(clear_break) else: # if flag == None # ORELSSE # else: # clear_break comparison = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.NONE)]) orelse_if = ast.If(test=comparison, body=orelse, orelse=clear_break) statements.append(orelse_if) else: # there is no `break` therefore `orelse` is always executed. statements.extend(orelse) return statements, for_interrupts, for_guranteed
def visit_FunctionDef(self, node): self.generic_visit(node) scopes = node.scopes shared = [ var for var, scope in scopes.iteritems() if scope == scoping.Scope.SHARED ] free = [ var for var, scope in scopes.iteritems() if scope == scoping.Scope.FREE ] # create cells first thing in body shared_stmts = [ mk_assign(var, mk_call("__pydron_new_cell__", [mk_str(var)])) for var in shared ] fixed_arguments = [] for arg in node.args.args: if isinstance(arg, ast.Attribute) and arg.attr == "cell_contents": # the generic_visit above made a mess. The attribute is # a free/shared variable and visit_Name replaced it with # an attribute. This makes no sense, so lets repair the damage. # # We don't want to change the parameter name. And I don't # want to change the variable name either. To solve this # we temporarily assign it to another variable: # # def foo(freevar): # freevar__U0 = freevar # freevar = __pydron_new_cell__('freevar') # freevar.cell_contents = freevar__U0 varname = arg.value.id tmp = self.id_factory(varname) # freevar__U0 = freevar shared_stmts.insert(0, mk_assign(tmp, mk_name(varname))) # freevar = __pydron_new_cell__('freevar') # is already in shared_stmts # reevar.cell_contents = freevar__U0 arg.ctx = ast.Store() shared_stmts.append( ast.Assign(targets=[arg], value=mk_name(tmp))) fixed_arguments.append(ast.Name(id=varname, ctx=ast.Param())) else: fixed_arguments.append(arg) node.args.args = fixed_arguments node.body = shared_stmts + node.body if free: # free vars become arguments free_args = [ast.Name(id=var, ctx=ast.Param()) for var in free] node.args.args = free_args + node.args.args # rename the function orig_name = node.name tmp_name = self.id_factory(node.name) node.name = tmp_name # wrap it if self._inside_class(): wrap_args = [ mk_name(naming.passthrough_var(var)) for var in free ] else: wrap_args = [mk_name(var) for var in free] wrap_call = mk_call( "__pydron_wrap_closure__", [mk_name(tmp_name), mk_tuple(wrap_args)]) wrap_stmt = mk_assign(orig_name, wrap_call) return [node, wrap_stmt] else: return node
def wrap_condition(node): comparison = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.NONE)]) node.test = ast.BoolOp(ast.And(), [comparison, node.test])
def visit_Continue(self, node): node = self.generic_visit(node) return mk_assign(self.flag_id, mk_str(self.CONTINUE)), {"continue"}, {"continue"}
def visit_Break(self, node): node = self.generic_visit(node) return mk_assign(self.flag_id, mk_str(self.BREAK)), {"break"}, {"break"}
def stop_iteration(node): cmp_break = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.BREAK)]) cmp_return = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.RETURN)]) test = ast.BoolOp(op=ast.Or(), values = [cmp_break, cmp_return]) break_stmt = ast.Break() ifstmt = ast.If(test=test, body=[break_stmt], orelse=[]) node.body.append(ifstmt)
def _visit_loop(self, node, stop_iteration): body, body_interrupts, body_guranteed = self.process_body(node.body) orelse, orelse_interrupts, orelse_guranteed = self.process_body( node.orelse) # All interrupts of `orelse` propagate. Of `body` both `break` and `continue` are consumed # and only `return` goes on. for_interrupts = orelse_interrupts if "return" in body_interrupts: for_interrupts.add("return") if "break" not in body_interrupts and "return" not in body_interrupts: for_guranteed = orelse_guranteed else: for_guranteed = set() statements = [] # Clear the 'continue' at the end of the body if "continue" in body_interrupts: # if flag == "continue": # flag == None clearflag = mk_assign(self.flag_id, mk_str(self.NONE)) if "continue" not in body_guranteed: comparison = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.CONTINUE)]) ifstmt = ast.If(test=comparison, body=[clearflag], orelse=[]) body.append(ifstmt) else: body.append(clearflag) # init the flags for "break" for the case when there isn't a single iteration. # If we 'pass on' some interrupts then body_visit will take care of this. if "break" in body_interrupts and "return" not in for_interrupts: statements.append(mk_assign(self.flag_id, mk_str(self.NONE))) # Assemble modified loop. node.body = body node.orelse = [] # make the loop stop on 'break' and 'return' if "break" in body_interrupts or "return" in body_interrupts: stop_iteration(node) # Add the loop to the result statements.append(node) # `orelse` if "break" in body_interrupts or "return" in body_interrupts: # the `orelse` section may or may not execute if "break" in body_interrupts: # if flag == "break": # flag == None clearflag = mk_assign(self.flag_id, mk_str(self.NONE)) comparison = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.BREAK)]) clear_break = [ ast.If(test=comparison, body=[clearflag], orelse=[]) ] else: clear_break = [] if not orelse: statements.extend(clear_break) else: # if flag == None # ORELSSE # else: # clear_break comparison = ast.Compare(left=mk_name(self.flag_id), ops=[ast.Eq()], comparators=[mk_str(self.NONE)]) orelse_if = ast.If(test=comparison, body=orelse, orelse=clear_break) statements.append(orelse_if) else: # there is no `break` therefore `orelse` is always executed. statements.extend(orelse) return statements, for_interrupts, for_guranteed