def _visitFuncOrLambda(self, node, isLambda=0, hasReturn=0): if not isLambda and node.decorators: for decorator in node.decorators.nodes: self.visit(decorator) ndecorators = len(node.decorators.nodes) else: ndecorators = 0 gen = self.FunctionGen(node, isLambda, self.get_module()) gen.hasReturn = hasReturn walk(node.code, gen) gen.finish() self.set_lineno(node) for default in node.defaults: self.visit(default) frees = gen.scope.get_free_vars() if frees: for name in frees: self.emit('LOAD_CLOSURE', name) self.emit('LOAD_CONST', gen.getCode()) self.emit('MAKE_CLOSURE', len(node.defaults)) else: self.emit('LOAD_CONST', gen.getCode()) self.emit('MAKE_FUNCTION', len(node.defaults)) for i in range(ndecorators): self.emit('CALL_FUNCTION', 1)
def _visitFuncOrLambda(self, node, isLambda=0, hasReturn=0): if not isLambda and node.decorators: for decorator in node.decorators.nodes: self.visit(decorator) ndecorators = len(node.decorators.nodes) else: ndecorators = 0 gen = self.FunctionGen(node, isLambda, self.get_module()) gen.hasReturn = hasReturn walk(node.code, gen) gen.finish() self.set_lineno (node) for default in node.defaults: self.visit(default) frees = gen.scope.get_free_vars() if frees: for name in frees: self.emit('LOAD_CLOSURE', name) self.emit('LOAD_CONST', gen.getCode ()) self.emit('MAKE_CLOSURE', len(node.defaults)) else: self.emit('LOAD_CONST', gen.getCode ()) self.emit('MAKE_FUNCTION', len(node.defaults)) for i in range(ndecorators): self.emit('CALL_FUNCTION', 1)
def type_optimizer(root): if GCO['arch'] == '2.3': # no LIST_APPEND in 2.3 return for i in root.Functions: if i.symtab.know_types(): walk(i, TypeVisitor(i.symtab.typeof))
def have_break(node): v = BreakFinder v.init() try: walk(node, v) return False except: return True
def __init__(self, tree): if GCO['gnt']: GCO['global_names_list'] = [] if GCO['gnc']: GCO['global_consts_list'] = ConstCollector() self.graph = pyassem.PyFlowGraph24("?") self.__super_init() walk(tree, self)
def __init__(self, tree): if GCO ['gnt']: GCO ['global_names_list'] = [] if GCO ['gnc']: GCO ['global_consts_list'] = ConstCollector() self.graph = pyassem.PyFlowGraph24("?") self.__super_init() walk(tree, self)
def have_break (node): v = BreakFinder v.init () try: walk (node, v) return False except: return True
def find_resources(data): resources = {} def post(d): if isinstance(d, Resource): resources[d.key] = d walk(data, post=post) return resources
def _visitFuncOrLambda(self, node, isLambda=0): gen = self.FunctionGen(node, self.filename, self.scopes, isLambda, self.class_name) walk(node.code, gen) gen.finish() self.set_lineno(node) for default in node.defaults: self.visit(default) self.emit('LOAD_CONST', gen) self.emit('MAKE_FUNCTION', len(node.defaults))
def __init__(self, source): tree = parse(source) self.functions = {} self.scopes = {} self.names = {} self.errors = [] walk(tree, self) for error in self.errors: print error
def compile(self, display=0): tree = parse(self.source) root, filename = os.path.split(self.filename) if "nested_scopes" in future.find_futures(tree): gen = NestedScopeModuleCodeGenerator(filename) else: gen = ModuleCodeGenerator(filename) walk(tree, gen, 1) if display: import pprint print pprint.pprint(tree) self.code = gen.getCode()
def find_resource_references(data): keys = [] def post(d): if is_resource_ref(d): keys.append(d['$resource']) elif is_resource_indirect_ref(d): keys.append(d['$indirect']) walk(data, post=post) return keys
def visitClass(self, node): gen = self.ClassGen(node, self.scopes, self.filename) walk(node.code, gen) gen.finish() self.set_lineno(node) self.emit('LOAD_CONST', node.name) for base in node.bases: self.visit(base) self.emit('BUILD_TUPLE', len(node.bases)) self.emit('LOAD_CONST', gen) self.emit('MAKE_FUNCTION', 0) self.emit('CALL_FUNCTION', 0) self.emit('BUILD_CLASS') self.storeName(node.name)
def __init__(self, func, filename, scopes, isLambda, class_name): self.scopes = scopes self.scope = scopes[func] self.class_name = class_name if isLambda: klass = FunctionCodeGenerator name = "<lambda.%d>" % klass.lambdaCount klass.lambdaCount = klass.lambdaCount + 1 else: name = func.name args, hasTupleArg = generateArgList(func.argnames) self.graph = pyassem.PyFlowGraph(name, filename, args, optimized=1) self.isLambda = isLambda self.super_init(filename) if not isLambda and func.doc: self.setDocstring(func.doc) lnf = walk(func.code, self.NameFinder(args), verbose=0) self.locals.push(lnf.getLocals()) if func.varargs: self.graph.setFlag(CO_VARARGS) if func.kwargs: self.graph.setFlag(CO_VARKEYWORDS) self.set_lineno(func) if hasTupleArg: self.generateArgUnpack(func.argnames)
def _visitFuncOrLambda(self, node, isLambda=0): gen = self.FunctionGen(node, self.filename, self.scopes, isLambda, self.class_name) walk(node.code, gen) gen.finish() self.set_lineno(node) for default in node.defaults: self.visit(default) frees = gen.scope.get_free_vars() if frees: for name in frees: self.emit('LOAD_CLOSURE', name) self.emit('LOAD_CONST', gen) self.emit('MAKE_CLOSURE', len(node.defaults)) else: self.emit('LOAD_CONST', gen) self.emit('MAKE_FUNCTION', len(node.defaults))
def test_parsesPythonLibraryCorrectly(self): class TreeVisitor: pass import time tree = createTree() t1 = time.time() for i in xrange(1000): visitor.walk(tree, TreeVisitor()) print "tree without yield", time.time() - t1 t1 = time.time() for i in xrange(1000): for node in visitor.walkAndGenerate(tree, TreeVisitor()): pass print "tree with yield", time.time() - t1
def test_parsesPythonLibraryCorrectly(self): class TreeVisitor: pass import time tree = createTree() t1 = time.time() for i in xrange(1000): visitor.walk(tree,TreeVisitor()) print "tree without yield",time.time()-t1 t1 = time.time() for i in xrange(1000): for node in visitor.walkAndGenerate(tree,TreeVisitor()): pass print "tree with yield",time.time()-t1
def visitClass(self, node): gen = self.ClassGen(node, self.get_module()) walk(node.code, gen) gen.finish() self.set_lineno(node) self.emit('LOAD_CONST', node.name) for base in node.bases: self.visit(base) self.emit('BUILD_TUPLE', len(node.bases)) frees = gen.scope.get_free_vars() for name in frees: self.emit('LOAD_CLOSURE', name) self.emit('LOAD_CONST', gen.getCode()) if frees: self.emit('MAKE_CLOSURE', 0) else: self.emit('MAKE_FUNCTION', 0) self.emit('CALL_FUNCTION', 0) self.emit('BUILD_CLASS') self.storeName(node.name)
def __init__(self, klass, scopes, filename): assert isinstance(filename, types.StringType) assert isinstance(scopes, types.DictType) self.graph = pyassem.PyFlowGraph(klass.name, filename, optimized=0) self.super_init(filename) lnf = walk(klass.code, self.NameFinder(), 0) self.locals.push(lnf.getLocals()) self.graph.setFlag(CO_NEWLOCALS) if klass.doc: self.setDocstring(klass.doc)
def visitGenExpr(self, node): gen = GenExprCodeGenerator(node, self.get_module()) walk(node.code, gen) gen.finish() frees = gen.scope.get_free_vars() if frees: for name in frees: self.emit('LOAD_CLOSURE', name) self.emit('LOAD_CONST', gen.getCode()) self.emit('MAKE_CLOSURE', 0) else: self.emit('LOAD_CONST', gen.getCode()) self.emit('MAKE_FUNCTION', 0) # precomputation of outmost iterable self.visit(node.code.quals[0].iter) self.emit('GET_ITER') self.emit('CALL_FUNCTION', 1)
def visitGenExpr(self, node): gen = GenExprCodeGenerator (node, self.get_module ()) walk(node.code, gen) gen.finish() frees = gen.scope.get_free_vars() if frees: for name in frees: self.emit('LOAD_CLOSURE', name) self.emit('LOAD_CONST', gen.getCode ()) self.emit('MAKE_CLOSURE', 0) else: self.emit('LOAD_CONST', gen.getCode ()) self.emit('MAKE_FUNCTION', 0) # precomputation of outmost iterable self.visit(node.code.quals[0].iter) self.emit('GET_ITER') self.emit('CALL_FUNCTION', 1)
def visitClass(self, node): gen = self.ClassGen (node, self.get_module ()) walk(node.code, gen) gen.finish() self.set_lineno (node) self.emit('LOAD_CONST', node.name) for base in node.bases: self.visit(base) self.emit('BUILD_TUPLE', len(node.bases)) frees = gen.scope.get_free_vars() for name in frees: self.emit('LOAD_CLOSURE', name) self.emit('LOAD_CONST', gen.getCode ()) if frees: self.emit('MAKE_CLOSURE', 0) else: self.emit('MAKE_FUNCTION', 0) self.emit('CALL_FUNCTION', 0) self.emit('BUILD_CLASS') self.storeName(node.name)
def visitModule(self, node): self.scopes = self.parseSymbols(node) self.scope = self.scopes[node] self.emit('SET_LINENO', 0) if node.doc: self.emit('LOAD_CONST', node.doc) self.storeName('__doc__') lnf = walk(node.node, self.NameFinder(), verbose=0) self.locals.push(lnf.getLocals()) self.visit(node.node) self.emit('LOAD_CONST', None) self.emit('RETURN_VALUE')
def test_callsVistorFunctions(self): tree = createTree() class TreeVisitor: def __init__(self): self.txt = [] def visitAClass(self,node): self.txt.append("visitAClass") self.txt.append(node.txt) return self.visitChildren(node) def visitCClass(self,node): self.txt.append("visitCClass") self.txt.append(node.txt) def getTxt(self): return ",".join(self.txt) self.assertEqual(visitor.walk(tree,TreeVisitor()).getTxt(), "visitAClass,aclass,visitCClass,cclass0,visitCClass,cclass1,visitCClass,cclass2")
def test_callsVistorFunctions(self): tree = createTree() class TreeVisitor: def __init__(self): self.txt = [] def visitAClass(self, node): self.txt.append("visitAClass") self.txt.append(node.txt) return self.visitChildren(node) def visitCClass(self, node): self.txt.append("visitCClass") self.txt.append(node.txt) def getTxt(self): return ",".join(self.txt) self.assertEqual( visitor.walk(tree, TreeVisitor()).getTxt(), "visitAClass,aclass,visitCClass,cclass0,visitCClass,cclass1,visitCClass,cclass2" )
def __init__(self, tree): self.graph = pyassem.PyFlowGraph24("<expression>") self.__super_init() walk(tree, self)
def __init__(self, tree): self.graph = pyassem.PyFlowGraph24("<interactive>") self.__super_init() walk(tree, self) self.emit('RETURN_VALUE')
def findOp(node): v = OpFinder v.init() walk(node, v) return v.op
def parseSymbols(self, tree): s = symbols.SymbolVisitor() walk(tree, s) return s.scopes
def findOp(node): v = OpFinder v.init () walk(node, v) return v.op
def hasRBC(node): try: walk(node, RBCFinder()) return False except RBCFinder: return True
import symtable def get_names(syms): return [s for s in [s.get_name() for s in syms.get_symbols()] if not (s.startswith('_[') or s.startswith('.'))] for file in sys.argv[1:]: print file f = open(file) buf = f.read() f.close() syms = symtable.symtable(buf, file, "exec") mod_names = get_names(syms) tree = parseFile(file) s = SymbolVisitor() walk(tree, s) # compare module-level symbols names2 = s.scopes[tree].get_names() if not list_eq(mod_names, names2): print print "oops", file print sort(mod_names) print sort(names2) sys.exit(-1) d = {} d.update(s.scopes) del d[tree] scopes = d.values()
def hasRBC (node): try: walk (node, RBCFinder ()) return False except RBCFinder: return True
def transform_tree (tree): walk(tree, TransformVisitor())
def parseSymbols(tree): walk(tree, SymbolVisitor())
return self.found.keys() class BadFutureParser: """Check for invalid future statements""" def visitFrom(self, node): if hasattr(node, 'valid_future'): return if node.modname != "__future__": return raise SyntaxError, "invalid future statement" def find_futures(node): p1 = FutureParser() p2 = BadFutureParser() walk(node, p1) walk(node, p2) return p1.get_features() if __name__ == "__main__": import sys from transformer import parseFile for file in sys.argv[1:]: print file tree = parseFile(file) v = FutureParser() walk(tree, v) print v.found print
def transform_tree(tree): walk(tree, TransformVisitor())
def optimize_tree(tree): walk(tree, FoldConstVisitor()) type_optimizer(tree)
def findOp(node): """Find the op (DELETE, LOAD, STORE) in an AssTuple tree""" v = OpFinder() walk(node, v, verbose=0) return v.op
def walk_params(expr): for subexpr in walk(expr): if isinstance(subexpr, ir.NameRef): yield subexpr
def find_futures(node): p1 = FutureParser() p2 = BadFutureParser() walk(node, p1) walk(node, p2) return p1.get_features()