def _visitFuncOrLambda(self, node, isLambda=0, hasReturn=0):
if not isLambda and node.decorators:
for decorator in node.decorators.nodes:
self.visit(decorator)
ndecorators = len(node.decorators.nodes)
else:
ndecorators = 0
gen = self.FunctionGen(node, isLambda, self.get_module())
gen.hasReturn = hasReturn
walk(node.code, gen)
gen.finish()
self.set_lineno(node)
for default in node.defaults:
self.visit(default)
frees = gen.scope.get_free_vars()
if frees:
for name in frees:
self.emit('LOAD_CLOSURE', name)
self.emit('LOAD_CONST', gen.getCode())
self.emit('MAKE_CLOSURE', len(node.defaults))
else:
self.emit('LOAD_CONST', gen.getCode())
self.emit('MAKE_FUNCTION', len(node.defaults))
for i in range(ndecorators):
self.emit('CALL_FUNCTION', 1)
def _visitFuncOrLambda(self, node, isLambda=0, hasReturn=0):
if not isLambda and node.decorators:
for decorator in node.decorators.nodes:
self.visit(decorator)
ndecorators = len(node.decorators.nodes)
else:
ndecorators = 0
gen = self.FunctionGen(node, isLambda, self.get_module())
gen.hasReturn = hasReturn
walk(node.code, gen)
gen.finish()
self.set_lineno (node)
for default in node.defaults:
self.visit(default)
frees = gen.scope.get_free_vars()
if frees:
for name in frees:
self.emit('LOAD_CLOSURE', name)
self.emit('LOAD_CONST', gen.getCode ())
self.emit('MAKE_CLOSURE', len(node.defaults))
else:
self.emit('LOAD_CONST', gen.getCode ())
self.emit('MAKE_FUNCTION', len(node.defaults))
for i in range(ndecorators):
self.emit('CALL_FUNCTION', 1)
def type_optimizer(root):
if GCO['arch'] == '2.3':
# no LIST_APPEND in 2.3
return
for i in root.Functions:
if i.symtab.know_types():
walk(i, TypeVisitor(i.symtab.typeof))
def have_break(node):
v = BreakFinder
v.init()
try:
walk(node, v)
return False
except:
return True
def __init__(self, tree):
if GCO['gnt']:
GCO['global_names_list'] = []
if GCO['gnc']:
GCO['global_consts_list'] = ConstCollector()
self.graph = pyassem.PyFlowGraph24("?")
self.__super_init()
walk(tree, self)
def __init__(self, tree):
if GCO ['gnt']:
GCO ['global_names_list'] = []
if GCO ['gnc']:
GCO ['global_consts_list'] = ConstCollector()
self.graph = pyassem.PyFlowGraph24("?")
self.__super_init()
walk(tree, self)
def have_break (node): v = BreakFinder v.init () try: walk (node, v) return False except: return True
def find_resources(data):
resources = {}
def post(d):
if isinstance(d, Resource):
resources[d.key] = d
walk(data, post=post)
return resources
def _visitFuncOrLambda(self, node, isLambda=0):
gen = self.FunctionGen(node, self.filename, self.scopes, isLambda,
self.class_name)
walk(node.code, gen)
gen.finish()
self.set_lineno(node)
for default in node.defaults:
self.visit(default)
self.emit('LOAD_CONST', gen)
self.emit('MAKE_FUNCTION', len(node.defaults))
def __init__(self, source):
tree = parse(source)
self.functions = {}
self.scopes = {}
self.names = {}
self.errors = []
walk(tree, self)
for error in self.errors:
print error
def compile(self, display=0):
tree = parse(self.source)
root, filename = os.path.split(self.filename)
if "nested_scopes" in future.find_futures(tree):
gen = NestedScopeModuleCodeGenerator(filename)
else:
gen = ModuleCodeGenerator(filename)
walk(tree, gen, 1)
if display:
import pprint
print pprint.pprint(tree)
self.code = gen.getCode()
def find_resource_references(data):
keys = []
def post(d):
if is_resource_ref(d):
keys.append(d['$resource'])
elif is_resource_indirect_ref(d):
keys.append(d['$indirect'])
walk(data, post=post)
return keys
def __init__(self, source):
tree = parse(source)
self.functions = {}
self.scopes = {}
self.names = {}
self.errors = []
walk(tree, self)
for error in self.errors:
print error
def visitClass(self, node):
gen = self.ClassGen(node, self.scopes, self.filename)
walk(node.code, gen)
gen.finish()
self.set_lineno(node)
self.emit('LOAD_CONST', node.name)
for base in node.bases:
self.visit(base)
self.emit('BUILD_TUPLE', len(node.bases))
self.emit('LOAD_CONST', gen)
self.emit('MAKE_FUNCTION', 0)
self.emit('CALL_FUNCTION', 0)
self.emit('BUILD_CLASS')
self.storeName(node.name)
def __init__(self, func, filename, scopes, isLambda, class_name):
self.scopes = scopes
self.scope = scopes[func]
self.class_name = class_name
if isLambda:
klass = FunctionCodeGenerator
name = "<lambda.%d>" % klass.lambdaCount
klass.lambdaCount = klass.lambdaCount + 1
else:
name = func.name
args, hasTupleArg = generateArgList(func.argnames)
self.graph = pyassem.PyFlowGraph(name, filename, args,
optimized=1)
self.isLambda = isLambda
self.super_init(filename)
if not isLambda and func.doc:
self.setDocstring(func.doc)
lnf = walk(func.code, self.NameFinder(args), verbose=0)
self.locals.push(lnf.getLocals())
if func.varargs:
self.graph.setFlag(CO_VARARGS)
if func.kwargs:
self.graph.setFlag(CO_VARKEYWORDS)
self.set_lineno(func)
if hasTupleArg:
self.generateArgUnpack(func.argnames)
def _visitFuncOrLambda(self, node, isLambda=0):
gen = self.FunctionGen(node, self.filename, self.scopes, isLambda,
self.class_name)
walk(node.code, gen)
gen.finish()
self.set_lineno(node)
for default in node.defaults:
self.visit(default)
frees = gen.scope.get_free_vars()
if frees:
for name in frees:
self.emit('LOAD_CLOSURE', name)
self.emit('LOAD_CONST', gen)
self.emit('MAKE_CLOSURE', len(node.defaults))
else:
self.emit('LOAD_CONST', gen)
self.emit('MAKE_FUNCTION', len(node.defaults))
def test_parsesPythonLibraryCorrectly(self):
class TreeVisitor:
pass
import time
tree = createTree()
t1 = time.time()
for i in xrange(1000):
visitor.walk(tree, TreeVisitor())
print "tree without yield", time.time() - t1
t1 = time.time()
for i in xrange(1000):
for node in visitor.walkAndGenerate(tree, TreeVisitor()):
pass
print "tree with yield", time.time() - t1
def test_parsesPythonLibraryCorrectly(self):
class TreeVisitor:
pass
import time
tree = createTree()
t1 = time.time()
for i in xrange(1000):
visitor.walk(tree,TreeVisitor())
print "tree without yield",time.time()-t1
t1 = time.time()
for i in xrange(1000):
for node in visitor.walkAndGenerate(tree,TreeVisitor()):
pass
print "tree with yield",time.time()-t1
def visitClass(self, node):
gen = self.ClassGen(node, self.get_module())
walk(node.code, gen)
gen.finish()
self.set_lineno(node)
self.emit('LOAD_CONST', node.name)
for base in node.bases:
self.visit(base)
self.emit('BUILD_TUPLE', len(node.bases))
frees = gen.scope.get_free_vars()
for name in frees:
self.emit('LOAD_CLOSURE', name)
self.emit('LOAD_CONST', gen.getCode())
if frees:
self.emit('MAKE_CLOSURE', 0)
else:
self.emit('MAKE_FUNCTION', 0)
self.emit('CALL_FUNCTION', 0)
self.emit('BUILD_CLASS')
self.storeName(node.name)
def __init__(self, klass, scopes, filename):
assert isinstance(filename, types.StringType)
assert isinstance(scopes, types.DictType)
self.graph = pyassem.PyFlowGraph(klass.name, filename,
optimized=0)
self.super_init(filename)
lnf = walk(klass.code, self.NameFinder(), 0)
self.locals.push(lnf.getLocals())
self.graph.setFlag(CO_NEWLOCALS)
if klass.doc:
self.setDocstring(klass.doc)
def visitGenExpr(self, node):
gen = GenExprCodeGenerator(node, self.get_module())
walk(node.code, gen)
gen.finish()
frees = gen.scope.get_free_vars()
if frees:
for name in frees:
self.emit('LOAD_CLOSURE', name)
self.emit('LOAD_CONST', gen.getCode())
self.emit('MAKE_CLOSURE', 0)
else:
self.emit('LOAD_CONST', gen.getCode())
self.emit('MAKE_FUNCTION', 0)
# precomputation of outmost iterable
self.visit(node.code.quals[0].iter)
self.emit('GET_ITER')
self.emit('CALL_FUNCTION', 1)
def visitGenExpr(self, node):
gen = GenExprCodeGenerator (node, self.get_module ())
walk(node.code, gen)
gen.finish()
frees = gen.scope.get_free_vars()
if frees:
for name in frees:
self.emit('LOAD_CLOSURE', name)
self.emit('LOAD_CONST', gen.getCode ())
self.emit('MAKE_CLOSURE', 0)
else:
self.emit('LOAD_CONST', gen.getCode ())
self.emit('MAKE_FUNCTION', 0)
# precomputation of outmost iterable
self.visit(node.code.quals[0].iter)
self.emit('GET_ITER')
self.emit('CALL_FUNCTION', 1)
def visitClass(self, node):
gen = self.ClassGen (node, self.get_module ())
walk(node.code, gen)
gen.finish()
self.set_lineno (node)
self.emit('LOAD_CONST', node.name)
for base in node.bases:
self.visit(base)
self.emit('BUILD_TUPLE', len(node.bases))
frees = gen.scope.get_free_vars()
for name in frees:
self.emit('LOAD_CLOSURE', name)
self.emit('LOAD_CONST', gen.getCode ())
if frees:
self.emit('MAKE_CLOSURE', 0)
else:
self.emit('MAKE_FUNCTION', 0)
self.emit('CALL_FUNCTION', 0)
self.emit('BUILD_CLASS')
self.storeName(node.name)
def visitModule(self, node):
self.scopes = self.parseSymbols(node)
self.scope = self.scopes[node]
self.emit('SET_LINENO', 0)
if node.doc:
self.emit('LOAD_CONST', node.doc)
self.storeName('__doc__')
lnf = walk(node.node, self.NameFinder(), verbose=0)
self.locals.push(lnf.getLocals())
self.visit(node.node)
self.emit('LOAD_CONST', None)
self.emit('RETURN_VALUE')
def test_callsVistorFunctions(self):
tree = createTree()
class TreeVisitor:
def __init__(self):
self.txt = []
def visitAClass(self,node):
self.txt.append("visitAClass")
self.txt.append(node.txt)
return self.visitChildren(node)
def visitCClass(self,node):
self.txt.append("visitCClass")
self.txt.append(node.txt)
def getTxt(self):
return ",".join(self.txt)
self.assertEqual(visitor.walk(tree,TreeVisitor()).getTxt(),
"visitAClass,aclass,visitCClass,cclass0,visitCClass,cclass1,visitCClass,cclass2")
def test_callsVistorFunctions(self):
tree = createTree()
class TreeVisitor:
def __init__(self):
self.txt = []
def visitAClass(self, node):
self.txt.append("visitAClass")
self.txt.append(node.txt)
return self.visitChildren(node)
def visitCClass(self, node):
self.txt.append("visitCClass")
self.txt.append(node.txt)
def getTxt(self):
return ",".join(self.txt)
self.assertEqual(
visitor.walk(tree, TreeVisitor()).getTxt(),
"visitAClass,aclass,visitCClass,cclass0,visitCClass,cclass1,visitCClass,cclass2"
)
def __init__(self, tree):
self.graph = pyassem.PyFlowGraph24("<expression>")
self.__super_init()
walk(tree, self)
def __init__(self, tree):
self.graph = pyassem.PyFlowGraph24("<interactive>")
self.__super_init()
walk(tree, self)
self.emit('RETURN_VALUE')
def findOp(node):
v = OpFinder
v.init()
walk(node, v)
return v.op
def parseSymbols(self, tree):
s = symbols.SymbolVisitor()
walk(tree, s)
return s.scopes
def findOp(node): v = OpFinder v.init () walk(node, v) return v.op
def hasRBC(node):
try:
walk(node, RBCFinder())
return False
except RBCFinder:
return True
import symtable
def get_names(syms):
return [s for s in [s.get_name() for s in syms.get_symbols()]
if not (s.startswith('_[') or s.startswith('.'))]
for file in sys.argv[1:]:
print file
f = open(file)
buf = f.read()
f.close()
syms = symtable.symtable(buf, file, "exec")
mod_names = get_names(syms)
tree = parseFile(file)
s = SymbolVisitor()
walk(tree, s)
# compare module-level symbols
names2 = s.scopes[tree].get_names()
if not list_eq(mod_names, names2):
print
print "oops", file
print sort(mod_names)
print sort(names2)
sys.exit(-1)
d = {}
d.update(s.scopes)
del d[tree]
scopes = d.values()
def hasRBC (node): try: walk (node, RBCFinder ()) return False except RBCFinder: return True
def transform_tree (tree): walk(tree, TransformVisitor())
def parseSymbols(tree):
walk(tree, SymbolVisitor())
def __init__(self, tree):
self.graph = pyassem.PyFlowGraph24("<expression>")
self.__super_init()
walk(tree, self)
return self.found.keys()
class BadFutureParser:
"""Check for invalid future statements"""
def visitFrom(self, node):
if hasattr(node, 'valid_future'):
return
if node.modname != "__future__":
return
raise SyntaxError, "invalid future statement"
def find_futures(node):
p1 = FutureParser()
p2 = BadFutureParser()
walk(node, p1)
walk(node, p2)
return p1.get_features()
if __name__ == "__main__":
import sys
from transformer import parseFile
for file in sys.argv[1:]:
print file
tree = parseFile(file)
v = FutureParser()
walk(tree, v)
print v.found
print
def transform_tree(tree):
walk(tree, TransformVisitor())
def __init__(self, tree):
self.graph = pyassem.PyFlowGraph24("<interactive>")
self.__super_init()
walk(tree, self)
self.emit('RETURN_VALUE')
def optimize_tree(tree):
walk(tree, FoldConstVisitor())
type_optimizer(tree)
def findOp(node):
"""Find the op (DELETE, LOAD, STORE) in an AssTuple tree"""
v = OpFinder()
walk(node, v, verbose=0)
return v.op
def walk_params(expr):
for subexpr in walk(expr):
if isinstance(subexpr, ir.NameRef):
yield subexpr
def find_futures(node):
p1 = FutureParser()
p2 = BadFutureParser()
walk(node, p1)
walk(node, p2)
return p1.get_features()
