def test_get_graph():
import os
def list_basic_ops(i, j):
l = [1,2,3]
l.insert(0, 42)
del l[1]
l.append(i)
listlen = len(l)
l.extend(l)
del l[listlen:]
l += [5,6]
l[1] = i
return l[j]
def external_function():
return os.system("ls")
graph, t = translate(list_basic_ops, [int, int], False)
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
print op
graph = get_graph(op.args[0], t)
assert graph is not None
graph, t = translate(external_function, [], False)
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
print op
graph = get_graph(op.args[0], t)
assert graph is None
def is_malloc_like(adi, graph, seen):
if graph in seen:
return seen[graph]
return_state = adi.getstate(graph.getreturnvar())
if return_state is None or len(return_state.creation_points) != 1:
seen[graph] = False
return False
crep, = return_state.creation_points
if crep.escapes:
seen[graph] = False
return False
if crep.creation_method in ["malloc", "malloc_varsize"]:
assert crep.returns
seen[graph] = True
return True
if crep.creation_method == "direct_call":
subgraph = get_graph(crep.op.args[0], adi.translation_context)
if subgraph is None:
seen[graph] = False
return False
res = is_malloc_like(adi, subgraph, seen)
seen[graph] = res
return res
seen[graph] = False
return False
def find_malloc_creps(graph, adi, translator, malloc_graphs):
# mapping from malloc creation point to graphs that it flows into
malloc_creps = {}
# find all mallocs that don't escape
for block, op in graph.iterblockops():
if op.opname == 'malloc':
STRUCT = op.args[0].value
# must not remove mallocs of structures that have a RTTI with a destructor
flags = op.args[1].value
if flags != {'flavor': 'gc'}:
continue
try:
destr_ptr = lltype.getRuntimeTypeInfo(
STRUCT)._obj.destructor_funcptr
if destr_ptr:
continue
except (ValueError, AttributeError), e:
pass
varstate = adi.getstate(op.result)
assert len(varstate.creation_points) == 1
crep, = varstate.creation_points
if not crep.escapes and not crep.returns:
malloc_creps[crep] = {}
if op.opname == 'direct_call':
called_graph = get_graph(op.args[0], translator)
if called_graph not in malloc_graphs:
continue
varstate = adi.getstate(op.result)
assert len(varstate.creation_points) == 1
crep, = varstate.creation_points
if not crep.escapes and not crep.returns:
malloc_creps[crep] = {}
def analyze(self, op, seen=None, graphinfo=None):
if op.opname == "direct_call":
graph = get_graph(op.args[0], self.translator)
if graph is None:
x = self.analyze_external_call(op, seen)
if self.verbose and x:
print '\tanalyze_external_call %s: %r' % (op, x)
return x
x = self.analyze_direct_call(graph, seen)
if self.verbose and x:
print '\tanalyze_direct_call(%s): %r' % (graph, x)
return x
elif op.opname == "indirect_call":
graphs = op.args[-1].value
if graphs is None:
if self.verbose:
print '\t%s to unknown' % (op,)
return self.top_result()
x = self.analyze_indirect_call(graphs, seen)
if self.verbose and x:
print '\tanalyze_indirect_call(%s): %r' % (graphs, x)
return x
elif op.opname == "oosend":
name = op.args[0].value
TYPE = op.args[1].concretetype
_, meth = TYPE._lookup(name)
graph = getattr(meth, 'graph', None)
if graph is None:
return self.analyze_external_method(op, TYPE, meth)
return self.analyze_oosend(TYPE, name, seen)
x = self.analyze_simple_operation(op, graphinfo)
if self.verbose and x:
print '\t%s: %r' % (op, x)
return x
def collect_called_graphs(graph, translator, include_oosend=True):
graphs_or_something = {}
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
graph = get_graph(op.args[0], translator)
if graph is not None:
graphs_or_something[graph] = True
else:
graphs_or_something[op.args[0]] = True
if op.opname == "indirect_call":
graphs = op.args[-1].value
if graphs is None:
graphs_or_something[op.args[0]] = True
else:
for graph in graphs:
graphs_or_something[graph] = True
if op.opname == 'oosend' and include_oosend:
meth = get_meth_from_oosend(op)
if hasattr(meth, 'graph'):
key = meth.graph
elif hasattr(meth, '_can_raise'):
key = CanRaise(meth._can_raise)
else:
key = op.args[0]
graphs_or_something[key] = True
return graphs_or_something
def collect_called_graphs(graph, translator, include_oosend=True):
graphs_or_something = {}
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
graph = get_graph(op.args[0], translator)
if graph is not None:
graphs_or_something[graph] = True
else:
graphs_or_something[op.args[0]] = True
if op.opname == "indirect_call":
graphs = op.args[-1].value
if graphs is None:
graphs_or_something[op.args[0]] = True
else:
for graph in graphs:
graphs_or_something[graph] = True
if op.opname == 'oosend' and include_oosend:
meth = get_meth_from_oosend(op)
if hasattr(meth, 'graph'):
key = meth.graph
elif hasattr(meth, '_can_raise'):
key = CanRaise(meth._can_raise)
else:
key = op.args[0]
graphs_or_something[key] = True
return graphs_or_something
def find_malloc_creps(graph, adi, translator, malloc_graphs):
# mapping from malloc creation point to graphs that it flows into
malloc_creps = {}
# find all mallocs that don't escape
for block, op in graph.iterblockops():
if op.opname == 'malloc':
STRUCT = op.args[0].value
# must not remove mallocs of structures that have a RTTI with a destructor
flags = op.args[1].value
if flags != {'flavor': 'gc'}:
continue
try:
destr_ptr = lltype.getRuntimeTypeInfo(
STRUCT)._obj.destructor_funcptr
if destr_ptr:
continue
except (ValueError, AttributeError), e:
pass
varstate = adi.getstate(op.result)
assert len(varstate.creation_points) == 1
crep, = varstate.creation_points
if not crep.escapes and not crep.returns:
malloc_creps[crep] = {}
if op.opname == 'direct_call':
called_graph = get_graph(op.args[0], translator)
if called_graph not in malloc_graphs:
continue
varstate = adi.getstate(op.result)
assert len(varstate.creation_points) == 1
crep, = varstate.creation_points
if not crep.escapes and not crep.returns:
malloc_creps[crep] = {}
def is_malloc_like(adi, graph, seen):
if graph in seen:
return seen[graph]
return_state = adi.getstate(graph.getreturnvar())
if return_state is None or len(return_state.creation_points) != 1:
seen[graph] = False
return False
crep, = return_state.creation_points
if crep.escapes:
seen[graph] = False
return False
if crep.creation_method in ["malloc", "malloc_varsize"]:
assert crep.returns
seen[graph] = True
return True
if crep.creation_method == "direct_call":
subgraph = get_graph(crep.op.args[0], adi.translation_context)
if subgraph is None:
seen[graph] = False
return False
res = is_malloc_like(adi, subgraph, seen)
seen[graph] = res
return res
seen[graph] = False
return False
def walkgraph(graph):
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
print op
subgraph = get_graph(op.args[0], t)
if subgraph is None:
found.append(op)
else:
walkgraph(subgraph)
def _find_calls_from(translator, graph):
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
called_graph = get_graph(op.args[0], translator)
if called_graph is not None:
yield block, called_graph
if op.opname == "indirect_call":
graphs = op.args[-1].value
if graphs is not None:
for called_graph in graphs:
yield block, called_graph
def analyze(self, op, seen=None):
if op.opname == "direct_call":
graph = get_graph(op.args[0], self.translator)
if graph is None:
return self.analyze_external_call(op)
return self.analyze_direct_call(graph, seen)
elif op.opname == "indirect_call":
if op.args[-1].value is None:
return True
return self.analyze_indirect_call(op.args[-1].value, seen)
if self.operation_is_true(op):
return True
def _find_calls_from(translator, graph):
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
called_graph = get_graph(op.args[0], translator)
if called_graph is not None:
yield block, called_graph
if op.opname == "indirect_call":
graphs = op.args[-1].value
if graphs is not None:
for called_graph in graphs:
yield block, called_graph
def get_statistics(graph, translator, save_per_graph_details=None, ignore_stack_checks=False):
seen_graphs = {}
stack = [graph]
num_graphs = 0
num_blocks = 0
num_ops = 0
num_mallocs = 0
per_graph = {}
while stack:
graph = stack.pop()
if graph in seen_graphs:
continue
seen_graphs[graph] = True
num_graphs += 1
old_num_blocks = num_blocks
old_num_ops = num_ops
old_num_mallocs = num_mallocs
for block in graph.iterblocks():
num_blocks += 1
for op in block.operations:
if op.opname == "direct_call":
called_graph = get_graph(op.args[0], translator)
if called_graph is not None and ignore_stack_checks:
if called_graph.name.startswith('ll_stack_check'):
continue
if called_graph is not None:
stack.append(called_graph)
elif op.opname == "indirect_call":
called_graphs = op.args[-1].value
if called_graphs is not None:
stack.extend(called_graphs)
elif op.opname.startswith("malloc"):
num_mallocs += 1
num_ops += 1
per_graph[graph] = (num_blocks-old_num_blocks, num_ops-old_num_ops, num_mallocs-old_num_mallocs)
if save_per_graph_details:
details = []
for graph, (nblocks, nops, nmallocs) in per_graph.iteritems():
try:
code = graph.func.func_code.co_code
except AttributeError:
code = "None"
hash = md5(code).hexdigest()
details.append((hash, graph.name, nblocks, nops, nmallocs))
details.sort()
f = open(save_per_graph_details, "w")
try:
for hash, name, nblocks, nops, nmallocs in details:
print >>f, hash, name, nblocks, nops, nmallocs
finally:
f.close()
return num_graphs, num_blocks, num_ops, num_mallocs
def find_calls_from(translator, graph):
if getattr(getattr(graph, "func", None), "suggested_primitive", False):
return
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
called_graph = get_graph(op.args[0], translator)
if called_graph is not None:
yield block, called_graph
if op.opname == "indirect_call":
graphs = op.args[-1].value
if graphs is not None:
for called_graph in graphs:
yield block, called_graph
def test_get_graph():
import os
def list_basic_ops(i, j):
l = [1, 2, 3]
l.insert(0, 42)
del l[1]
l.append(i)
listlen = len(l)
l.extend(l)
del l[listlen:]
l += [5, 6]
l[1] = i
return l[j]
def external_function():
return os.system("ls")
graph, t = translate(list_basic_ops, [int, int], False)
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
print op
graph = get_graph(op.args[0], t)
assert graph is not None
# an external function in RPython turns currently into
# a call to a wrapper function which itself contains the
# real call to a graph-less external ll function, so
# we check recursively
graph, t = translate(external_function, [], False)
found = []
def walkgraph(graph):
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
print op
subgraph = get_graph(op.args[0], t)
if subgraph is None:
found.append(op)
else:
walkgraph(subgraph)
walkgraph(graph)
assert len(found) == 1
def analyze(self, op, seen=None):
if op.opname == "direct_call":
graph = get_graph(op.args[0], self.translator)
if graph is None:
return self.analyze_external_call(op)
return self.analyze_direct_call(graph, seen)
elif op.opname == "indirect_call":
if op.args[-1].value is None:
return self.top_result()
return self.analyze_indirect_call(op.args[-1].value, seen)
elif op.opname == "oosend":
name = op.args[0].value
TYPE = op.args[1].concretetype
_, meth = TYPE._lookup(name)
graph = getattr(meth, 'graph', None)
if graph is None:
return self.analyze_external_method(op, TYPE, meth)
return self.analyze_oosend(TYPE, name, seen)
return self.analyze_simple_operation(op)
def remove_tail_calls_to_self(translator, graph):
entrymap = mkentrymap(graph)
changed = False
for link in entrymap[graph.returnblock]:
block = link.prevblock
if (len(block.exits) == 1 and len(block.operations) > 0
and block.operations[-1].opname == 'direct_call'
and block.operations[-1].result == link.args[0]):
call = get_graph(block.operations[-1].args[0], translator)
print "getgraph", graph
if graph is graph:
_remove_tail_call(translator, graph, block)
changed = True
if changed:
from pypy.translator import simplify
checkgraph(graph)
simplify.remove_identical_vars(graph)
simplify.eliminate_empty_blocks(graph)
simplify.join_blocks(graph)
def analyze(self, op, seen=None):
if op.opname == "direct_call":
graph = get_graph(op.args[0], self.translator)
if graph is None:
return self.analyze_external_call(op)
return self.analyze_direct_call(graph, seen)
elif op.opname == "indirect_call":
if op.args[-1].value is None:
return self.top_result()
return self.analyze_indirect_call(op.args[-1].value, seen)
elif op.opname == "oosend":
name = op.args[0].value
TYPE = op.args[1].concretetype
_, meth = TYPE._lookup(name)
graph = getattr(meth, 'graph', None)
if graph is None:
return self.analyze_external_method(op, TYPE, meth)
return self.analyze_oosend(TYPE, name, seen)
return self.analyze_simple_operation(op)
def remove_tail_calls_to_self(translator, graph):
entrymap = mkentrymap(graph)
changed = False
for link in entrymap[graph.returnblock]:
block = link.prevblock
if (len(block.exits) == 1 and
len(block.operations) > 0 and
block.operations[-1].opname == 'direct_call' and
block.operations[-1].result == link.args[0]):
call = get_graph(block.operations[-1].args[0], translator)
print "getgraph", graph
if graph is graph:
_remove_tail_call(translator, graph, block)
changed = True
if changed:
from pypy.translator import simplify
checkgraph(graph)
simplify.remove_identical_vars(graph)
simplify.eliminate_empty_blocks(graph)
simplify.join_blocks(graph)
def op_direct_call(self, op, function, *args):
graph = get_graph(op.args[0], self.translation_context)
if graph is None:
for arg in args:
if arg is None:
continue
# an external function can escape every parameter:
self.escapes(arg)
funcargs = [None] * len(args)
else:
result, funcargs = self.schedule_function(graph)
assert len(args) == len(funcargs)
for localarg, funcarg in zip(args, funcargs):
if localarg is None:
assert funcarg is None
continue
if funcarg is not None:
self.register_state_dependency(localarg, funcarg)
if isonheap(op.result):
# assume that a call creates a new value
return VarState(self.get_creationpoint(op.result, "direct_call", op))
def test_get_graph():
import os
def list_basic_ops(i, j):
l = [1,2,3]
l.insert(0, 42)
del l[1]
l.append(i)
listlen = len(l)
l.extend(l)
del l[listlen:]
l += [5,6]
l[1] = i
return l[j]
def external_function():
return os.system("ls")
graph, t = translate(list_basic_ops, [int, int], False)
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
print op
graph = get_graph(op.args[0], t)
assert graph is not None
# an external function in RPython turns currently into
# a call to a wrapper function which itself contains the
# real call to a graph-less external ll function, so
# we check recursively
graph, t = translate(external_function, [], False)
found = []
def walkgraph(graph):
for block in graph.iterblocks():
for op in block.operations:
if op.opname == "direct_call":
print op
subgraph = get_graph(op.args[0], t)
if subgraph is None:
found.append(op)
else:
walkgraph(subgraph)
walkgraph(graph)
assert len(found) == 1
def op_direct_call(self, op, function, *args):
graph = get_graph(op.args[0], self.translation_context)
if graph is None:
for arg in args:
if arg is None:
continue
# an external function can escape every parameter:
self.escapes(arg)
funcargs = [None] * len(args)
else:
result, funcargs = self.schedule_function(graph)
assert len(args) == len(funcargs)
for localarg, funcarg in zip(args, funcargs):
if localarg is None:
assert funcarg is None
continue
if funcarg is not None:
self.register_state_dependency(localarg, funcarg)
if isonheap(op.result):
# assume that a call creates a new value
return VarState(
self.get_creationpoint(op.result, "direct_call", op))
def find_calls_where_creps_go(interesting_creps, graph, adi, translator, seen):
#print "find_calls_where_creps_go", interesting_creps, graph.name
#print seen
# drop creps that are merged with another creation point
for block in graph.iterblocks():
for var in block.getvariables():
varstate = adi.getstate(var)
if varstate is None:
continue
for crep in varstate.creation_points:
if crep in interesting_creps:
if len(varstate.creation_points) != 1:
del interesting_creps[crep]
break
# drop creps that are passed into an indirect_call
for block, op in graph.iterblockops():
if not interesting_creps:
return
if op.opname == "indirect_call":
for var in op.args[:-1]:
varstate = adi.getstate(var)
if varstate is None:
continue
for crep in varstate.creation_points:
if crep in interesting_creps:
del interesting_creps[crep]
elif op.opname == "direct_call":
#print op, interesting_creps
called_graph = simplify.get_graph(op.args[0], translator)
interesting = {}
for i, var in enumerate(op.args[1:]):
#print i, var,
varstate = adi.getstate(var)
if varstate is None:
#print "no varstate"
continue
if len(varstate.creation_points) == 1:
crep = varstate.creation_points.keys()[0]
if crep not in interesting_creps:
#print "not interesting"
continue
if called_graph is None:
del interesting_creps[crep]
#print "graph not found"
continue
if (called_graph, i) in seen:
seen[(called_graph, i)][graph] = True
#print "seen already"
else:
#print "taking", crep
seen[(called_graph, i)] = {graph: True}
arg = called_graph.startblock.inputargs[i]
argstate = adi.getstate(arg)
argcrep = [
c for c in argstate.creation_points
if c.creation_method == "arg"
][0]
interesting[argcrep] = True
#print interesting
if interesting:
find_calls_where_creps_go(interesting, called_graph, adi,
translator, seen)
return interesting_creps
def find_calls_where_creps_go(interesting_creps, graph, adi, translator, seen):
#print "find_calls_where_creps_go", interesting_creps, graph.name
#print seen
# drop creps that are merged with another creation point
for block in graph.iterblocks():
for var in block.getvariables():
varstate = adi.getstate(var)
if varstate is None:
continue
for crep in varstate.creation_points:
if crep in interesting_creps:
if len(varstate.creation_points) != 1:
del interesting_creps[crep]
break
# drop creps that are passed into an indirect_call
for block, op in graph.iterblockops():
if not interesting_creps:
return
if op.opname == "indirect_call":
for var in op.args[:-1]:
varstate = adi.getstate(var)
if varstate is None:
continue
for crep in varstate.creation_points:
if crep in interesting_creps:
del interesting_creps[crep]
elif op.opname == "direct_call":
#print op, interesting_creps
called_graph = get_graph(op.args[0], translator)
interesting = {}
if called_graph is None:
graphvars = [None] * len(op.args)
else:
graphvars = called_graph.getargs() + [
called_graph.getreturnvar()
]
for var, graphvar in zip(op.args[1:] + [op.result], graphvars):
varstate = adi.getstate(var)
if varstate is None:
#print "no varstate"
continue
if len(varstate.creation_points) == 1:
crep, = varstate.creation_points
if crep not in interesting_creps:
#print "not interesting"
continue
if called_graph is None:
del interesting_creps[crep]
#print "graph not found"
continue
if called_graph in seen:
seen[called_graph][graph] = True
#print "seen already"
else:
#print "taking", crep
seen[called_graph] = {graph: True}
argstate = adi.getstate(graphvar)
argcrep, = argstate.creation_points
interesting[argcrep] = True
#print interesting
if interesting:
find_calls_where_creps_go(interesting, called_graph, adi,
translator, seen)
return interesting_creps
def find_calls_where_creps_go(interesting_creps, graph, adi,
translator, seen):
#print "find_calls_where_creps_go", interesting_creps, graph.name
#print seen
# drop creps that are merged with another creation point
for block in graph.iterblocks():
for var in block.getvariables():
varstate = adi.getstate(var)
if varstate is None:
continue
for crep in varstate.creation_points:
if crep in interesting_creps:
if len(varstate.creation_points) != 1:
del interesting_creps[crep]
break
# drop creps that are passed into an indirect_call
for block, op in graph.iterblockops():
if not interesting_creps:
return
if op.opname == "indirect_call":
for var in op.args[:-1]:
varstate = adi.getstate(var)
if varstate is None:
continue
for crep in varstate.creation_points:
if crep in interesting_creps:
del interesting_creps[crep]
elif op.opname == "direct_call":
#print op, interesting_creps
called_graph = get_graph(op.args[0], translator)
interesting = {}
if called_graph is None:
graphvars = [None] * len(op.args)
else:
graphvars = called_graph.getargs() + [called_graph.getreturnvar()]
for var, graphvar in zip(op.args[1:] + [op.result], graphvars):
varstate = adi.getstate(var)
if varstate is None:
#print "no varstate"
continue
if len(varstate.creation_points) == 1:
crep, = varstate.creation_points
if crep not in interesting_creps:
#print "not interesting"
continue
if called_graph is None:
del interesting_creps[crep]
#print "graph not found"
continue
if called_graph in seen:
seen[called_graph][graph] = True
#print "seen already"
else:
#print "taking", crep
seen[called_graph] = {graph: True}
argstate = adi.getstate(graphvar)
argcrep, = argstate.creation_points
interesting[argcrep] = True
#print interesting
if interesting:
find_calls_where_creps_go(interesting, called_graph,
adi, translator, seen)
return interesting_creps
def find_calls_where_creps_go(interesting_creps, graph, adi,
translator, seen):
#print "find_calls_where_creps_go", interesting_creps, graph.name
#print seen
# drop creps that are merged with another creation point
for block in graph.iterblocks():
for var in block.getvariables():
varstate = adi.getstate(var)
if varstate is None:
continue
for crep in varstate.creation_points:
if crep in interesting_creps:
if len(varstate.creation_points) != 1:
del interesting_creps[crep]
break
# drop creps that are passed into an indirect_call
for block, op in graph.iterblockops():
if not interesting_creps:
return
if op.opname == "indirect_call":
for var in op.args[:-1]:
varstate = adi.getstate(var)
if varstate is None:
continue
for crep in varstate.creation_points:
if crep in interesting_creps:
del interesting_creps[crep]
elif op.opname == "direct_call":
#print op, interesting_creps
called_graph = simplify.get_graph(op.args[0], translator)
interesting = {}
for i, var in enumerate(op.args[1:]):
#print i, var,
varstate = adi.getstate(var)
if varstate is None:
#print "no varstate"
continue
if len(varstate.creation_points) == 1:
crep = varstate.creation_points.keys()[0]
if crep not in interesting_creps:
#print "not interesting"
continue
if called_graph is None:
del interesting_creps[crep]
#print "graph not found"
continue
if (called_graph, i) in seen:
seen[(called_graph, i)][graph] = True
#print "seen already"
else:
#print "taking", crep
seen[(called_graph, i)] = {graph: True}
arg = called_graph.startblock.inputargs[i]
argstate = adi.getstate(arg)
argcrep = [c for c in argstate.creation_points
if c.creation_method == "arg"][0]
interesting[argcrep] = True
#print interesting
if interesting:
find_calls_where_creps_go(interesting, called_graph,
adi, translator, seen)
return interesting_creps
