def remove_identical_vars(graph):
"""When the same variable is passed multiple times into the next block,
pass it only once. This enables further optimizations by the annotator,
which otherwise doesn't realize that tests performed on one of the copies
of the variable also affect the other."""
# This algorithm is based on DataFlowFamilyBuilder, used as a
# "phi node remover" (in the SSA sense). 'variable_families' is a
# UnionFind object that groups variables by families; variables from the
# same family can be identified, and if two input arguments of a block
# end up in the same family, then we really remove one of them in favor
# of the other.
#
# The idea is to identify as much variables as possible by trying
# iteratively two kinds of phi node removal:
#
# * "vertical", by identifying variables from different blocks, when
# we see that a value just flows unmodified into the next block without
# needing any merge (this is what backendopt.ssa.SSI_to_SSA() would do
# as well);
#
# * "horizontal", by identifying two input variables of the same block,
# when these two variables' phi nodes have the same argument -- i.e.
# when for all possible incoming paths they would get twice the same
# value (this is really the purpose of remove_identical_vars()).
#
if simplify_disabled(graph): return
from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder
builder = DataFlowFamilyBuilder(graph)
variable_families = builder.get_variable_families() # vertical removal
while True:
if not builder.merge_identical_phi_nodes(): # horizontal removal
break
if not builder.complete(): # vertical removal
break
for block, links in mkentrymap(graph).items():
if block is graph.startblock:
continue
renaming = {}
family2blockvar = {}
kills = []
for i, v in enumerate(block.inputargs):
v1 = variable_families.find_rep(v)
if v1 in family2blockvar:
# already seen -- this variable can be shared with the
# previous one
renaming[v] = family2blockvar[v1]
kills.append(i)
else:
family2blockvar[v1] = v
if renaming:
block.renamevariables(renaming)
# remove the now-duplicate input variables
kills.reverse() # starting from the end
for i in kills:
del block.inputargs[i]
for link in links:
del link.args[i]
def remove_identical_vars(graph):
"""When the same variable is passed multiple times into the next block,
pass it only once. This enables further optimizations by the annotator,
which otherwise doesn't realize that tests performed on one of the copies
of the variable also affect the other."""
# This algorithm is based on DataFlowFamilyBuilder, used as a
# "phi node remover" (in the SSA sense). 'variable_families' is a
# UnionFind object that groups variables by families; variables from the
# same family can be identified, and if two input arguments of a block
# end up in the same family, then we really remove one of them in favor
# of the other.
#
# The idea is to identify as much variables as possible by trying
# iteratively two kinds of phi node removal:
#
# * "vertical", by identifying variables from different blocks, when
# we see that a value just flows unmodified into the next block without
# needing any merge (this is what backendopt.ssa.SSI_to_SSA() would do
# as well);
#
# * "horizontal", by identifying two input variables of the same block,
# when these two variables' phi nodes have the same argument -- i.e.
# when for all possible incoming paths they would get twice the same
# value (this is really the purpose of remove_identical_vars()).
#
from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder
builder = DataFlowFamilyBuilder(graph)
variable_families = builder.get_variable_families() # vertical removal
while True:
if not builder.merge_identical_phi_nodes(): # horizontal removal
break
if not builder.complete(): # vertical removal
break
for block, links in mkentrymap(graph).items():
if block is graph.startblock:
continue
renaming = {}
family2blockvar = {}
kills = []
for i, v in enumerate(block.inputargs):
v1 = variable_families.find_rep(v)
if v1 in family2blockvar:
# already seen -- this variable can be shared with the
# previous one
renaming[v] = family2blockvar[v1]
kills.append(i)
else:
family2blockvar[v1] = v
if renaming:
block.renamevariables(renaming)
# remove the now-duplicate input variables
kills.reverse() # starting from the end
for i in kills:
del block.inputargs[i]
for link in links:
del link.args[i]
def detect_list_comprehension(graph):
"""Look for the pattern: Replace it with marker operations:
v0 = newlist()
v2 = newlist() v1 = hint(v0, iterable, {'maxlength'})
loop start loop start
... ...
exactly one append per loop v1.append(..)
and nothing else done with v2
... ...
loop end v2 = hint(v1, {'fence'})
"""
# NB. this assumes RPythonicity: we can only iterate over something
# that has a len(), and this len() cannot change as long as we are
# using the iterator.
from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder
builder = DataFlowFamilyBuilder(graph)
variable_families = builder.get_variable_families()
c_append = Constant('append')
newlist_v = {}
iter_v = {}
append_v = []
loopnextblocks = []
# collect relevant operations based on the family of their result
for block in graph.iterblocks():
if (len(block.operations) == 1 and
block.operations[0].opname == 'next' and
block.exitswitch == c_last_exception and
len(block.exits) >= 2):
cases = [link.exitcase for link in block.exits]
if None in cases and StopIteration in cases:
# it's a straightforward loop start block
loopnextblocks.append((block, block.operations[0].args[0]))
continue
for op in block.operations:
if op.opname == 'newlist' and not op.args:
vlist = variable_families.find_rep(op.result)
newlist_v[vlist] = block
if op.opname == 'iter':
viter = variable_families.find_rep(op.result)
iter_v[viter] = block
loops = []
for block, viter in loopnextblocks:
viterfamily = variable_families.find_rep(viter)
if viterfamily in iter_v:
# we have a next(viter) operation where viter comes from a
# single known iter() operation. Check that the iter()
# operation is in the block just before.
iterblock = iter_v[viterfamily]
if (len(iterblock.exits) == 1 and iterblock.exitswitch is None
and iterblock.exits[0].target is block):
# yes - simple case.
loops.append((block, iterblock, viterfamily))
if not newlist_v or not loops:
return
# XXX works with Python >= 2.4 only: find calls to append encoded as
# getattr/simple_call pairs, as produced by the LIST_APPEND bytecode.
for block in graph.iterblocks():
for i in range(len(block.operations)-1):
op = block.operations[i]
if op.opname == 'getattr' and op.args[1] == c_append:
vlist = variable_families.find_rep(op.args[0])
if vlist in newlist_v:
op2 = block.operations[i+1]
if (op2.opname == 'simple_call' and len(op2.args) == 2
and op2.args[0] is op.result):
append_v.append((op.args[0], op.result, block))
if not append_v:
return
detector = ListComprehensionDetector(graph, loops, newlist_v,
variable_families)
graphmutated = False
for location in append_v:
if graphmutated:
# new variables introduced, must restart the whole process
return detect_list_comprehension(graph)
try:
detector.run(*location)
except DetectorFailed:
pass
else:
graphmutated = True
def detect_list_comprehension(graph):
"""Look for the pattern: Replace it with marker operations:
v0 = newlist()
v2 = newlist() v1 = hint(v0, iterable, {'maxlength'})
loop start loop start
... ...
exactly one append per loop v1.append(..)
and nothing else done with v2
... ...
loop end v2 = hint(v1, {'fence'})
"""
# NB. this assumes RPythonicity: we can only iterate over something
# that has a len(), and this len() cannot change as long as we are
# using the iterator.
if simplify_disabled(graph): return
from pypy.translator.backendopt.ssa import DataFlowFamilyBuilder
builder = DataFlowFamilyBuilder(graph)
variable_families = builder.get_variable_families()
c_append = Constant('append')
newlist_v = {}
iter_v = {}
append_v = []
loopnextblocks = []
# collect relevant operations based on the family of their result
for block in graph.iterblocks():
if (len(block.operations) == 1 and block.operations[0].opname == 'next'
and block.exitswitch == c_last_exception
and len(block.exits) >= 2):
cases = [link.exitcase for link in block.exits]
if None in cases and StopIteration in cases:
# it's a straightforward loop start block
loopnextblocks.append((block, block.operations[0].args[0]))
continue
for op in block.operations:
if op.opname == 'newlist' and not op.args:
vlist = variable_families.find_rep(op.result)
newlist_v[vlist] = block
if op.opname == 'iter':
viter = variable_families.find_rep(op.result)
iter_v[viter] = block
loops = []
for block, viter in loopnextblocks:
viterfamily = variable_families.find_rep(viter)
if viterfamily in iter_v:
# we have a next(viter) operation where viter comes from a
# single known iter() operation. Check that the iter()
# operation is in the block just before.
iterblock = iter_v[viterfamily]
if (len(iterblock.exits) == 1 and iterblock.exitswitch is None
and iterblock.exits[0].target is block):
# yes - simple case.
loops.append((block, iterblock, viterfamily))
if not newlist_v or not loops:
return
# XXX works with Python >= 2.4 only: find calls to append encoded as
# getattr/simple_call pairs, as produced by the LIST_APPEND bytecode.
for block in graph.iterblocks():
for i in range(len(block.operations) - 1):
op = block.operations[i]
if op.opname == 'getattr' and op.args[1] == c_append:
vlist = variable_families.find_rep(op.args[0])
if vlist in newlist_v:
op2 = block.operations[i + 1]
if (op2.opname == 'simple_call' and len(op2.args) == 2
and op2.args[0] is op.result):
append_v.append((op.args[0], op.result, block))
if not append_v:
return
detector = ListComprehensionDetector(graph, loops, newlist_v,
variable_families)
graphmutated = False
for location in append_v:
if graphmutated:
# new variables introduced, must restart the whole process
return detect_list_comprehension(graph)
try:
detector.run(*location)
except DetectorFailed:
pass
else:
graphmutated = True
def find_inner_loops(graph, check_exitswitch_type=None):
"""Enumerate what look like the innermost loops of the graph.
Returns a list of non-overlapping Loop() instances.
"""
# Heuristic (thanks Stakkars for the idea):
# to find the "best" cycle to pick,
#
# * look for variables that don't change over the whole cycle
#
# * cycles with _more_ of them are _inside_ cycles with less of them,
# because any variable that doesn't change over an outer loop will
# not change over an inner loop either, and on the other hand the
# outer loop is likely to use and modify variables that remain
# constant over the inner loop.
#
# * if the numbers are the same, fall back to a more arbitrary
# measure: loops involving less blocks may be more important
# to optimize
#
# * in a cycle, which block is the head of the loop? Somewhat
# arbitrarily we pick the first Bool-switching block that has
# two exits. The "first" means the one closest to the
# startblock of the graph.
#
startdistance = {} # {block: distance-from-startblock}
pending = [graph.startblock]
edge_list = []
dist = 0
while pending:
newblocks = []
for block in pending:
if block not in startdistance:
startdistance[block] = dist
for link in block.exits:
newblocks.append(link.target)
edge = graphlib.Edge(block, link.target)
edge.link = link
edge_list.append(edge)
dist += 1
pending = newblocks
vertices = startdistance
edges = graphlib.make_edge_dict(edge_list)
cycles = graphlib.all_cycles(graph.startblock, vertices, edges)
loops = []
variable_families = None
for cycle in cycles:
# find the headblock
candidates = []
for i in range(len(cycle)):
block = cycle[i].source
v = block.exitswitch
if isinstance(v, Variable) and len(block.exits) == 2:
if getattr(v, 'concretetype', None) is check_exitswitch_type:
dist = startdistance[block]
candidates.append((dist, i))
if not candidates:
continue
_, i = min(candidates)
links = [edge.link for edge in cycle[i:] + cycle[:i]]
loop = Loop(cycle[i].source, links)
# count the variables that remain constant across the cycle,
# detected as having its SSA family present across all blocks.
if variable_families is None:
dffb = DataFlowFamilyBuilder(graph)
variable_families = dffb.get_variable_families()
num_loop_constants = 0
for v in loop.headblock.inputargs:
v = variable_families.find_rep(v)
for link in loop.links:
block1 = link.target
for v1 in block1.inputargs:
v1 = variable_families.find_rep(v1)
if v1 is v:
break # ok, found in this block
else:
break # not found in this block, fail
else:
# found in all blocks, this variable is a loop constant
num_loop_constants += 1
# smaller keys are "better"
key = (-num_loop_constants, # maximize num_loop_constants
len(cycle)) # minimize len(cycle)
loops.append((key, loop))
loops.sort()
# returns 'loops' without overlapping blocks
result = []
blocks_seen = {}
for key, loop in loops:
for link in loop.links:
if link.target in blocks_seen:
break # overlapping
else:
# non-overlapping
result.append(loop)
for link in loop.links:
blocks_seen[link.target] = True
return result
def find_inner_loops(graph, check_exitswitch_type=None):
"""Enumerate what look like the innermost loops of the graph.
Returns a list of non-overlapping Loop() instances.
"""
# Heuristic (thanks Stakkars for the idea):
# to find the "best" cycle to pick,
#
# * look for variables that don't change over the whole cycle
#
# * cycles with _more_ of them are _inside_ cycles with less of them,
# because any variable that doesn't change over an outer loop will
# not change over an inner loop either, and on the other hand the
# outer loop is likely to use and modify variables that remain
# constant over the inner loop.
#
# * if the numbers are the same, fall back to a more arbitrary
# measure: loops involving less blocks may be more important
# to optimize
#
# * in a cycle, which block is the head of the loop? Somewhat
# arbitrarily we pick the first Bool-switching block that has
# two exits. The "first" means the one closest to the
# startblock of the graph.
#
startdistance = {} # {block: distance-from-startblock}
pending = [graph.startblock]
edge_list = []
dist = 0
while pending:
newblocks = []
for block in pending:
if block not in startdistance:
startdistance[block] = dist
for link in block.exits:
newblocks.append(link.target)
edge = graphlib.Edge(block, link.target)
edge.link = link
edge_list.append(edge)
dist += 1
pending = newblocks
vertices = startdistance
edges = graphlib.make_edge_dict(edge_list)
cycles = graphlib.all_cycles(graph.startblock, vertices, edges)
loops = []
variable_families = None
for cycle in cycles:
# find the headblock
candidates = []
for i in range(len(cycle)):
block = cycle[i].source
v = block.exitswitch
if isinstance(v, Variable) and len(block.exits) == 2:
if getattr(v, 'concretetype', None) is check_exitswitch_type:
dist = startdistance[block]
candidates.append((dist, i))
if not candidates:
continue
_, i = min(candidates)
links = [edge.link for edge in cycle[i:] + cycle[:i]]
loop = Loop(cycle[i].source, links)
# count the variables that remain constant across the cycle,
# detected as having its SSA family present across all blocks.
if variable_families is None:
dffb = DataFlowFamilyBuilder(graph)
variable_families = dffb.get_variable_families()
num_loop_constants = 0
for v in loop.headblock.inputargs:
v = variable_families.find_rep(v)
for link in loop.links:
block1 = link.target
for v1 in block1.inputargs:
v1 = variable_families.find_rep(v1)
if v1 is v:
break # ok, found in this block
else:
break # not found in this block, fail
else:
# found in all blocks, this variable is a loop constant
num_loop_constants += 1
# smaller keys are "better"
key = (
-num_loop_constants, # maximize num_loop_constants
len(cycle)) # minimize len(cycle)
loops.append((key, loop))
loops.sort()
# returns 'loops' without overlapping blocks
result = []
blocks_seen = {}
for key, loop in loops:
for link in loop.links:
if link.target in blocks_seen:
break # overlapping
else:
# non-overlapping
result.append(loop)
for link in loop.links:
blocks_seen[link.target] = True
return result