def test_frozen_user_class2(self): class C: def __add__(self, other): return 4 def _freeze_(self): return True c = C() d = C() def f(): return c + d graph = self.codetest(f) results = [] def visit(link): if isinstance(link, Link): if link.target == graph.returnblock: results.extend(link.args) traverse(visit, graph) assert results == [Constant(4)]
def remove_assertion_errors(graph): """Remove branches that go directly to raising an AssertionError, assuming that AssertionError shouldn't occur at run-time. Note that this is how implicit exceptions are removed (see _implicit_ in flowcontext.py). """ if simplify_disabled(graph): return def visit(block): if isinstance(block, Block): for i in range(len(block.exits) - 1, -1, -1): exit = block.exits[i] if not (exit.target is graph.exceptblock and exit.args[0] == Constant(AssertionError)): continue # can we remove this exit without breaking the graph? if len(block.exits) < 2: break if block.exitswitch == c_last_exception: if exit.exitcase is None: break if len(block.exits) == 2: # removing the last non-exceptional exit block.exitswitch = None exit.exitcase = None # remove this exit lst = list(block.exits) del lst[i] block.recloseblock(*lst) traverse(visit, graph)
def remove_assertion_errors(graph): """Remove branches that go directly to raising an AssertionError, assuming that AssertionError shouldn't occur at run-time. Note that this is how implicit exceptions are removed (see _implicit_ in flowcontext.py). """ def visit(block): if isinstance(block, Block): for i in range(len(block.exits)-1, -1, -1): exit = block.exits[i] if not (exit.target is graph.exceptblock and exit.args[0] == Constant(AssertionError)): continue # can we remove this exit without breaking the graph? if len(block.exits) < 2: break if block.exitswitch == c_last_exception: if exit.exitcase is None: break if len(block.exits) == 2: # removing the last non-exceptional exit block.exitswitch = None exit.exitcase = None # remove this exit lst = list(block.exits) del lst[i] block.recloseblock(*lst) traverse(visit, graph)
def test_jump_target_specialization(self): x = self.codetest(self.jump_target_specialization) def visitor(node): if isinstance(node, Block): for op in node.operations: assert op.opname != 'mul', "mul should have disappeared" traverse(visitor, x)
def test_frozen_user_class1(self): class C: def __nonzero__(self): return True def _freeze_(self): return True c = C() def f(): if c: return 1 else: return 2 graph = self.codetest(f) results = [] def visit(link): if isinstance(link, Link): if link.target == graph.returnblock: results.extend(link.args) traverse(visit, graph) assert len(results) == 1
def eliminate_empty_blocks(graph): """Eliminate basic blocks that do not contain any operations. When this happens, we need to replace the preceeding link with the following link. Arguments of the links should be updated.""" if simplify_disabled(graph): return def visit(link): if isinstance(link, Link): while not link.target.operations: block1 = link.target if block1.exitswitch is not None: break if not block1.exits: break exit = block1.exits[0] assert block1 is not exit.target, ( "the graph contains an empty infinite loop") outputargs = [] for v in exit.args: if isinstance(v, Variable): # this variable is valid in the context of block1 # but it must come from 'link' i = block1.inputargs.index(v) v = link.args[i] outputargs.append(v) link.args = outputargs link.target = exit.target # the while loop above will simplify recursively the new link traverse(visit, graph)
def eliminate_empty_blocks(graph): """Eliminate basic blocks that do not contain any operations. When this happens, we need to replace the preceeding link with the following link. Arguments of the links should be updated.""" def visit(link): if isinstance(link, Link): while not link.target.operations: block1 = link.target if block1.exitswitch is not None: break if not block1.exits: break exit = block1.exits[0] assert block1 is not exit.target, ( "the graph contains an empty infinite loop") outputargs = [] for v in exit.args: if isinstance(v, Variable): # this variable is valid in the context of block1 # but it must come from 'link' i = block1.inputargs.index(v) v = link.args[i] outputargs.append(v) link.args = outputargs link.target = exit.target # the while loop above will simplify recursively the new link traverse(visit, graph)
def remove_dead_exceptions(graph): """Exceptions can be removed if they are unreachable""" clastexc = c_last_exception def issubclassofmember(cls, seq): for member in seq: if member and issubclass(cls, member): return True return False def visit(block): if not (isinstance(block, Block) and block.exitswitch == clastexc): return exits = [] seen = [] for link in block.exits: case = link.exitcase # check whether exceptions are shadowed if issubclassofmember(case, seen): continue # see if the previous case can be merged while len(exits) > 1: prev = exits[-1] if not (issubclass(prev.exitcase, link.exitcase) and prev.target is link.target and prev.args == link.args): break exits.pop() exits.append(link) seen.append(case) block.recloseblock(*exits) traverse(visit, graph)
def test_implicitAttributeError(self): x = self.codetest(self.implicitAttributeError) simplify_graph(x) self.show(x) def cannot_reach_exceptblock(link): if isinstance(link, Link): assert link.target is not x.exceptblock traverse(cannot_reach_exceptblock, x)
def all_operations(self, graph): result = {} def visit(node): if isinstance(node, Block): for op in node.operations: result.setdefault(op.opname, 0) result[op.opname] += 1 traverse(visit, graph) return result
def transform_dead_op_vars(graph, translator=None): """Remove dead operations and variables that are passed over a link but not used in the target block. Input is a graph.""" blocks = {} def visit(block): if isinstance(block, Block): blocks[block] = True traverse(visit, graph) return transform_dead_op_vars_in_blocks(blocks, translator)
def test_reraiseTypeError(self): x = self.codetest(self.reraiseTypeError) simplify_graph(x) self.show(x) found = [] def can_reach_exceptblock(link): if isinstance(link, Link): if link.target is x.exceptblock: found.append(link) traverse(can_reach_exceptblock, x) assert found
def test_reraiseAnything(self): x = self.codetest(self.reraiseAnything) simplify_graph(x) self.show(x) found = {} def find_exceptions(link): if isinstance(link, Link): if link.target is x.exceptblock: assert isinstance(link.args[0], Constant) found[link.args[0].value] = True traverse(find_exceptions, x) assert found == {ValueError: True, ZeroDivisionError: True, OverflowError: True}
def test_reraiseAttributeError(self): x = self.codetest(self.reraiseAttributeError) simplify_graph(x) self.show(x) found_AttributeError = [] def only_raise_AttributeError(link): if isinstance(link, Link): if link.target is x.exceptblock: assert link.args[0] == Constant(AttributeError) found_AttributeError.append(link) traverse(only_raise_AttributeError, x) assert found_AttributeError
def coalesce_is_true(graph): """coalesce paths that go through an is_true and a directly successive is_true both on the same value, transforming the link into the second is_true from the first to directly jump to the correct target out of the second.""" if simplify_disabled(graph): return candidates = [] def has_is_true_exitpath(block): tgts = [] start_op = block.operations[-1] cond_v = start_op.args[0] if block.exitswitch == start_op.result: for exit in block.exits: tgt = exit.target if tgt == block: continue rrenaming = dict(zip(tgt.inputargs, exit.args)) if len(tgt.operations ) == 1 and tgt.operations[0].opname == 'is_true': tgt_op = tgt.operations[0] if tgt.exitswitch == tgt_op.result and rrenaming.get( tgt_op.args[0]) == cond_v: tgts.append((exit.exitcase, tgt)) return tgts def visit(block): if isinstance( block, Block ) and block.operations and block.operations[-1].opname == 'is_true': tgts = has_is_true_exitpath(block) if tgts: candidates.append((block, tgts)) traverse(visit, graph) while candidates: cand, tgts = candidates.pop() newexits = list(cand.exits) for case, tgt in tgts: exit = cand.exits[case] rrenaming = dict(zip(tgt.inputargs, exit.args)) rrenaming[tgt.operations[0].result] = cand.operations[-1].result def rename(v): return rrenaming.get(v, v) newlink = tgt.exits[case].copy(rename) newexits[case] = newlink cand.recloseblock(*newexits) newtgts = has_is_true_exitpath(cand) if newtgts: candidates.append((cand, newtgts))
def test_const_star_call(self): def g(a=1,b=2,c=3): pass def f(): return g(1,*(2,3)) graph = self.codetest(f) call_args = [] def visit(block): if isinstance(block, Block): for op in block.operations: if op.opname == "call_args": call_args.append(op) traverse(visit, graph) assert not call_args
def test_reraiseAnythingDicCase(self): x = self.codetest(self.reraiseAnythingDicCase) simplify_graph(x) self.show(x) found = {} def find_exceptions(link): if isinstance(link, Link): if link.target is x.exceptblock: if isinstance(link.args[0], Constant): found[link.args[0].value] = True else: found[link.exitcase] = None traverse(find_exceptions, x) assert found == {IndexError: True, KeyError: True, Exception: None}
def graph_footprint(graph): class Counter: blocks = 0 links = 0 ops = 0 count = Counter() def visit(block): if isinstance(block, flowmodel.Block): count.blocks += 1 count.ops += len(block.operations) elif isinstance(block, flowmodel.Link): count.links += 1 flowmodel.traverse(visit, graph) return count.blocks, count.links, count.ops
def test_reraiseTypeError(self): x = self.codetest(self.reraiseTypeError) simplify_graph(x) self.show(x) excfound = [] def check(link): if isinstance(link, Link): if link.target is x.exceptblock: excfound.append(link.exitcase) traverse(check, x) assert len(excfound) == 2 excfound.sort() expected = [Exception, TypeError] expected.sort() assert excfound == expected
def coalesce_is_true(graph): """coalesce paths that go through an is_true and a directly successive is_true both on the same value, transforming the link into the second is_true from the first to directly jump to the correct target out of the second.""" candidates = [] def has_is_true_exitpath(block): tgts = [] start_op = block.operations[-1] cond_v = start_op.args[0] if block.exitswitch == start_op.result: for exit in block.exits: tgt = exit.target if tgt == block: continue rrenaming = dict(zip(tgt.inputargs, exit.args)) if len(tgt.operations) == 1 and tgt.operations[0].opname == "is_true": tgt_op = tgt.operations[0] if tgt.exitswitch == tgt_op.result and rrenaming.get(tgt_op.args[0]) == cond_v: tgts.append((exit.exitcase, tgt)) return tgts def visit(block): if isinstance(block, Block) and block.operations and block.operations[-1].opname == "is_true": tgts = has_is_true_exitpath(block) if tgts: candidates.append((block, tgts)) traverse(visit, graph) while candidates: cand, tgts = candidates.pop() newexits = list(cand.exits) for case, tgt in tgts: exit = cand.exits[case] rrenaming = dict(zip(tgt.inputargs, exit.args)) rrenaming[tgt.operations[0].result] = cand.operations[-1].result def rename(v): return rrenaming.get(v, v) newlink = tgt.exits[case].copy(rename) newexits[case] = newlink cand.recloseblock(*newexits) newtgts = has_is_true_exitpath(cand) if newtgts: candidates.append((cand, newtgts))
def patch_graphs(self): def patch_consts(args): for arg in args: if isinstance(arg, Constant) and arg in self.constants: arg.value = self.constants[arg] def visit(obj): if isinstance(obj, Link): patch_consts(obj.args) if (hasattr(obj, "llexitcase") and Constant(obj.llexitcase) in self.constants): obj.llexitcase = self.constants[Constant(obj.llexitcase)] elif isinstance(obj, Block): for op in obj.operations: patch_consts(op.args) for graph in self.graphs: traverse(visit, graph)
def ordered_blocks(graph): # collect all blocks allblocks = [] def visit(block): if isinstance(block, Block): # first we order by offset in the code string if block.operations: ofs = block.operations[0].offset else: ofs = sys.maxint # then we order by input variable name or value if block.inputargs: txt = str(block.inputargs[0]) else: txt = "dummy" allblocks.append((ofs, txt, block)) traverse(visit, graph) allblocks.sort() #for ofs, txt, block in allblocks: # print ofs, txt, block return [block for ofs, txt, block in allblocks]
def compute_lifetimes(self, graph): """Compute the static data flow of the graph: returns a list of LifeTime instances, each of which corresponds to a set of Variables from the graph. The variables are grouped in the same LifeTime if a value can pass from one to the other by following the links. Each LifeTime also records all places where a Variable in the set is used (read) or build (created). """ lifetimes = UnionFind(LifeTime) def set_creation_point(block, var, *cp): _, _, info = lifetimes.find((block, var)) info.creationpoints[cp] = True def set_use_point(block, var, *up): _, _, info = lifetimes.find((block, var)) info.usepoints[up] = True def union(block1, var1, block2, var2): if isinstance(var1, Variable): lifetimes.union((block1, var1), (block2, var2)) elif isinstance(var1, Constant): set_creation_point(block2, var2, "constant", var1) else: raise TypeError(var1) for var in graph.startblock.inputargs: set_creation_point(graph.startblock, var, "inputargs") set_use_point(graph.returnblock, graph.returnblock.inputargs[0], "return") set_use_point(graph.exceptblock, graph.exceptblock.inputargs[0], "except") set_use_point(graph.exceptblock, graph.exceptblock.inputargs[1], "except") def visit(node): if isinstance(node, Block): for op in node.operations: if op.opname in self.IDENTITY_OPS: # special-case these operations to identify their input # and output variables union(node, op.args[0], node, op.result) continue if op.opname in self.SUBSTRUCT_OPS: if self.visit_substruct_op(node, union, op): continue for i in range(len(op.args)): if isinstance(op.args[i], Variable): set_use_point(node, op.args[i], "op", node, op, i) set_creation_point(node, op.result, "op", node, op) if isinstance(node.exitswitch, Variable): set_use_point(node, node.exitswitch, "exitswitch", node) if isinstance(node, Link): if isinstance(node.last_exception, Variable): set_creation_point(node.prevblock, node.last_exception, "last_exception") if isinstance(node.last_exc_value, Variable): set_creation_point(node.prevblock, node.last_exc_value, "last_exc_value") d = {} for i, arg in enumerate(node.args): union(node.prevblock, arg, node.target, node.target.inputargs[i]) if isinstance(arg, Variable): if arg in d: # same variable present several times in link.args # consider it as a 'use' of the variable, which # will disable malloc optimization (aliasing problems) set_use_point(node.prevblock, arg, "dup", node, i) else: d[arg] = True traverse(visit, graph) return lifetimes.infos()
def sanity_check(t): # look for missing '.concretetype' for graph in t.graphs: checkgraph(graph) traverse(no_missing_concretetype, graph)
def transform_ovfcheck(graph): """The special function calls ovfcheck and ovfcheck_lshift need to be translated into primitive operations. ovfcheck is called directly after an operation that should be turned into an overflow-checked version. It is considered a syntax error if the resulting <op>-ovf is not defined in baseobjspace.py . ovfcheck_lshift is special because there is no preceding operation. Instead, it will be replaced by an OP_LSHIFT_OVF operation. The exception handling of the original operation is completely ignored. Only exception handlers for the ovfcheck function call are taken into account. This gives us the best possible control over situations where we want exact contol over certain operations. Example: try: array1[idx-1] = ovfcheck(array1[idx-1] + array2[idx+1]) except OverflowError: ... assuming two integer arrays, we are only checking the element addition for overflows, but the indexing is not checked. """ # General assumption: # empty blocks have been eliminated. # ovfcheck can appear in the same block with its operation. # this is the case if no exception handling was provided. # Otherwise, we have a block ending in the operation, # followed by a block with a single ovfcheck call. from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift from pypy.objspace.flow.objspace import op_appendices from pypy.objspace.flow.objspace import implicit_exceptions covf = Constant(ovfcheck) covfls = Constant(ovfcheck_lshift) appendix = op_appendices[OverflowError] renaming = {} seen_ovfblocks = {} # get all blocks blocks = {} def visit(block): if isinstance(block, Block): blocks[block] = True traverse(visit, graph) def is_ovfcheck(bl): ops = bl.operations return ops and ops[-1].opname == "simple_call" and ops[-1].args[0] == covf def is_ovfshiftcheck(bl): ops = bl.operations return ops and ops[-1].opname == "simple_call" and ops[-1].args[0] == covfls def is_single(bl): return is_ovfcheck(bl) and len(bl.operations) > 1 def is_paired(bl): if bl.exits: ovfblock = bl.exits[0].target return bl.exits and is_ovfcheck(ovfblock) and len(ovfblock.operations) == 1 def rename(v): return renaming.get(v, v) def remove_last_op(bl): delop = bl.operations.pop() assert delop.opname == "simple_call" assert len(delop.args) == 2 renaming[delop.result] = rename(delop.args[1]) for exit in bl.exits: exit.args = [rename(a) for a in exit.args] def check_syntax(ovfblock, block=None): """check whether ovfblock is reachable more than once or if they cheated about the argument""" if block: link = block.exits[0] for lprev, ltarg in zip(link.args, ovfblock.inputargs): renaming[ltarg] = rename(lprev) arg = ovfblock.operations[0].args[-1] res = block.operations[-1].result opname = block.operations[-1].opname else: arg = ovfblock.operations[-1].args[-1] res = ovfblock.operations[-2].result opname = ovfblock.operations[-2].opname if rename(arg) != rename(res) or ovfblock in seen_ovfblocks: raise SyntaxError("ovfcheck in %s: The checked operation %s" " is misplaced" % (graph.name, opname)) exlis = implicit_exceptions.get("%s_%s" % (opname, appendix), []) if OverflowError not in exlis: raise SyntaxError("ovfcheck in %s: Operation %s has no" " overflow variant" % (graph.name, opname)) blocks_to_join = False for block in blocks: if is_ovfshiftcheck(block): # ovfcheck_lshift: # simply rewrite the operation op = block.operations[-1] op.opname = "lshift" # augmented later op.args = op.args[1:] elif is_single(block): # remove the call to ovfcheck and keep the exceptions check_syntax(block) remove_last_op(block) seen_ovfblocks[block] = True elif is_paired(block): # remove the block's exception links link = block.exits[0] ovfblock = link.target check_syntax(ovfblock, block) block.recloseblock(link) block.exitswitch = None # remove the ovfcheck call from the None target remove_last_op(ovfblock) seen_ovfblocks[ovfblock] = True blocks_to_join = True else: continue op = block.operations[-1] op.opname = "%s_%s" % (op.opname, appendix) if blocks_to_join: join_blocks(graph)
def simplify_exceptions(graph): """The exception handling caused by non-implicit exceptions starts with an exitswitch on Exception, followed by a lengthy chain of is_/issubtype tests. We collapse them all into the block's single list of exits. """ clastexc = c_last_exception renaming = {} def rename(v): return renaming.get(v, v) def visit(block): if not (isinstance(block, Block) and block.exitswitch == clastexc and block.exits[-1].exitcase is Exception): return covered = [link.exitcase for link in block.exits[1:-1]] seen = [] preserve = list(block.exits[:-1]) exc = block.exits[-1] last_exception = exc.last_exception last_exc_value = exc.last_exc_value query = exc.target switches = [] # collect the targets while len(query.exits) == 2: newrenaming = {} for lprev, ltarg in zip(exc.args, query.inputargs): newrenaming[ltarg] = rename(lprev) op = query.operations[0] if not (op.opname in ("is_", "issubtype") and newrenaming.get(op.args[0]) == last_exception): break renaming.update(newrenaming) case = query.operations[0].args[-1].value assert issubclass(case, py.builtin.BaseException) lno, lyes = query.exits assert lno.exitcase == False and lyes.exitcase == True if case not in seen: is_covered = False for cov in covered: if issubclass(case, cov): is_covered = True break if not is_covered: switches.append( (case, lyes) ) seen.append(case) exc = lno query = exc.target if Exception not in seen: switches.append( (Exception, exc) ) # construct the block's new exits exits = [] for case, oldlink in switches: link = oldlink.copy(rename) assert case is not None link.last_exception = last_exception link.last_exc_value = last_exc_value # make the above two variables unique renaming2 = {} def rename2(v): return renaming2.get(v, v) for v in link.getextravars(): renaming2[v] = Variable(v) link = link.copy(rename2) link.exitcase = case link.prevblock = block exits.append(link) block.recloseblock(*(preserve + exits)) traverse(visit, graph)
link.last_exc_value = last_exc_value # make the above two variables unique renaming2 = {} def rename2(v): return renaming2.get(v, v) for v in link.getextravars(): renaming2[v] = Variable(v) link = link.copy(rename2) link.exitcase = case link.prevblock = block exits.append(link) block.recloseblock(*(preserve + exits)) traverse(visit, graph) def transform_xxxitem(graph): # xxx setitem too if simplify_disabled(graph): return for block in graph.iterblocks(): if block.operations and block.exitswitch == c_last_exception: last_op = block.operations[-1] if last_op.opname == 'getitem': postfx = [] for exit in block.exits: if exit.exitcase is IndexError: postfx.append('idx') elif exit.exitcase is KeyError: postfx.append('key')
def transform_ovfcheck(graph): """The special function calls ovfcheck and ovfcheck_lshift need to be translated into primitive operations. ovfcheck is called directly after an operation that should be turned into an overflow-checked version. It is considered a syntax error if the resulting <op>-ovf is not defined in baseobjspace.py . ovfcheck_lshift is special because there is no preceding operation. Instead, it will be replaced by an OP_LSHIFT_OVF operation. The exception handling of the original operation is completely ignored. Only exception handlers for the ovfcheck function call are taken into account. This gives us the best possible control over situations where we want exact contol over certain operations. Example: try: array1[idx-1] = ovfcheck(array1[idx-1] + array2[idx+1]) except OverflowError: ... assuming two integer arrays, we are only checking the element addition for overflows, but the indexing is not checked. """ # General assumption: # empty blocks have been eliminated. # ovfcheck can appear in the same block with its operation. # this is the case if no exception handling was provided. # Otherwise, we have a block ending in the operation, # followed by a block with a single ovfcheck call. if simplify_disabled(graph): return from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift from pypy.objspace.flow.objspace import op_appendices from pypy.objspace.flow.objspace import implicit_exceptions covf = Constant(ovfcheck) covfls = Constant(ovfcheck_lshift) appendix = op_appendices[OverflowError] renaming = {} seen_ovfblocks = {} # get all blocks blocks = {} def visit(block): if isinstance(block, Block): blocks[block] = True traverse(visit, graph) def is_ovfcheck(bl): ops = bl.operations return (ops and ops[-1].opname == "simple_call" and ops[-1].args[0] == covf) def is_ovfshiftcheck(bl): ops = bl.operations return (ops and ops[-1].opname == "simple_call" and ops[-1].args[0] == covfls) def is_single(bl): return is_ovfcheck(bl) and len(bl.operations) > 1 def is_paired(bl): if bl.exits: ovfblock = bl.exits[0].target return (bl.exits and is_ovfcheck(ovfblock) and len(ovfblock.operations) == 1) def rename(v): return renaming.get(v, v) def remove_last_op(bl): delop = bl.operations.pop() assert delop.opname == "simple_call" assert len(delop.args) == 2 renaming[delop.result] = rename(delop.args[1]) for exit in bl.exits: exit.args = [rename(a) for a in exit.args] def check_syntax(ovfblock, block=None): """check whether ovfblock is reachable more than once or if they cheated about the argument""" if block: link = block.exits[0] for lprev, ltarg in zip(link.args, ovfblock.inputargs): renaming[ltarg] = rename(lprev) arg = ovfblock.operations[0].args[-1] res = block.operations[-1].result opname = block.operations[-1].opname else: arg = ovfblock.operations[-1].args[-1] res = ovfblock.operations[-2].result opname = ovfblock.operations[-2].opname if rename(arg) != rename(res) or ovfblock in seen_ovfblocks: raise SyntaxError("ovfcheck in %s: The checked operation %s" " is misplaced" % (graph.name, opname)) exlis = implicit_exceptions.get("%s_%s" % (opname, appendix), []) if OverflowError not in exlis: raise SyntaxError("ovfcheck in %s: Operation %s has no" " overflow variant" % (graph.name, opname)) blocks_to_join = False for block in blocks: if is_ovfshiftcheck(block): # ovfcheck_lshift: # simply rewrite the operation op = block.operations[-1] op.opname = "lshift" # augmented later op.args = op.args[1:] elif is_single(block): # remove the call to ovfcheck and keep the exceptions check_syntax(block) remove_last_op(block) seen_ovfblocks[block] = True elif is_paired(block): # remove the block's exception links link = block.exits[0] ovfblock = link.target check_syntax(ovfblock, block) block.recloseblock(link) block.exitswitch = None # remove the ovfcheck call from the None target remove_last_op(ovfblock) seen_ovfblocks[ovfblock] = True blocks_to_join = True else: continue op = block.operations[-1] op.opname = "%s_%s" % (op.opname, appendix) if blocks_to_join: join_blocks(graph)