Example #1
0
def remove_identical_vars(graph):
    """When the same variable is passed multiple times into the next block,
    pass it only once.  This enables further optimizations by the annotator,
    which otherwise doesn't realize that tests performed on one of the copies
    of the variable also affect the other."""

    # This algorithm is based on DataFlowFamilyBuilder, used as a
    # "phi node remover" (in the SSA sense).  'variable_families' is a
    # UnionFind object that groups variables by families; variables from the
    # same family can be identified, and if two input arguments of a block
    # end up in the same family, then we really remove one of them in favor
    # of the other.
    #
    # The idea is to identify as much variables as possible by trying
    # iteratively two kinds of phi node removal:
    #
    #  * "vertical", by identifying variables from different blocks, when
    #    we see that a value just flows unmodified into the next block without
    #    needing any merge (this is what backendopt.ssa.SSI_to_SSA() would do
    #    as well);
    #
    #  * "horizontal", by identifying two input variables of the same block,
    #    when these two variables' phi nodes have the same argument -- i.e.
    #    when for all possible incoming paths they would get twice the same
    #    value (this is really the purpose of remove_identical_vars()).
    #
    builder = DataFlowFamilyBuilder(graph)
    variable_families = builder.get_variable_families()  # vertical removal
    while True:
        if not builder.merge_identical_phi_nodes():  # horizontal removal
            break
        if not builder.complete():  # vertical removal
            break

    for block, links in mkentrymap(graph).items():
        if block is graph.startblock:
            continue
        renaming = {}
        family2blockvar = {}
        kills = []
        for i, v in enumerate(block.inputargs):
            v1 = variable_families.find_rep(v)
            if v1 in family2blockvar:
                # already seen -- this variable can be shared with the
                # previous one
                renaming[v] = family2blockvar[v1]
                kills.append(i)
            else:
                family2blockvar[v1] = v
        if renaming:
            block.renamevariables(renaming)
            # remove the now-duplicate input variables
            kills.reverse()  # starting from the end
            for i in kills:
                del block.inputargs[i]
                for link in links:
                    del link.args[i]
Example #2
0
def remove_identical_vars(graph):
    """When the same variable is passed multiple times into the next block,
    pass it only once.  This enables further optimizations by the annotator,
    which otherwise doesn't realize that tests performed on one of the copies
    of the variable also affect the other."""

    # This algorithm is based on DataFlowFamilyBuilder, used as a
    # "phi node remover" (in the SSA sense).  'variable_families' is a
    # UnionFind object that groups variables by families; variables from the
    # same family can be identified, and if two input arguments of a block
    # end up in the same family, then we really remove one of them in favor
    # of the other.
    #
    # The idea is to identify as much variables as possible by trying
    # iteratively two kinds of phi node removal:
    #
    #  * "vertical", by identifying variables from different blocks, when
    #    we see that a value just flows unmodified into the next block without
    #    needing any merge (this is what backendopt.ssa.SSI_to_SSA() would do
    #    as well);
    #
    #  * "horizontal", by identifying two input variables of the same block,
    #    when these two variables' phi nodes have the same argument -- i.e.
    #    when for all possible incoming paths they would get twice the same
    #    value (this is really the purpose of remove_identical_vars()).
    #
    builder = DataFlowFamilyBuilder(graph)
    variable_families = builder.get_variable_families()  # vertical removal
    while True:
        if not builder.merge_identical_phi_nodes():    # horizontal removal
            break
        if not builder.complete():                     # vertical removal
            break

    for block, links in mkentrymap(graph).items():
        if block is graph.startblock:
            continue
        renaming = {}
        family2blockvar = {}
        kills = []
        for i, v in enumerate(block.inputargs):
            v1 = variable_families.find_rep(v)
            if v1 in family2blockvar:
                # already seen -- this variable can be shared with the
                # previous one
                renaming[v] = family2blockvar[v1]
                kills.append(i)
            else:
                family2blockvar[v1] = v
        if renaming:
            block.renamevariables(renaming)
            # remove the now-duplicate input variables
            kills.reverse()   # starting from the end
            for i in kills:
                del block.inputargs[i]
                for link in links:
                    del link.args[i]
Example #3
0
    def compute_borrowed_vars(self, graph):
        # the input args are borrowed, and stay borrowed for as long as they
        # are not merged with other values.
        var_families = DataFlowFamilyBuilder(graph).get_variable_families()
        borrowed_reps = {}
        for v in graph.getargs():
            borrowed_reps[var_families.find_rep(v)] = True
        # no support for returning borrowed values so far
        retvar = graph.getreturnvar()

        def is_borrowed(v1):
            return (var_families.find_rep(v1) in borrowed_reps
                    and v1 is not retvar)
        return is_borrowed
Example #4
0
    def compute_borrowed_vars(self, graph):
        # the input args are borrowed, and stay borrowed for as long as they
        # are not merged with other values.
        var_families = DataFlowFamilyBuilder(graph).get_variable_families()
        borrowed_reps = {}
        for v in graph.getargs():
            borrowed_reps[var_families.find_rep(v)] = True
        # no support for returning borrowed values so far
        retvar = graph.getreturnvar()

        def is_borrowed(v1):
            return (var_families.find_rep(v1) in borrowed_reps
                    and v1 is not retvar)
        return is_borrowed
Example #5
0
 def analyze_exceptblock_in_graph(self, graph, block, seen=None):
     if self.ignore_exact_class is not None:
         from rpython.translator.backendopt.ssa import DataFlowFamilyBuilder
         dff = DataFlowFamilyBuilder(graph)
         variable_families = dff.get_variable_families()
         v_exc_instance = variable_families.find_rep(block.inputargs[1])
         for link1 in graph.iterlinks():
             v = link1.last_exc_value
             if v is not None:
                 if variable_families.find_rep(v) is v_exc_instance:
                     # this is a case of re-raise the exception caught;
                     # it doesn't count.  We'll see the place that really
                     # raises the exception in the first place.
                     return False
     return True
Example #6
0
 def analyze_exceptblock_in_graph(self, graph, block, seen=None):
     if self.ignore_exact_class is not None:
         from rpython.translator.backendopt.ssa import DataFlowFamilyBuilder
         dff = DataFlowFamilyBuilder(graph)
         variable_families = dff.get_variable_families()
         v_exc_instance = variable_families.find_rep(block.inputargs[1])
         for link1 in graph.iterlinks():
             v = link1.last_exc_value
             if v is not None:
                 if variable_families.find_rep(v) is v_exc_instance:
                     # this is a case of re-raise the exception caught;
                     # it doesn't count.  We'll see the place that really
                     # raises the exception in the first place.
                     return False
     return True
Example #7
0
def detect_list_comprehension(graph):
    """Look for the pattern:            Replace it with marker operations:

                                         v0 = newlist()
        v2 = newlist()                   v1 = hint(v0, iterable, {'maxlength'})
        loop start                       loop start
        ...                              ...
        exactly one append per loop      v1.append(..)
        and nothing else done with v2
        ...                              ...
        loop end                         v2 = hint(v1, {'fence'})
    """
    # NB. this assumes RPythonicity: we can only iterate over something
    # that has a len(), and this len() cannot change as long as we are
    # using the iterator.
    builder = DataFlowFamilyBuilder(graph)
    variable_families = builder.get_variable_families()
    c_append = Constant('append')
    newlist_v = {}
    iter_v = {}
    append_v = []
    loopnextblocks = []

    # collect relevant operations based on the family of their result
    for block in graph.iterblocks():
        if (len(block.operations) == 1 and block.operations[0].opname == 'next'
                and block.canraise and len(block.exits) >= 2):
            cases = [link.exitcase for link in block.exits]
            if None in cases and StopIteration in cases:
                # it's a straightforward loop start block
                loopnextblocks.append((block, block.operations[0].args[0]))
                continue
        for op in block.operations:
            if op.opname == 'newlist' and not op.args:
                vlist = variable_families.find_rep(op.result)
                newlist_v[vlist] = block
            if op.opname == 'iter':
                viter = variable_families.find_rep(op.result)
                iter_v[viter] = block
    loops = []
    for block, viter in loopnextblocks:
        viterfamily = variable_families.find_rep(viter)
        if viterfamily in iter_v:
            # we have a next(viter) operation where viter comes from a
            # single known iter() operation.  Check that the iter()
            # operation is in the block just before.
            iterblock = iter_v[viterfamily]
            if (len(iterblock.exits) == 1 and iterblock.exitswitch is None
                    and iterblock.exits[0].target is block):
                # yes - simple case.
                loops.append((block, iterblock, viterfamily))
    if not newlist_v or not loops:
        return

    # XXX works with Python >= 2.4 only: find calls to append encoded as
    # getattr/simple_call pairs, as produced by the LIST_APPEND bytecode.
    for block in graph.iterblocks():
        for i in range(len(block.operations) - 1):
            op = block.operations[i]
            if op.opname == 'getattr' and op.args[1] == c_append:
                vlist = variable_families.find_rep(op.args[0])
                if vlist in newlist_v:
                    for j in range(i + 1, len(block.operations)):
                        op2 = block.operations[j]
                        if (op2.opname == 'simple_call' and len(op2.args) == 2
                                and op2.args[0] is op.result):
                            append_v.append((op.args[0], op.result, block))
                            break
    if not append_v:
        return
    detector = ListComprehensionDetector(graph, loops, newlist_v,
                                         variable_families)
    graphmutated = False
    for location in append_v:
        if graphmutated:
            # new variables introduced, must restart the whole process
            return detect_list_comprehension(graph)
        try:
            detector.run(*location)
        except DetectorFailed:
            pass
        else:
            graphmutated = True
Example #8
0
def find_inner_loops(graph, check_exitswitch_type=None):
    """Enumerate what look like the innermost loops of the graph.
    Returns a list of non-overlapping Loop() instances.
    """
    # Heuristic (thanks Stakkars for the idea):
    # to find the "best" cycle to pick,
    #
    #   * look for variables that don't change over the whole cycle
    #
    #   * cycles with _more_ of them are _inside_ cycles with less of them,
    #     because any variable that doesn't change over an outer loop will
    #     not change over an inner loop either, and on the other hand the
    #     outer loop is likely to use and modify variables that remain
    #     constant over the inner loop.
    #
    #   * if the numbers are the same, fall back to a more arbitrary
    #     measure: loops involving less blocks may be more important
    #     to optimize
    #
    #   * in a cycle, which block is the head of the loop?  Somewhat
    #     arbitrarily we pick the first Bool-switching block that has
    #     two exits.  The "first" means the one closest to the
    #     startblock of the graph.
    #
    startdistance = {}     # {block: distance-from-startblock}
    pending = [graph.startblock]
    edge_list = []
    dist = 0
    while pending:
        newblocks = []
        for block in pending:
            if block not in startdistance:
                startdistance[block] = dist
                for link in block.exits:
                    newblocks.append(link.target)
                    edge = graphlib.Edge(block, link.target)
                    edge.link = link
                    edge_list.append(edge)
        dist += 1
        pending = newblocks

    vertices = startdistance
    edges = graphlib.make_edge_dict(edge_list)
    cycles = graphlib.all_cycles(graph.startblock, vertices, edges)
    loops = []
    variable_families = None

    for cycle in cycles:
        # find the headblock
        candidates = []
        for i in range(len(cycle)):
            block = cycle[i].source
            v = block.exitswitch
            if isinstance(v, Variable) and len(block.exits) == 2:
                if getattr(v, 'concretetype', None) is check_exitswitch_type:
                    dist = startdistance[block]
                    candidates.append((dist, i))
        if not candidates:
            continue
        _, i = min(candidates)
        links = [edge.link for edge in cycle[i:] + cycle[:i]]
        loop = Loop(cycle[i].source, links)

        # count the variables that remain constant across the cycle,
        # detected as having its SSA family present across all blocks.
        if variable_families is None:
            dffb = DataFlowFamilyBuilder(graph)
            variable_families = dffb.get_variable_families()

        num_loop_constants = 0
        for v in loop.headblock.inputargs:
            v = variable_families.find_rep(v)
            for link in loop.links:
                block1 = link.target
                for v1 in block1.inputargs:
                    v1 = variable_families.find_rep(v1)
                    if v1 is v:
                        break    # ok, found in this block
                else:
                    break   # not found in this block, fail
            else:
                # found in all blocks, this variable is a loop constant
                num_loop_constants += 1

        # smaller keys are "better"
        key = (-num_loop_constants,   # maximize num_loop_constants
               len(cycle))            # minimize len(cycle)
        loops.append((key, loop))

    loops.sort()

    # returns 'loops' without overlapping blocks
    result = []
    blocks_seen = {}
    for key, loop in loops:
        for link in loop.links:
            if link.target in blocks_seen:
                break     # overlapping
        else:
            # non-overlapping
            result.append(loop)
            for link in loop.links:
                blocks_seen[link.target] = True
    return result
Example #9
0
def detect_list_comprehension(graph):
    """Look for the pattern:            Replace it with marker operations:

                                         v0 = newlist()
        v2 = newlist()                   v1 = hint(v0, iterable, {'maxlength'})
        loop start                       loop start
        ...                              ...
        exactly one append per loop      v1.append(..)
        and nothing else done with v2
        ...                              ...
        loop end                         v2 = hint(v1, {'fence'})
    """
    # NB. this assumes RPythonicity: we can only iterate over something
    # that has a len(), and this len() cannot change as long as we are
    # using the iterator.
    builder = DataFlowFamilyBuilder(graph)
    variable_families = builder.get_variable_families()
    c_append = Constant('append')
    newlist_v = {}
    iter_v = {}
    append_v = []
    loopnextblocks = []

    # collect relevant operations based on the family of their result
    for block in graph.iterblocks():
        if (len(block.operations) == 1 and
            block.operations[0].opname == 'next' and
            block.exitswitch == c_last_exception and
            len(block.exits) >= 2):
            cases = [link.exitcase for link in block.exits]
            if None in cases and StopIteration in cases:
                # it's a straightforward loop start block
                loopnextblocks.append((block, block.operations[0].args[0]))
                continue
        for op in block.operations:
            if op.opname == 'newlist' and not op.args:
                vlist = variable_families.find_rep(op.result)
                newlist_v[vlist] = block
            if op.opname == 'iter':
                viter = variable_families.find_rep(op.result)
                iter_v[viter] = block
    loops = []
    for block, viter in loopnextblocks:
        viterfamily = variable_families.find_rep(viter)
        if viterfamily in iter_v:
            # we have a next(viter) operation where viter comes from a
            # single known iter() operation.  Check that the iter()
            # operation is in the block just before.
            iterblock = iter_v[viterfamily]
            if (len(iterblock.exits) == 1 and iterblock.exitswitch is None
                and iterblock.exits[0].target is block):
                # yes - simple case.
                loops.append((block, iterblock, viterfamily))
    if not newlist_v or not loops:
        return

    # XXX works with Python >= 2.4 only: find calls to append encoded as
    # getattr/simple_call pairs, as produced by the LIST_APPEND bytecode.
    for block in graph.iterblocks():
        for i in range(len(block.operations)-1):
            op = block.operations[i]
            if op.opname == 'getattr' and op.args[1] == c_append:
                vlist = variable_families.find_rep(op.args[0])
                if vlist in newlist_v:
                    for j in range(i + 1, len(block.operations)):
                        op2 = block.operations[j]
                        if (op2.opname == 'simple_call' and len(op2.args) == 2
                            and op2.args[0] is op.result):
                            append_v.append((op.args[0], op.result, block))
                            break
    if not append_v:
        return
    detector = ListComprehensionDetector(graph, loops, newlist_v,
                                         variable_families)
    graphmutated = False
    for location in append_v:
        if graphmutated:
            # new variables introduced, must restart the whole process
            return detect_list_comprehension(graph)
        try:
            detector.run(*location)
        except DetectorFailed:
            pass
        else:
            graphmutated = True
Example #10
0
def find_inner_loops(graph, check_exitswitch_type=None):
    """Enumerate what look like the innermost loops of the graph.
    Returns a list of non-overlapping Loop() instances.
    """
    # Heuristic (thanks Stakkars for the idea):
    # to find the "best" cycle to pick,
    #
    #   * look for variables that don't change over the whole cycle
    #
    #   * cycles with _more_ of them are _inside_ cycles with less of them,
    #     because any variable that doesn't change over an outer loop will
    #     not change over an inner loop either, and on the other hand the
    #     outer loop is likely to use and modify variables that remain
    #     constant over the inner loop.
    #
    #   * if the numbers are the same, fall back to a more arbitrary
    #     measure: loops involving less blocks may be more important
    #     to optimize
    #
    #   * in a cycle, which block is the head of the loop?  Somewhat
    #     arbitrarily we pick the first Bool-switching block that has
    #     two exits.  The "first" means the one closest to the
    #     startblock of the graph.
    #
    startdistance = {}  # {block: distance-from-startblock}
    pending = [graph.startblock]
    edge_list = []
    dist = 0
    while pending:
        newblocks = []
        for block in pending:
            if block not in startdistance:
                startdistance[block] = dist
                for link in block.exits:
                    newblocks.append(link.target)
                    edge = graphlib.Edge(block, link.target)
                    edge.link = link
                    edge_list.append(edge)
        dist += 1
        pending = newblocks

    vertices = startdistance
    edges = graphlib.make_edge_dict(edge_list)
    cycles = graphlib.all_cycles(graph.startblock, vertices, edges)
    loops = []
    variable_families = None

    for cycle in cycles:
        # find the headblock
        candidates = []
        for i in range(len(cycle)):
            block = cycle[i].source
            v = block.exitswitch
            if isinstance(v, Variable) and len(block.exits) == 2:
                if getattr(v, 'concretetype', None) is check_exitswitch_type:
                    dist = startdistance[block]
                    candidates.append((dist, i))
        if not candidates:
            continue
        _, i = min(candidates)
        links = [edge.link for edge in cycle[i:] + cycle[:i]]
        loop = Loop(cycle[i].source, links)

        # count the variables that remain constant across the cycle,
        # detected as having its SSA family present across all blocks.
        if variable_families is None:
            dffb = DataFlowFamilyBuilder(graph)
            variable_families = dffb.get_variable_families()

        num_loop_constants = 0
        for v in loop.headblock.inputargs:
            v = variable_families.find_rep(v)
            for link in loop.links:
                block1 = link.target
                for v1 in block1.inputargs:
                    v1 = variable_families.find_rep(v1)
                    if v1 is v:
                        break  # ok, found in this block
                else:
                    break  # not found in this block, fail
            else:
                # found in all blocks, this variable is a loop constant
                num_loop_constants += 1

        # smaller keys are "better"
        key = (
            -num_loop_constants,  # maximize num_loop_constants
            len(cycle))  # minimize len(cycle)
        loops.append((key, loop))

    loops.sort()

    # returns 'loops' without overlapping blocks
    result = []
    blocks_seen = {}
    for key, loop in loops:
        for link in loop.links:
            if link.target in blocks_seen:
                break  # overlapping
        else:
            # non-overlapping
            result.append(loop)
            for link in loop.links:
                blocks_seen[link.target] = True
    return result