Ejemplo n.º 1
0
    def do(self, *args, **kwargs):
        logger.info("Computation graph statistics:")
        cost_cg = ComputationGraph(self.main_loop.algorithm.cost)
        updates_cg = ComputationGraph([
            u[1] for u in self.main_loop.algorithm.updates
            if isinstance(u[1], theano.Variable)
        ])
        cost_nodes = io_toposort(cost_cg.inputs, cost_cg.outputs)
        updates_nodes = io_toposort(updates_cg.inputs, updates_cg.outputs)

        cost_scan_nodes = [
            node for node in cost_nodes if isinstance(node.op, Scan)
        ]
        updates_scan_nodes = [
            node for node in updates_nodes if isinstance(node.op, Scan)
        ]
        final_scan_nodes = [
            node for node in
            self.main_loop.algorithm._function.maker.fgraph.apply_nodes
            if isinstance(node.op, Scan)
        ]

        logger.info("SCAN NODES IN THE COST GRAPH:")
        for n in cost_scan_nodes:
            logger.info(n.op.name)
        logger.info("SCAN NODES IN THE UPDATES GRAPH:")
        for n in updates_scan_nodes:
            logger.info(n.op.name)
        logger.info("SCAN NODES IN THE FINAL GRAPH:")
        for n in final_scan_nodes:
            logger.info(n.op.name)
Ejemplo n.º 2
0
    def test_2(self):
        """Test a graph where the inputs have owners"""
        r1, r2, r5 = MyVariable(1), MyVariable(2), MyVariable(5)
        o = MyOp.make_node(r1, r1)
        r2b = o.outputs[0]
        o2 = MyOp.make_node(r2b, r2b)
        all = io_toposort([r2b], o2.outputs)
        assert all == [o2]

        o2 = MyOp.make_node(r2b, r5)
        all = io_toposort([r2b], o2.outputs)
        assert all == [o2]
Ejemplo n.º 3
0
    def test_2(self):
        """Test a graph where the inputs have owners"""
        r1, r5 = MyVariable(1), MyVariable(5)
        o = MyOp.make_node(r1, r1)
        r2b = o.outputs[0]
        o2 = MyOp.make_node(r2b, r2b)
        all = io_toposort([r2b], o2.outputs)
        assert all == [o2]

        o2 = MyOp.make_node(r2b, r5)
        all = io_toposort([r2b], o2.outputs)
        assert all == [o2]
Ejemplo n.º 4
0
 def test_4(self):
     """Test inputs and outputs mixed together in a chain graph"""
     r1, r2 = MyVariable(1), MyVariable(2)
     o0 = MyOp.make_node(r1, r2)
     o1 = MyOp.make_node(o0.outputs[0], r1)
     all = io_toposort([r1, o0.outputs[0]], [o0.outputs[0], o1.outputs[0]])
     assert all == [o1]
Ejemplo n.º 5
0
    def __import__(self, node, check = True):
        # We import the nodes in topological order. We only are interested
        # in new nodes, so we use all variables we know of as if they were the input set.
        # (the functions in the graph module only use the input set to
        # know where to stop going down)
        new_nodes = graph.io_toposort(self.variables, node.outputs)

        if check:
            for node in new_nodes:
                if hasattr(node, 'env') and node.env is not self:
                    raise Exception("%s is already owned by another env" % node)
                for r in node.inputs:
                    if hasattr(r, 'env') and r.env is not self:
                        raise Exception("%s is already owned by another env" % r)

        for node in new_nodes:
            assert node not in self.nodes
            self.__setup_node__(node)
            for output in node.outputs:
                self.__setup_r__(output)
            for i, input in enumerate(node.inputs):
                if input not in self.variables:
                    self.__setup_r__(input)
                self.__add_clients__(input, [(node, i)])
            assert node.env is self
            self.execute_callbacks('on_import', node)
Ejemplo n.º 6
0
 def test_5(self):
     """Test when outputs have clients"""
     r1, r2, r4 = MyVariable(1), MyVariable(2), MyVariable(4)
     o0 = MyOp.make_node(r1, r2)
     MyOp.make_node(o0.outputs[0], r4)
     all = io_toposort([], o0.outputs)
     assert all == [o0]
Ejemplo n.º 7
0
    def toposort(self):
        """WRITEME
        Returns an ordering of the graph's Apply nodes such that:
          - All the nodes of the inputs of a node are before that node.
          - Satisfies the orderings provided by each feature that has
            an 'orderings' method.

        If a feature has an 'orderings' method, it will be called with
        this FunctionGraph as sole argument. It should return a dictionary of
        {node: predecessors} where predecessors is a list of nodes
        that should be computed before the key node.
        """
        if len(self.apply_nodes) < 2:
            # optimization
            # when there are 0 or 1 nodes, no sorting is necessary
            # This special case happens a lot because the OpWiseCLinker
            # produces 1-element graphs.
            return list(self.apply_nodes)
        fg = self

        ords = self.orderings()

        order = graph.io_toposort(fg.inputs, fg.outputs, ords)

        return order
Ejemplo n.º 8
0
 def test_3(self):
     """Test a graph which is not connected"""
     r1, r2, r3, r4 = MyVariable(1), MyVariable(2), MyVariable(3), MyVariable(4)
     o0 = MyOp.make_node(r1, r2)
     o1 = MyOp.make_node(r3, r4)
     all = io_toposort([r1, r2, r3, r4], o0.outputs + o1.outputs)
     assert all == [o1, o0]
Ejemplo n.º 9
0
 def on_detach(self, fgraph):
     """
     Should remove any dynamically added functionality
     that it installed into the function_graph
     """
     for node in graph.io_toposort(fgraph.inputs, fgraph.outputs):
         self.on_prune(fgraph, node, 'Bookkeeper.detach')
Ejemplo n.º 10
0
    def _get_variables(self):
        """Collect variables, updates and auxiliary variables.

        In addition collects all :class:`.Scan` ops and recurses in the
        respective inner Theano graphs.

        """
        updates = OrderedDict()

        shared_outputs = [o for o in self.outputs if is_shared_variable(o)]
        usual_outputs = [o for o in self.outputs if not is_shared_variable(o)]
        variables = shared_outputs

        if usual_outputs:
            # Sort apply nodes topologically, get variables and remove
            # duplicates
            inputs = graph.inputs(self.outputs)
            sorted_apply_nodes = graph.io_toposort(inputs, usual_outputs)
            self.scans = list(unique([node.op for node in sorted_apply_nodes
                                     if isinstance(node.op, Scan)],
                                     key=lambda op: id(op)))
            self._scan_graphs = [ComputationGraph(scan.outputs)
                                 for scan in self.scans]

            seen = set()
            main_vars = (
                [var for var in list(chain(
                    *[apply_node.inputs for apply_node in sorted_apply_nodes]))
                 if not (var in seen or seen.add(var))] +
                [var for var in self.outputs if var not in seen])

            # While preserving order add auxiliary variables, and collect
            # updates
            seen = set()
            # Intermediate variables could be auxiliary
            seen_avs = set(main_vars)
            variables = []
            for var in main_vars:
                variables.append(var)
                for annotation in getattr(var.tag, 'annotations', []):
                    if annotation not in seen:
                        seen.add(annotation)
                        new_avs = [
                            av for av in annotation.auxiliary_variables
                            if not (av in seen_avs or seen_avs.add(av))]
                        variables.extend(new_avs)
                        updates = dict_union(updates, annotation.updates)

        # If shared_variables is assigned default_update (cloned), we cannot eval()
        # it to get the real numpy array value, hence, try to trace back
        # original shared variable
        def shared_variable_filter(var):
            if is_shared_variable(var) and hasattr(var, 'default_update'):
                for annotation in var.tag.annotations:
                    if hasattr(annotation, var.name) and \
                       is_shared_variable(getattr(annotation, var.name)):
                        return getattr(annotation, var.name)
            return var
        self.variables = map(shared_variable_filter, variables)
        self.updates = updates
Ejemplo n.º 11
0
 def test_4(self):
     """Test inputs and outputs mixed together in a chain graph"""
     r1, r2 = MyVariable(1), MyVariable(2)
     o0 = MyOp.make_node(r1, r2)
     o1 = MyOp.make_node(o0.outputs[0], r1)
     all = io_toposort([r1, o0.outputs[0]], [o0.outputs[0], o1.outputs[0]])
     assert all == [o1]
Ejemplo n.º 12
0
    def _get_variables(self):
        """Collect variables, updates and auxiliary variables.

        In addition collects all :class:`.Scan` ops and recurses in the
        respective inner Theano graphs.

        """
        updates = OrderedDict()

        shared_outputs = [o for o in self.outputs if is_shared_variable(o)]
        usual_outputs = [o for o in self.outputs if not is_shared_variable(o)]
        variables = shared_outputs

        if usual_outputs:
            # Sort apply nodes topologically, get variables and remove
            # duplicates
            inputs = graph.inputs(self.outputs)
            self.sorted_apply_nodes = graph.io_toposort(inputs, usual_outputs)
            self.scans = list(
                unique([
                    node.op for node in self.sorted_apply_nodes
                    if isinstance(node.op, Scan)
                ]))
            self.sorted_scan_nodes = [
                node for node in self.sorted_apply_nodes
                if isinstance(node.op, Scan)
            ]
            self._scan_graphs = [
                ComputationGraph(scan.outputs) for scan in self.scans
            ]

            seen = set()
            main_vars = ([
                var for var in list(
                    chain(*[
                        apply_node.inputs
                        for apply_node in self.sorted_apply_nodes
                    ])) if not (var in seen or seen.add(var))
            ] + [var for var in self.outputs if var not in seen])

            # While preserving order add auxiliary variables, and collect
            # updates
            seen = set()
            # Intermediate variables could be auxiliary
            seen_avs = set(main_vars)
            variables = []
            for var in main_vars:
                variables.append(var)
                for annotation in getattr(var.tag, 'annotations', []):
                    if annotation not in seen:
                        seen.add(annotation)
                        new_avs = [
                            av for av in annotation.auxiliary_variables
                            if not (av in seen_avs or seen_avs.add(av))
                        ]
                        variables.extend(new_avs)
                        updates = dict_union(updates, annotation.updates)

        self.variables = variables
        self.updates = updates
Ejemplo n.º 13
0
 def test_3(self):
     """Test a graph which is not connected"""
     r1, r2, r3, r4 = MyVariable(1), MyVariable(2), MyVariable(3), MyVariable(4)
     o0 = MyOp.make_node(r1, r2)
     o1 = MyOp.make_node(r3, r4)
     all = io_toposort([r1, r2, r3, r4], o0.outputs + o1.outputs)
     assert all == [o1,o0]
Ejemplo n.º 14
0
 def test_5(self):
     """Test when outputs have clients"""
     r1, r2, r4 = MyVariable(1), MyVariable(2), MyVariable(4)
     o0 = MyOp.make_node(r1, r2)
     MyOp.make_node(o0.outputs[0], r4)
     all = io_toposort([], o0.outputs)
     assert all == [o0]
Ejemplo n.º 15
0
    def test_simple(self):
        # Test a simple graph
        r1, r2, r5 = MyVariable(1), MyVariable(2), MyVariable(5)
        o = MyOp(r1, r2)
        o.name = "o1"
        o2 = MyOp(o, r5)
        o2.name = "o2"

        clients = {}
        res = general_toposort([o2], prenode, clients=clients)

        assert clients == {
            o2.owner: [o2],
            o: [o2.owner],
            r5: [o2.owner],
            o.owner: [o],
            r1: [o.owner],
            r2: [o.owner],
        }
        assert res == [r5, r2, r1, o.owner, o, o2.owner, o2]

        with pytest.raises(ValueError):
            general_toposort([o2],
                             prenode,
                             compute_deps_cache=lambda x: None,
                             deps_cache=None)

        res = io_toposort([r5], [o2])
        assert res == [o.owner, o2.owner]
Ejemplo n.º 16
0
 def on_detach(self, fgraph):
     """
     Should remove any dynamically added functionality
     that it installed into the function_graph
     """
     for node in graph.io_toposort(fgraph.inputs, fgraph.outputs):
         self.on_prune(fgraph, node, 'Bookkeeper.detach')
Ejemplo n.º 17
0
    def import_node(self, apply_node, check=True, reason=None):
        """Recursively import everything between an `Apply` node and the `FunctionGraph`'s outputs.

        Parameters:
        ----------
        apply_node : theano.gof.graph.Apply
            The node to be imported.
        check : bool
            Check that the inputs for the imported nodes are also present in
            the `FunctionGraph`.
        reason : str
            The name of the optimization or operation in progress.
        """
        node = apply_node

        # We import the nodes in topological order. We only are interested
        # in new nodes, so we use all variables we know of as if they were the input set.
        # (the functions in the graph module only use the input set to
        # know where to stop going down)
        new_nodes = graph.io_toposort(self.variables, apply_node.outputs)

        if check:
            for node in new_nodes:
                if hasattr(node, "fgraph") and node.fgraph is not self:
                    raise Exception(
                        f"{node} is already owned by another fgraph")
                for var in node.inputs:
                    if hasattr(var, "fgraph") and var.fgraph is not self:
                        raise Exception(
                            f"{var} is already owned by another fgraph")
                    if (var.owner is None
                            and not isinstance(var, graph.Constant)
                            and var not in self.inputs):
                        # Standard error message
                        error_msg = (
                            f"Input {int(node.inputs.index(var))} of the graph (indices start "
                            f"from 0), used to compute {node}, was not "
                            "provided and not given a value. Use the "
                            "Theano flag exception_verbosity='high', "
                            "for more information on this error.")
                        raise MissingInputError(error_msg, variable=var)

        for node in new_nodes:
            assert node not in self.apply_nodes
            self.setup_node(node)
            self.apply_nodes.add(node)
            if not hasattr(node.tag, "imported_by"):
                node.tag.imported_by = []
            node.tag.imported_by.append(str(reason))
            for output in node.outputs:
                self.setup_var(output)
                self.variables.add(output)
            for i, input in enumerate(node.inputs):
                if input not in self.variables:
                    self.setup_var(input)
                    self.variables.add(input)
                self.add_client(input, (node, i))
            assert node.fgraph is self
            self.execute_callbacks("on_import", node, reason)
Ejemplo n.º 18
0
 def on_attach(self, fgraph):
     """
     Called by FunctionGraph.attach_feature, the method that attaches
     the feature to the FunctionGraph. Since this is called after the
     FunctionGraph is initially populated, this is where you should
     run checks on the initial contents of the FunctionGraph.
     """
     for node in graph.io_toposort(fgraph.inputs, fgraph.outputs):
         self.on_import(fgraph, node, "on_attach")
Ejemplo n.º 19
0
def test_dependence():
    dependence = make_dependence_cmp()

    x = tensor.matrix("x")
    y = tensor.dot(x * 2, x + 1)
    nodes = io_toposort([x], [y])

    for a, b in zip(nodes[:-1], nodes[1:]):
        assert dependence(a, b) <= 0
Ejemplo n.º 20
0
def test_dependence():
    dependence = make_dependence_cmp()

    x = tensor.matrix('x')
    y = tensor.dot(x * 2, x + 1)
    nodes = io_toposort([x], [y])

    for a, b in zip(nodes[:-1], nodes[1:]):
        assert dependence(a, b) <= 0
Ejemplo n.º 21
0
 def on_attach(self, fgraph):
     """
     Called by FunctionGraph.attach_feature, the method that attaches
     the feature to the FunctionGraph. Since this is called after the
     FunctionGraph is initially populated, this is where you should
     run checks on the initial contents of the FunctionGraph.
     """
     for node in graph.io_toposort(fgraph.inputs, fgraph.outputs):
         self.on_import(fgraph, node, "on_attach")
Ejemplo n.º 22
0
    def test_0(self):
        """Test a simple graph"""
        r1, r2, r5 = MyVariable(1), MyVariable(2), MyVariable(5)
        o = MyOp.make_node(r1, r2)
        o2 = MyOp.make_node(o.outputs[0], r5)

        all = general_toposort(o2.outputs, prenode)
        assert all == [r5, r2, r1, o, o.outputs[0], o2, o2.outputs[0]]

        all = io_toposort([r5], o2.outputs)
        assert all == [o, o2]
Ejemplo n.º 23
0
    def test_0(self):
        """Test a simple graph"""
        r1, r2, r5 = MyVariable(1), MyVariable(2), MyVariable(5)
        o = MyOp.make_node(r1, r2)
        o2 = MyOp.make_node(o.outputs[0], r5)

        all = general_toposort(o2.outputs, prenode)
        assert all == [r5, r2, r1, o, o.outputs[0], o2, o2.outputs[0]]

        all = io_toposort([r5], o2.outputs)
        assert all == [o, o2]
Ejemplo n.º 24
0
    def __import__(self, apply_node, check=True, reason=None):
        """
        Given an apply_node, recursively search from this node to know graph,
        and then add all unknown variables and apply_nodes to this graph.
        """
        node = apply_node

        # We import the nodes in topological order. We only are interested
        # in new nodes, so we use all variables we know of as if they were the input set.
        # (the functions in the graph module only use the input set to
        # know where to stop going down)
        new_nodes = graph.io_toposort(self.variables, apply_node.outputs)

        if check:
            for node in new_nodes:
                if hasattr(node, 'fgraph') and node.fgraph is not self:
                    raise Exception("%s is already owned by another fgraph" %
                                    node)
                for r in node.inputs:
                    if hasattr(r, 'fgraph') and r.fgraph is not self:
                        raise Exception(
                            "%s is already owned by another fgraph" % r)
                    if (r.owner is None and not isinstance(r, graph.Constant)
                            and r not in self.inputs):
                        # Standard error message
                        error_msg = ("Input %d of the graph (indices start "
                                     "from 0), used to compute %s, was not "
                                     "provided and not given a value. Use the "
                                     "Theano flag exception_verbosity='high', "
                                     "for more information on this error." %
                                     (node.inputs.index(r), str(node)))
                        error_msg += get_variable_trace_string(r)
                        raise MissingInputError(error_msg, variable=r)

        for node in new_nodes:
            assert node not in self.apply_nodes
            self.__setup_node__(node)
            self.apply_nodes.add(node)
            if not hasattr(node.tag, 'imported_by'):
                node.tag.imported_by = []
            node.tag.imported_by.append(str(reason))
            for output in node.outputs:
                self.__setup_r__(output)
                self.variables.add(output)
            for i, input in enumerate(node.inputs):
                if input not in self.variables:
                    self.__setup_r__(input)
                    self.variables.add(input)
                self.__add_client__(input, (node, i))
            assert node.fgraph is self
            self.execute_callbacks('on_import', node, reason)
Ejemplo n.º 25
0
    def _get_variables(self):
        """Collect variables, updates and auxiliary variables.

        In addition collects all :class:`.Scan` ops and recurses in the
        respective inner Theano graphs.

        """
        updates = OrderedDict()

        shared_outputs = [o for o in self.outputs if is_shared_variable(o)]
        usual_outputs = [o for o in self.outputs if not is_shared_variable(o)]
        variables = shared_outputs

        if usual_outputs:
            # Sort apply nodes topologically, get variables and remove
            # duplicates
            inputs = graph.inputs(self.outputs)
            self.sorted_apply_nodes = graph.io_toposort(inputs, usual_outputs)
            self.scans = list(unique([node.op for node in self.sorted_apply_nodes
                                     if isinstance(node.op, Scan)]))
            self.sorted_scan_nodes = [node for node in self.sorted_apply_nodes
                                      if isinstance(node.op, Scan)]
            self._scan_graphs = [ComputationGraph(scan.outputs)
                                 for scan in self.scans]

            seen = set()
            main_vars = (
                [var for var in list(chain(
                    *[apply_node.inputs for apply_node in self.sorted_apply_nodes]))
                 if not (var in seen or seen.add(var))] +
                [var for var in self.outputs if var not in seen])

            # While preserving order add auxiliary variables, and collect
            # updates
            seen = set()
            # Intermediate variables could be auxiliary
            seen_avs = set(main_vars)
            variables = []
            for var in main_vars:
                variables.append(var)
                for annotation in getattr(var.tag, 'annotations', []):
                    if annotation not in seen:
                        seen.add(annotation)
                        new_avs = [
                            av for av in annotation.auxiliary_variables
                            if not (av in seen_avs or seen_avs.add(av))]
                        variables.extend(new_avs)
                        updates = dict_union(updates, annotation.updates)

        self.variables = variables
        self.updates = updates
Ejemplo n.º 26
0
    def __import__(self, apply_node, check=True, reason=None):
        """
        Given an apply_node, recursively search from this node to know graph,
        and then add all unknown variables and apply_nodes to this graph.
        """
        node = apply_node

        # We import the nodes in topological order. We only are interested
        # in new nodes, so we use all variables we know of as if they were the input set.
        # (the functions in the graph module only use the input set to
        # know where to stop going down)
        new_nodes = graph.io_toposort(self.variables, apply_node.outputs)

        if check:
            for node in new_nodes:
                if hasattr(node, 'fgraph') and node.fgraph is not self:
                    raise Exception("%s is already owned by another fgraph" % node)
                for r in node.inputs:
                    if hasattr(r, 'fgraph') and r.fgraph is not self:
                        raise Exception("%s is already owned by another fgraph" % r)
                    if (r.owner is None and
                            not isinstance(r, graph.Constant) and
                            r not in self.inputs):
                        # Standard error message
                        error_msg = ("Input %d of the graph (indices start "
                                     "from 0), used to compute %s, was not "
                                     "provided and not given a value. Use the "
                                     "Theano flag exception_verbosity='high', "
                                     "for more information on this error."
                                     % (node.inputs.index(r), str(node)))
                        error_msg += get_variable_trace_string(r)
                        raise MissingInputError(error_msg, variable=r)

        for node in new_nodes:
            assert node not in self.apply_nodes
            self.__setup_node__(node)
            self.apply_nodes.add(node)
            if not hasattr(node.tag, 'imported_by'):
                node.tag.imported_by = []
            node.tag.imported_by.append(str(reason))
            for output in node.outputs:
                self.__setup_r__(output)
                self.variables.add(output)
            for i, input in enumerate(node.inputs):
                if input not in self.variables:
                    self.__setup_r__(input)
                    self.variables.add(input)
                self.__add_client__(input, (node, i))
            assert node.fgraph is self
            self.execute_callbacks('on_import', node, reason)
Ejemplo n.º 27
0
def clone_get_equiv(i, o, replacements=None):
    """Duplicate nodes from `i` to `o` inclusive.

    Returns replacements dictionary, mapping each old node to its new one.

    i - sequence of variables
    o - sequence of variables
    replacements - initial value for return value, modified in place.

    """
    if replacements is None:
        d = {}
    else:
        d = replacements

    for old, new in replacements.items():
        if new in replacements:
            # I think we want to do something recursive here, but
            # it feels like it might get tricky? This reminds me of the
            # 'sorted_givens' branch on github/jaberg/Theano
            raise NotImplementedError('think before implementing')
        replacements[new] = new

    for input in i:
        if input not in d:
            d[input] = input

    for apply in graph.io_toposort(i, o):
        for input in apply.inputs:
            if input not in d:
                d[input] = input

        new_apply = apply.clone_with_new_inputs([d[i] for i in apply.inputs])
        if apply not in d:
            d[apply] = new_apply

        for output, new_output in zip(apply.outputs, new_apply.outputs):
            if output not in d:
                d[output] = new_output

    for output in o:
        if output not in d:
            d[output] = output.clone()

    return d
Ejemplo n.º 28
0
def clone_get_equiv(i, o, replacements=None):
    """Duplicate nodes from `i` to `o` inclusive.

    Returns replacements dictionary, mapping each old node to its new one.

    i - sequence of variables
    o - sequence of variables
    replacements - initial value for return value, modified in place.

    """
    if replacements is None:
        d = {}
    else:
        d = replacements

#    for old, new in replacements.items():
#        if new in replacements:
#            # I think we want to do something recursive here, but
#            # it feels like it might get tricky? This reminds me of the
#            # 'sorted_givens' branch on github/jaberg/Theano
#            raise NotImplementedError('think before implementing')
#        replacements[new] = new

    for input in i:
        if input not in d:
            d[input] = input

    for apply in graph.io_toposort(i, o):
        for input in apply.inputs:
            if input not in d:
                d[input] = input

        new_apply = apply.clone_with_new_inputs([d[i] for i in apply.inputs])
        if apply not in d:
            d[apply] = new_apply

        for output, new_output in zip(apply.outputs, new_apply.outputs):
            if output not in d:
                d[output] = new_output

    for output in o:
        if output not in d:
            d[output] = output.clone()

    return d
Ejemplo n.º 29
0
    def _get_variables(self):
        """Collect variables, updates and auxiliary variables."""
        updates = OrderedDict()

        shared_outputs = [o for o in self.outputs if is_shared_variable(o)]
        usual_outputs = [o for o in self.outputs if not is_shared_variable(o)]
        variables = shared_outputs

        if usual_outputs:
            # Sort apply nodes topologically, get variables and remove
            # duplicates
            inputs = graph.inputs(self.outputs)
            sorted_apply_nodes = graph.io_toposort(inputs, usual_outputs)

            seen = set()
            main_vars = [
                var for var in list(
                    chain(*[
                        apply_node.inputs for apply_node in sorted_apply_nodes
                    ])) if not (var in seen or seen.add(var))
            ] + self.outputs

            # While preserving order add auxiliary variables, and collect
            # updates
            seen = set()
            # Intermediate variables could be auxiliary
            seen_avs = set(main_vars)
            variables = []
            for var in main_vars:
                variables.append(var)
                for annotation in getattr(var.tag, 'annotations', []):
                    if annotation not in seen:
                        seen.add(annotation)
                        new_avs = [
                            av for av in annotation.auxiliary_variables
                            if not (av in seen_avs or seen_avs.add(av))
                        ]
                        variables.extend(new_avs)
                        updates = dict_union(updates, annotation.updates)

        self.variables = variables
        self.updates = updates
Ejemplo n.º 30
0
Archivo: fg.py Proyecto: poolio/Theano
    def __import__(self, apply_node, check=True, reason=None):
        """
        Given an apply_node, recursively search from this node to know graph,
        and then add all unknown variables and apply_nodes to this graph.
        """
        node = apply_node

        # We import the nodes in topological order. We only are interested
        # in new nodes, so we use all variables we know of as if they were the input set.
        # (the functions in the graph module only use the input set to
        # know where to stop going down)
        new_nodes = graph.io_toposort(self.variables, apply_node.outputs)

        if check:
            for node in new_nodes:
                if hasattr(node, "fgraph") and node.fgraph is not self:
                    raise Exception("%s is already owned by another fgraph" % node)
                for r in node.inputs:
                    if hasattr(r, "fgraph") and r.fgraph is not self:
                        raise Exception("%s is already owned by another fgraph" % r)
                    if r.owner is None and not isinstance(r, graph.Constant) and r not in self.inputs:
                        # Verbose error message
                        # Show a complete chain of variables from the missing input to an output
                        if config.exception_verbosity == "high":

                            def find_path_to(output_var, input_var):
                                """
                                Returns a list of each variable on a (not
                                necessarily unique) path from input_var to
                                output_var, where each variable in the list has
                                the preceding variable as one of its inputs.
                                Returns None if no path exists.

                                """
                                # If output and input are the same we have a singleton path
                                if output_var is input_var:
                                    return [output_var]

                                # If output has no inputs then there is no path
                                owner = output_var.owner

                                if owner is None:
                                    return None

                                # If input_var is an input to the output node, there is a
                                # simple two element path
                                inputs = owner.inputs

                                if input_var in inputs:
                                    return [input_var, output_var]

                                # Otherwise we must recurse by searching for a path to one
                                # of our inputs, then appending the output to that path
                                for ipt in inputs:
                                    path = find_path_to(ipt, input_var)

                                    if path is not None:
                                        path.append(output_var)

                                        return path

                                # Since none of the above methods returned a path, there is none
                                return None

                            # Try different outputs until we find one that has a path to the missing input
                            for output in self.outputs:
                                path = find_path_to(output, r)

                                if path is not None:
                                    break

                            # if there is no path then r isn't really a graph input so we shouldn't be running error
                            # handler code in the first place
                            assert path is not None
                            tr = getattr(r.tag, "trace", [])
                            detailed_err_msg = ""
                            if type(tr) is list and len(tr) > 0:
                                detailed_err_msg += "\nBacktrace when the variable is created:\n"

                                # Print separate message for each element in
                                # the list of batcktraces
                                sio = StringIO()
                                for subtr in tr:
                                    traceback.print_list(subtr, sio)
                                detailed_err_msg += str(sio.getvalue())
                            raise MissingInputError(
                                "A variable that is an input to the graph was "
                                "neither provided as an input to the function "
                                "nor given a value. A chain of variables "
                                "leading from this input to an output is %s. "
                                "This chain may not be unique" % str(path) + detailed_err_msg
                            )

                        # Standard error message
                        raise MissingInputError(
                            (
                                "An input of the graph, used to compute %s, "
                                "was not provided and not given a value."
                                "Use the Theano flag exception_verbosity='high',"
                                "for more information on this error." % str(node)
                            ),
                            r,
                        )

        for node in new_nodes:
            assert node not in self.apply_nodes
            self.__setup_node__(node)
            self.apply_nodes.add(node)
            for output in node.outputs:
                self.__setup_r__(output)
                self.variables.add(output)
            for i, input in enumerate(node.inputs):
                if input not in self.variables:
                    self.__setup_r__(input)
                    self.variables.add(input)
                self.__add_clients__(input, [(node, i)])
            assert node.fgraph is self
            self.execute_callbacks("on_import", node, reason)
Ejemplo n.º 31
0
    def __import__(self, apply_node, check=True, reason=None):
        node = apply_node

        # We import the nodes in topological order. We only are interested
        # in new nodes, so we use all variables we know of as if they were the input set.
        # (the functions in the graph module only use the input set to
        # know where to stop going down)
        new_nodes = graph.io_toposort(self.variables, node.outputs)

        if check:
            for node in new_nodes:
                if hasattr(node, 'fgraph') and node.fgraph is not self:
                    raise Exception("%s is already owned by another fgraph" % node)
                for r in node.inputs:
                    if hasattr(r, 'fgraph') and r.fgraph is not self:
                        raise Exception("%s is already owned by another fgraph" % r)
                    if (r.owner is None and
                        not isinstance(r, graph.Constant) and
                        r not in self.inputs):

                        #Verbose error message
                        #Show a complete chain of variables from the missing input to an output
                        if config.exception_verbosity == 'high':

                            def find_path_to(output_var, input_var):
                                """ Returns a list of each variable on a (not necessarily unique)
                                    path from input_var to output_var, where each variable in the
                                    list has the preceding variable as one of its inputs.
                                    Returns None if no path exists"""

                                #If output and input are the same we have a singleton path
                                if output_var is input_var:
                                    return [output_var]

                                #If output has no inputs then there is no path
                                owner = output_var.owner

                                if owner is None:
                                    return None

                                #If input_var is an input to the output node, there is a
                                #simple two element path
                                inputs = owner.inputs

                                if input_var in inputs:
                                    return [input_var, output_var]

                                #Otherwise we must recurse by searching for a path to one
                                #of our inputs, then appending the output to that path
                                for ipt in inputs:
                                    path = find_path_to(ipt, input_var)

                                    if path is not None:
                                        path.append(output_var)

                                        return path

                                #Since none of the above methods returned a path, there is none
                                return None

                            #Try different outputs until we find one that has a path to the missing input
                            for output in self.outputs:
                                path = find_path_to(output, r)

                                if path is not None:
                                    break

                            #if there is no path then r isn't really a graph input so we shouldn't be running error
                            #handler code in the first place
                            assert path is not None

                            raise MissingInputError((
                                'A variable that is an input to the graph was '
                                'neither provided as an input to the function '
                                'nor given a value. A chain of variables '
                                'leading from this input to an output is %s. '
                                'This chain may not be unique' % str(path)))

                        #Standard error message
                        raise MissingInputError((
                            "An input of the graph, used to compute %s, "
                            "was not provided and not given a value"
                            % str(node)),
                            r)

        for node in new_nodes:
            assert node not in self.apply_nodes
            self.__setup_node__(node)
            self.apply_nodes.add(node)
            for output in node.outputs:
                self.__setup_r__(output)
                self.variables.add(output)
            for i, input in enumerate(node.inputs):
                if input not in self.variables:
                    self.__setup_r__(input)
                    self.variables.add(input)
                self.__add_clients__(input, [(node, i)])
            assert node.fgraph is self
            self.execute_callbacks('on_import', node, reason)
Ejemplo n.º 32
0
def clone_get_equiv(i, o, replaced_inputs=[]):
    """ WRITEME

    :type i: list
    :param i: input L{Variable}s
    :type o: list
    :param o: output L{Variable}s
    :type copy_inputs_and_orphans: bool
    :param copy_inputs_and_orphans:
        if True, the inputs and the orphans will be replaced in the cloned graph by copies
        available in the equiv dictionary returned by the function (copy_inputs_and_orphans
        defaults to True)

    :rtype: a dictionary
    :return:
        equiv mapping each L{Variable} and L{Op} in the graph delimited by i and o to a copy
        (akin to deepcopy's memo).
    """

    from theano.gof.graph import io_toposort
    from theano.gof import Container
    from copy import deepcopy

    copy_inputs_and_orphans = True
    d = {}
    for input in i:
        if input in replaced_inputs:
            cpy = input.clone()
            # deep-copying the container, otherwise the copied input's container will point to the same place
            cont = input.container
            cpy.container = Container(
                cpy,
                storage=[
                    input.type.filter(deepcopy(cpy.value),
                                      strict=cont.strict,
                                      allow_downcast=cont.allow_downcast)
                ],
                readonly=cont.readonly,
                strict=cont.strict,
                allow_downcast=cont.allow_downcast)
            cpy.owner = None
            cpy.index = None
            d[input] = cpy
        else:
            d[input] = input

    for apply in io_toposort(i, o):
        for input in apply.inputs:
            if input not in d:
                if copy_inputs_and_orphans and input in replaced_inputs:
                    # TODO: not quite sure what to do here
                    cpy = input.clone()
                    d[input] = cpy
                else:
                    d[input] = input

        new_apply = apply.clone_with_new_inputs([d[i] for i in apply.inputs])
        d[apply] = new_apply
        for output, new_output in zip(apply.outputs, new_apply.outputs):
            d[output] = new_output

    for output in o:
        if output not in d:
            d[output] = output.clone()

    return d
Ejemplo n.º 33
0
    def inplace_elemwise_optimizer(fgraph):
        """
        Usage: inplace_elemwise_optimizer.optimize(fgraph)

        Attempts to replace all Broadcast ops by versions of them
        that operate inplace. It operates greedily: for each Broadcast
        Op that is encountered, for each output, tries each input to
        see if it can operate inplace on that input. If so, makes the
        change and go to the next output or Broadcast Op.

        Examples
        --------
        x + y + z -> x += y += z
        (x + y) * (x * y) -> (x += y) *= (x * y) or (x + y) *= (x *= y)

        """
        # We should not validate too often as this takes too much time to
        # execute!
        # It is the _dfs_toposort() fct in theano/gof/destroyhandler.py
        # that takes so much time.
        # Should we try to use another lib that does toposort?
        #   igraph: http://igraph.sourceforge.net/
        #   networkx: https://networkx.lanl.gov/
        # Should we try to use cython?
        #   Compiling only that fct is not enough, should we try to add the
        #   deque class too?
        #   And init the deque and other list to an upper bound number of
        #   elements?
        # Maybe Theano should do online toposort as in
        #   http://code.google.com/p/acyclic
        #
        # The next longest optimizer is the canonizer phase.
        # Then I think it is the [io_?]toposort (need to validate) so check if
        # the solution is also applicable there.

        # We execute `validate` after this number of change.
        check_each_change = config.tensor.insert_inplace_optimizer_validate_nb
        if check_each_change == -1:
            if len(fgraph.apply_nodes) > 500:
                check_each_change = 10
            else:
                check_each_change = 1

        nb_change_no_validate = 0
        chk = fgraph.checkpoint()

        if fgraph.update_mapping:
            update_outs = [fgraph.outputs[i] for i in fgraph.update_mapping]
        else:
            update_outs = []

        for node in list(graph.io_toposort(fgraph.inputs, fgraph.outputs)):
            op = node.op
            # gpuarray GpuElemwise inherit from Elemwise
            if not type(op) == OP:
                continue
            # If big graph and the outputs are scalar, do not make it
            # inplace.
            if (check_each_change != 1 and
                all([getattr(o.type, 'ndim', -1) == 0
                     for o in node.outputs])):
                continue

            baseline = op.inplace_pattern
            protected_inputs = [
                f.protected for f in node.fgraph._features if
                isinstance(f, theano.compile.function_module.Supervisor)]
            protected_inputs = sum(protected_inputs, [])  # flatten the list
            protected_inputs.extend(fgraph.outputs)
            candidate_outputs = [i for i in xrange(len(node.outputs))
                                 if i not in baseline]
            # node inputs that are Constant, already destroyed,
            # fgraph protected inputs and fgraph outputs can't be used as inplace
            # target.
            # Remove here as faster.
            candidate_inputs = [i for i in xrange(len(node.inputs))
                                if i not in baseline.values() and
                                not isinstance(node.inputs[i], Constant) and
                                not fgraph.destroyers(node.inputs[i]) and
                                node.inputs[i] not in protected_inputs]

            verbose = False

            raised_warning = not verbose

            for candidate_output in candidate_outputs:

                # If the output of the node can be established as an update
                # output of the fgraph, visit the candidate_inputs in an order
                # that will improve the chances of making the node operate
                # inplace on the input it's meant to update
                candidate_out_var = node.outputs[candidate_output]
                sorted_candidate_inputs = candidate_inputs

                if candidate_out_var in update_outs:

                    # The candidate output is an update. Sort the
                    # variables in candidate_inputs in the following order:
                    # - Vars corresponding to the actual updated input
                    #   (best case scenario is for the node that procudes
                    #   an update to operate inplace on the variable to
                    #   update)
                    # - Vars computed inplace on the updates input (second
                    #   best scenario if for the node to work inplace on
                    #   a variable obtained by a chain of inplace on the
                    #   variable to update. In some cases, this will be
                    #   equivalent to operating inplace on the variable to
                    #   update)
                    # - Remaining variables
                    updated_inputs = []
                    for i, f_out in enumerate(fgraph.outputs):
                        if (f_out is candidate_out_var and i in fgraph.update_mapping):
                            updated_inp_idx = fgraph.update_mapping[i]
                            updated_inputs.append(fgraph.inputs[updated_inp_idx])

                    updated_vars = []
                    vars_from_inplace = []
                    other_vars = []
                    for inp_idx in candidate_inputs:
                        inp = node.inputs[inp_idx]
                        if inp in updated_inputs:
                            # the candidate input is the actual updated input
                            updated_vars.append(inp_idx)
                        elif (hasattr(fgraph, 'destroy_handler') and
                              inp.owner and
                              any([fgraph.destroy_handler.root_destroyer.get(up_inp, None) is inp.owner
                                   for up_inp in updated_inputs])):

                            # the candidate input is a variable computed
                            # inplace on the updated input via a sequence of
                            # one or more inplace operations
                            vars_from_inplace.append(inp_idx)
                        else:
                            other_vars.append(inp_idx)

                    sorted_candidate_inputs = (updated_vars +
                                               vars_from_inplace + other_vars)

                for candidate_input in sorted_candidate_inputs:
                    # remove inputs that don't have the same dtype as the output
                    if node.inputs[candidate_input].type != node.outputs[
                            candidate_output].type:
                        continue

                    inplace_pattern = dict(baseline)
                    inplace_pattern[candidate_output] = candidate_input
                    try:
                        if hasattr(op.scalar_op, "make_new_inplace"):
                            new_scal = op.scalar_op.make_new_inplace(
                                scalar.transfer_type(
                                    *[inplace_pattern.get(i, o.dtype)
                                      for i, o in enumerate(node.outputs)]))
                        else:
                            new_scal = op.scalar_op.__class__(
                                scalar.transfer_type(
                                    *[inplace_pattern.get(i, None)
                                      for i in xrange(len(node.outputs))]))
                        new_outputs = OP(new_scal, inplace_pattern)(
                            *node.inputs, **dict(return_list=True))
                        new_node = new_outputs[0].owner

                        for r, new_r in zip(node.outputs, new_outputs):
                            fgraph.replace(r, new_r,
                                           reason="inplace_elemwise_optimizer")
                        nb_change_no_validate += 1
                        if nb_change_no_validate >= check_each_change:
                            fgraph.validate()
                            chk = fgraph.checkpoint()
                            nb_change_no_validate = 0
                    except (ValueError, InconsistencyError) as e:
                        if check_each_change != 1 and not raised_warning:
                            print(("Some inplace optimization was not "
                                   "performed due to unexpected error:"),
                                  file=sys.stderr)
                            print(e, file=sys.stderr)
                            raised_warning = True
                        fgraph.revert(chk)
                        continue
                    candidate_inputs.remove(candidate_input)
                    node = new_node
                    baseline = inplace_pattern
                    break

        if nb_change_no_validate > 0:
            try:
                fgraph.validate()
            except Exception:
                if not raised_warning:
                    print(("Some inplace optimization was not "
                           "performed due to unexpected error"),
                          file=sys.stderr)
                fgraph.revert(chk)
Ejemplo n.º 34
0
def replace_input_nodes(inputs,
                        outputs,
                        replacements=None,
                        memo=None,
                        clone_inputs=True):
    """Recreate a graph, replacing input variables according to a given map.

    This is helpful if you want to replace the variable dependencies of
    an existing variable according to a `clone_get_equiv` map and/or
    replacement variables that already exist within a `FunctionGraph`.

    The latter is especially annoying, because you can't simply make a
    `FunctionGraph` for the variable to be adjusted and then use that to
    perform the replacement; if the variables to be replaced are already in a
    `FunctionGraph` any such replacement will err-out saying "...these
    variables are already owned by another graph..."

    Parameters
    ----------
    inputs: list
        List of input nodes.
    outputs: list
        List of output nodes.  Everything between `inputs` and these `outputs`
        is the graph under consideration.
    replacements: dict (optional)
        A dictionary mapping existing nodes to their new ones.
        These values in this map will be used instead of newly generated
        clones.  This dict is not altered.
    memo: dict (optional)
        A dictionary to update with the initial `replacements` and maps from
        any old-to-new nodes arising from an actual replacement.
        It serves the same role as `replacements`, but it is updated
        as elements are cloned.
    clone_inputs: bool (optional)
        If enabled, clone all the input nodes that aren't mapped in
        `replacements`.  These cloned nodes are mapped in `memo`, as well.

    Results
    -------
    out: memo

    """
    if memo is None:
        memo = {}
    if replacements is not None:
        memo.update(replacements)
    for apply in io_toposort(inputs, outputs):

        walked_inputs = []
        for i in apply.inputs:
            if clone_inputs:
                # TODO: What if all the inputs are in the memo?
                walked_inputs.append(memo.setdefault(i, i.clone()))
            else:
                walked_inputs.append(walk(i, memo))

        if any(w != i for w, i in zip(apply.inputs, walked_inputs)):
            new_apply = apply.clone_with_new_inputs(walked_inputs)

            memo.setdefault(apply, new_apply)
            for output, new_output in zip(apply.outputs, new_apply.outputs):
                memo.setdefault(output, new_output)
    return memo
Ejemplo n.º 35
0
    def _get_variables(self):
        """Collect variables, updates and auxiliary variables.

        In addition collects all :class:`.Scan` ops and recurses in the
        respective inner Theano graphs.

        """
        updates = OrderedDict()

        shared_outputs = [o for o in self.outputs if is_trainable_variable(o)]
        usual_outputs = [
            o for o in self.outputs if not is_trainable_variable(o)
        ]
        variables = shared_outputs

        if usual_outputs:
            # Sort apply nodes topologically, get variables and remove
            # duplicates
            inputs = graph.inputs(self.outputs)
            sorted_apply_nodes = graph.io_toposort(inputs, usual_outputs)
            self.scans = list(
                _unique([
                    node.op for node in sorted_apply_nodes
                    if isinstance(node.op, Scan)
                ],
                        key=lambda op: id(op)))
            self._scan_graphs = [
                ComputationGraph(scan.outputs) for scan in self.scans
            ]

            seen = set()
            main_vars = ([
                var for var in list(
                    chain(*[
                        apply_node.inputs for apply_node in sorted_apply_nodes
                    ])) if not (var in seen or seen.add(var))
            ] + [var for var in self.outputs if var not in seen])

            # While preserving order add auxiliary variables, and collect
            # updates
            seen = set()
            # Intermediate variables could be auxiliary
            seen_avs = set(main_vars)
            variables = []
            for var in main_vars:
                variables.append(var)
                # updates
                _ = getattr(var.tag, 'updates', OrderedDict())
                _ = OrderedDict([(i, j) for i, j in _.iteritems()
                                 if is_variable(i)])
                updates = dict_union(updates, _)
                # auxiliary_variables
                for _ in getattr(var.tag, 'auxiliary_variables', []):
                    if _ not in seen and \
                    not (_ in seen_avs or seen_avs.add(_)):
                        variables.append(_)

        # If trainable_variables is assigned default_update (cloned), we cannot eval()
        # it to get the real numpy array value, hence, try to trace back
        # original shared variable
        def shared_variable_filter(var):
            if is_trainable_variable(var) and hasattr(var, 'default_update'):
                for v in _CREATED_VARIABLE.values():
                    if v.name == var.name and v.ndim == var.ndim:
                        return v
            return var

        self.variables = map(shared_variable_filter, variables)
        self.updates = updates
Ejemplo n.º 36
0
def clone_get_equiv(i, o, replaced_inputs = []):
    """ WRITEME

    :type i: list
    :param i: input L{Variable}s
    :type o: list
    :param o: output L{Variable}s
    :type copy_inputs_and_orphans: bool
    :param copy_inputs_and_orphans:
        if True, the inputs and the orphans will be replaced in the cloned graph by copies
        available in the equiv dictionary returned by the function (copy_inputs_and_orphans
        defaults to True)

    :rtype: a dictionary
    :return:
        equiv mapping each L{Variable} and L{Op} in the graph delimited by i and o to a copy
        (akin to deepcopy's memo).
    """

    from theano.gof.graph import io_toposort
    from theano.gof import Container
    from copy import deepcopy

    copy_inputs_and_orphans = True
    d = {}
    for input in i:
        if input in replaced_inputs:
            cpy = input.clone()
            # deep-copying the container, otherwise the copied input's container will point to the same place
            cont = input.container
            cpy.container = Container(cpy,
                    storage=[input.type.filter(deepcopy(cpy.value), strict=cont.strict, allow_downcast=cont.allow_downcast)],
                    readonly=cont.readonly,
                    strict=cont.strict,
                    allow_downcast=cont.allow_downcast)
            cpy.owner = None
            cpy.index = None
            d[input] = cpy
        else:
            d[input] = input

    for apply in io_toposort(i, o):
        for input in apply.inputs:
            if input not in d:
                if copy_inputs_and_orphans and input in replaced_inputs:
                    # TODO: not quite sure what to do here
                    cpy = input.clone()
                    d[input] = cpy
                else:
                    d[input] = input

        new_apply = apply.clone_with_new_inputs([d[i] for i in apply.inputs])
        d[apply] = new_apply
        for output, new_output in zip(apply.outputs, new_apply.outputs):
            d[output] = new_output

    for output in o:
        if output not in d:
            d[output] = output.clone()

    return d
Ejemplo n.º 37
0
 def on_detach(self, fgraph):
     for node in graph.io_toposort(fgraph.inputs, fgraph.outputs):
         self.on_prune(fgraph, node, 'Bookkeeper.detach')
Ejemplo n.º 38
0
 def on_attach(self, fgraph):
     for node in graph.io_toposort(fgraph.inputs, fgraph.outputs):
         self.on_import(fgraph, node, "on_attach")
Ejemplo n.º 39
0
 def on_detach(self, fgraph):
     for node in graph.io_toposort(fgraph.inputs, fgraph.outputs):
         self.on_prune(fgraph, node, 'Bookkeeper.detach')
Ejemplo n.º 40
0
 def on_attach(self, fgraph):
     for node in graph.io_toposort(fgraph.inputs, fgraph.outputs):
         self.on_import(fgraph, node, "on_attach")