def test_array_split_extention_axis_1(self):
        a = np.array([[1, 2, 3],
                      [4, 5, 6]])

        split_extended = array_extend(a, {1: [1]})

        np.testing.assert_array_almost_equal(split_extended, np.array([[1, 2, 3, 2], [4, 5, 6, 5]]))
Exemplo n.º 2
0
    def test_array_split_extention_axis_1(self):
        a = np.array([[1, 2, 3], [4, 5, 6]])

        split_extended = array_extend(a, {1: [1]})

        np.testing.assert_array_almost_equal(
            split_extended, np.array([[1, 2, 3, 2], [4, 5, 6, 5]]))
Exemplo n.º 3
0
    def test_array_split_extention_vector(self):
        a = np.array([1, 2, 3])

        split_extended = array_extend(a, {0: [0]})

        np.testing.assert_array_almost_equal(split_extended,
                                             np.array([1, 2, 3, 1]))
Exemplo n.º 4
0
    def test_array_split_extention_axis_2(self):
        a = np.array([[1, 2, 3], [4, 5, 6]])

        split_extended = array_extend(a, {0: [0]})

        np.testing.assert_array_almost_equal(
            split_extended, np.array([[1, 2, 3], [4, 5, 6], [1, 2, 3]]))
Exemplo n.º 5
0
    def test_array_split_extention_halve_splits(self):
        a = np.array([[2., 4., 8.], [1., 2., 3.]])

        split_extended = array_extend(a, {0: [0]}, halve_extended_vectors=True)

        np.testing.assert_array_almost_equal(
            split_extended, np.array([[1., 2., 4.], [1., 2., 3.], [1., 2.,
                                                                   4.]]))
Exemplo n.º 6
0
    def test_array_split_extention_axis_3(self):
        a = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])

        split_extended = array_extend(a, {2: [0]})

        np.testing.assert_array_almost_equal(
            split_extended,
            np.array([[[1, 2, 1], [3, 4, 3]], [[5, 6, 5], [7, 8, 7]]]))
    def test_array_split_extention_halve_splits(self):
        a = np.array([[2., 4., 8.],
                      [1., 2., 3.]])

        split_extended = array_extend(a, {0: [0]}, halve_extended_vectors=True)

        np.testing.assert_array_almost_equal(split_extended, np.array([[1., 2., 4.],
                                                                       [1., 2., 3.],
                                                                       [1., 2., 4.]]))
    def test_array_split_extention_axis_2(self):
        a = np.array([[1, 2, 3],
                      [4, 5, 6]])

        split_extended = array_extend(a, {0: [0]})

        np.testing.assert_array_almost_equal(split_extended, np.array([[1, 2, 3],
                                                                       [4, 5, 6],
                                                                       [1, 2, 3]]))
Exemplo n.º 9
0
    def resize(self, new_output_nodes=None,
               output_nodes_to_prune=None,
               input_nodes_to_prune=None,
               split_output_nodes=None,
               split_input_nodes=None,
               data_set_train=None,
               data_set_validation=None,
               no_splitting_or_pruning=False,
               split_nodes_noise_std=.1):
        """Resize this layer by changing the number of output nodes. Will also resize any downstream layers

        Args:
            data_set_validation (DataSet):Data set used for validating this network
            data_set_train (DataSet): Data set used for training this network
            no_splitting_or_pruning (bool): If set to true then noise is just added randomly rather than splitting nodes
            new_output_nodes (int | tuple of ints): If passed we change the number of output nodes of this layer to be new_output_nodes
            output_nodes_to_prune ([int]): list of indexes of the output nodes we want pruned e.g. [1, 3] would remove
                the 1st and 3rd output node from this layer
            input_nodes_to_prune ([int]): list of indexes of the input nodes we want pruned e.g. [1, 3] would remove the
                1st and 3rd input node from this layer
            split_output_nodes ([int]): list of indexes of nodes to split. This is for growing the layer
            split_input_nodes: ([int]): list of indexes of nodes that where split in the previous layer.
            split_nodes_noise_std (float): standard deviation of noise to add when splitting a node
        """
        if isinstance(new_output_nodes, tuple):
            new_output_nodes = new_output_nodes[self.get_resizable_dimension()]
        elif new_output_nodes is not None and not isinstance(new_output_nodes, int):
            raise ValueError("new_output_nodes must be tuple of int %s" % (new_output_nodes,))

        if not no_splitting_or_pruning:
            # choose nodes to split or prune
            if new_output_nodes is not None:
                if output_nodes_to_prune is None and split_output_nodes is None:
                    if new_output_nodes < self.get_resizable_dimension_size():
                        output_nodes_to_prune = self._choose_nodes_to_prune(new_output_nodes, data_set_train,
                                                                            data_set_validation)
                    elif new_output_nodes > self.get_resizable_dimension_size():
                        split_output_nodes = self._choose_nodes_to_split(new_output_nodes, data_set_train,
                                                                         data_set_validation)
            elif self.has_resizable_dimension():
                new_output_nodes = self.get_resizable_dimension_size()
                if output_nodes_to_prune:
                    new_output_nodes -= len(output_nodes_to_prune)
                if split_output_nodes:
                    new_output_nodes += len(split_output_nodes)

        new_input_nodes = self.input_layer.output_nodes
        input_nodes_changed = new_input_nodes != self._input_nodes

        if self.has_resizable_dimension() and new_output_nodes is not None:
            output_nodes_changed = new_output_nodes != self.get_resizable_dimension_size()
            temp_output_nodes = list(self._output_nodes)
            temp_output_nodes[self.get_resizable_dimension()] = new_output_nodes

            self._output_nodes = tuple(temp_output_nodes)
        else:
            output_nodes_changed = False

        self._input_nodes = new_input_nodes

        for name, bound_variable in self._bound_variables.iteritems():
            if input_nodes_changed and self._bound_dimensions_contains_input(bound_variable.dimensions) or \
                            output_nodes_changed and self._bound_dimensions_contains_output(bound_variable.dimensions):

                self._forget_assign_op(name)

                int_dims = self._bound_dimensions_to_ints(bound_variable.dimensions)

                if isinstance(bound_variable.variable, tf.Variable):
                    old_values = self._session.run(bound_variable.variable)
                    if output_nodes_to_prune or split_output_nodes:
                        output_bound_axis = self._get_axis(bound_variable.dimensions,
                                                           self.OUTPUT_BOUND_VALUE, self.OUTPUT_DIM_3_BOUND_VALUE)
                        if output_nodes_to_prune:
                            old_values = np.delete(old_values, output_nodes_to_prune, output_bound_axis)
                        else:  # split
                            old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
                                                      noise_std=split_nodes_noise_std)
                    if input_nodes_to_prune or split_input_nodes:
                        input_bound_axis = self._get_axis(bound_variable.dimensions,
                                                          self.INPUT_BOUND_VALUE, self.INPUT_DIM_3_BOUND_VALUE)
                        if input_nodes_to_prune:
                            old_values = np.delete(old_values, input_nodes_to_prune, input_bound_axis)
                        else:  # split
                            old_values = array_extend(old_values, {input_bound_axis: split_input_nodes},
                                                      halve_extended_vectors=True)
                    if no_splitting_or_pruning:
                        new_values = self._weight_extender_func(old_values, int_dims)
                    else:
                        new_values = old_values

                    tf_resize(self._session, bound_variable.variable, int_dims,
                              new_values, self._get_assign_function(name))
                else:
                    # this is a tensor, not a variable so has no weights
                    tf_resize(self._session, bound_variable.variable, int_dims)

        if input_nodes_changed and self._batch_normalize_input:
            if self._batch_norm_mean_train is not None:
                tf_resize(self._session, self._batch_norm_mean_train, self._input_nodes)
                tf_resize(self._session, self._batch_norm_var_train, self._input_nodes)
            if self._batch_norm_mean_predict is not None:
                tf_resize(self._session, self._batch_norm_mean_predict, self._input_nodes)
                tf_resize(self._session, self._batch_norm_var_predict, self._input_nodes)
            if self._normalized_train is not None:
                tf_resize(self._session, self._normalized_train, (None,) + self._input_nodes)
            if self._normalized_predict is not None:
                tf_resize(self._session, self._normalized_predict, (None,) + self._input_nodes)
                tf_resize(self.session, self._normalized_predict.op.inputs[0].op.inputs[1].op.inputs[1], self._input_nodes)
            # THIS needs fixing -> self._normalized_predict.op.inputs[0].op.inputs[1].op.inputs[1]
            # self._normalized_predict.op.inputs[0].op.inputs[1].op.inputs[1].op.inputs[1] returns [2] and should be [4]

            # This line fixed the issue, this is all very hacky...
            # self._mat_mul.op.inputs[0]._shape = TensorShape((None,) + self._input_nodes)
            from tensorflow.python.framework.tensor_shape import TensorShape

            if '_mat_mul_is_train_equal_' + str(True) in self.__dict__:
                self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0]._shape = TensorShape(
                    (None,) + self._input_nodes)
                self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0].op.inputs[0]._shape = TensorShape(
                    (None,) + self._input_nodes)
                self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0].op.inputs[0].op.inputs[
                    0]._shape = TensorShape((None,) + self._input_nodes)
                self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0].op.inputs[0].op.inputs[0].op.inputs[
                    0]._shape = TensorShape((None,) + self._input_nodes)
                # tf_resize(self._session, self.__dict__['_mat_mul_is_train_equal_' + str(True)], (None,) + self._input_nodes)
            if '_mat_mul_is_train_equal_' + str(False) in self.__dict__:
                self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0]._shape = TensorShape(
                    (None,) + self._input_nodes)
                self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0].op.inputs[0]._shape = TensorShape(
                    (None,) + self._input_nodes)
                self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0].op.inputs[0].op.inputs[
                    0]._shape = TensorShape((None,) + self._input_nodes)
                self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0].op.inputs[0].op.inputs[0].op.inputs[
                    0]._shape = TensorShape((None,) + self._input_nodes)
                # tf_resize(self._session, self.__dict__['_mat_mul_is_train_equal_' + str(False)], (None,) + self._input_nodes)

        if output_nodes_changed:
            if has_lazyprop(self, 'activation_predict'):
                tf_resize(self._session, self.activation_predict, (None,) + self._output_nodes)
            if has_lazyprop(self, 'activation_train'):
                tf_resize(self._session, self.activation_train, (None,) + self._output_nodes)

        if input_nodes_changed and self.bactivate:
            if has_lazyprop(self, 'bactivation_train'):
                tf_resize(self._session, self.bactivation_train, (None,) + self._input_nodes)
            if has_lazyprop(self, 'bactivation_predict'):
                tf_resize(self._session, self.bactivation_predict, (None,) + self._input_nodes)

        if self._next_layer and self._next_layer._resize_needed():
            self._next_layer.resize(input_nodes_to_prune=output_nodes_to_prune, split_input_nodes=split_output_nodes,
                                    no_splitting_or_pruning=no_splitting_or_pruning)
    def test_array_split_extention_vector(self):
        a = np.array([1, 2, 3])

        split_extended = array_extend(a, {0: [0]})

        np.testing.assert_array_almost_equal(split_extended, np.array([1, 2, 3, 1]))
Exemplo n.º 11
0
    def resize(self, new_output_nodes=None, output_nodes_to_prune=None, input_nodes_to_prune=None,
               split_output_nodes=None,
               split_input_nodes=None,
               split_nodes_noise_std=.01):
        """Resize this layer by changing the number of output nodes. Will also resize any downstream layers

        Args:
            new_output_nodes (int): If passed we change the number of output nodes of this layer to be new_output_nodes
                Otherwise we change the size to current output nodes+1
            output_nodes_to_prune ([int]): list of indexes of the output nodes we want pruned e.g. [1, 3] would remove
                the 1st and 3rd output node from this layer
            input_nodes_to_prune ([int]): list of indexes of the input nodes we want pruned e.g. [1, 3] would remove the
                1st and 3rd input node from this layer
            split_output_nodes ([int]): list of indexes of nodes to split. This is for growing the layer
            split_input_nodes: ([int]): list of indexes of nodes that where split in the prevous layer.
            split_nodes_noise_std (float): standard deviation of noise to add when splitting a node
        """
        if output_nodes_to_prune:
            if split_output_nodes:
                raise NotImplementedError("At the moment must either split or prune")
            if not (new_output_nodes is None or new_output_nodes != self._output_nodes - len(output_nodes_to_prune)):
                raise Exception("Different number of output nodes set from that left after pruning")
            else:
                new_output_nodes = self._output_nodes - len(output_nodes_to_prune)
        elif split_output_nodes:
            if not (new_output_nodes is None or new_output_nodes != self._output_nodes + len(split_output_nodes)):
                raise Exception("Different number of output nodes set from that left after splitting")
            else:
                new_output_nodes = self._output_nodes + len(split_output_nodes)
        else:
            new_output_nodes = new_output_nodes or self._output_nodes

        new_input_nodes = self.input_layer.output_nodes
        input_nodes_changed = new_input_nodes != self._input_nodes
        output_nodes_changed = new_output_nodes != self._output_nodes

        self._output_nodes = new_output_nodes
        self._input_nodes = new_input_nodes

        for bound_variable in self._bound_variables:
            if input_nodes_changed and self._bound_dimensions_contains_input(bound_variable.dimensions) or \
                            output_nodes_changed and self._bound_dimensions_contains_output(bound_variable.dimensions):
                int_dims = self._bound_dimensions_to_ints(bound_variable.dimensions)

                if isinstance(bound_variable.variable, tf.Variable):
                    old_values = self._session.run(bound_variable.variable)
                    if output_nodes_to_prune or split_output_nodes:
                        try:
                            output_bound_axis = bound_variable.dimensions.index(self.OUTPUT_BOUND_VALUE)
                            if output_nodes_to_prune:
                                old_values = np.delete(old_values, output_nodes_to_prune, output_bound_axis)
                            else:  # split
                                old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
                                                          noise_std=split_nodes_noise_std)
                        except ValueError, e:
                            pass
                    if input_nodes_to_prune or split_input_nodes:
                        try:
                            input_bound_axis = bound_variable.dimensions.index(self.INPUT_BOUND_VALUE)
                            if input_nodes_to_prune:
                                old_values = np.delete(old_values, input_nodes_to_prune, input_bound_axis)
                            else:  # split
                                old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
                                                          noise_std=split_nodes_noise_std)
                        except ValueError, e:
                            pass

                    new_values = self._weight_extender_func(old_values, int_dims)

                    tf_resize(self._session, bound_variable.variable, int_dims,
                              new_values)
                else:
                    # this is a tensor so no need to provide values
                    tf_resize(self._session, bound_variable.variable, int_dims)