def test_array_split_extention_axis_1(self):
a = np.array([[1, 2, 3],
[4, 5, 6]])
split_extended = array_extend(a, {1: [1]})
np.testing.assert_array_almost_equal(split_extended, np.array([[1, 2, 3, 2], [4, 5, 6, 5]]))
def test_array_split_extention_axis_1(self):
a = np.array([[1, 2, 3], [4, 5, 6]])
split_extended = array_extend(a, {1: [1]})
np.testing.assert_array_almost_equal(
split_extended, np.array([[1, 2, 3, 2], [4, 5, 6, 5]]))
def test_array_split_extention_vector(self):
a = np.array([1, 2, 3])
split_extended = array_extend(a, {0: [0]})
np.testing.assert_array_almost_equal(split_extended,
np.array([1, 2, 3, 1]))
def test_array_split_extention_axis_2(self):
a = np.array([[1, 2, 3], [4, 5, 6]])
split_extended = array_extend(a, {0: [0]})
np.testing.assert_array_almost_equal(
split_extended, np.array([[1, 2, 3], [4, 5, 6], [1, 2, 3]]))
def test_array_split_extention_halve_splits(self):
a = np.array([[2., 4., 8.], [1., 2., 3.]])
split_extended = array_extend(a, {0: [0]}, halve_extended_vectors=True)
np.testing.assert_array_almost_equal(
split_extended, np.array([[1., 2., 4.], [1., 2., 3.], [1., 2.,
4.]]))
def test_array_split_extention_axis_3(self):
a = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
split_extended = array_extend(a, {2: [0]})
np.testing.assert_array_almost_equal(
split_extended,
np.array([[[1, 2, 1], [3, 4, 3]], [[5, 6, 5], [7, 8, 7]]]))
def test_array_split_extention_halve_splits(self):
a = np.array([[2., 4., 8.],
[1., 2., 3.]])
split_extended = array_extend(a, {0: [0]}, halve_extended_vectors=True)
np.testing.assert_array_almost_equal(split_extended, np.array([[1., 2., 4.],
[1., 2., 3.],
[1., 2., 4.]]))
def test_array_split_extention_axis_2(self):
a = np.array([[1, 2, 3],
[4, 5, 6]])
split_extended = array_extend(a, {0: [0]})
np.testing.assert_array_almost_equal(split_extended, np.array([[1, 2, 3],
[4, 5, 6],
[1, 2, 3]]))
def resize(self, new_output_nodes=None,
output_nodes_to_prune=None,
input_nodes_to_prune=None,
split_output_nodes=None,
split_input_nodes=None,
data_set_train=None,
data_set_validation=None,
no_splitting_or_pruning=False,
split_nodes_noise_std=.1):
"""Resize this layer by changing the number of output nodes. Will also resize any downstream layers
Args:
data_set_validation (DataSet):Data set used for validating this network
data_set_train (DataSet): Data set used for training this network
no_splitting_or_pruning (bool): If set to true then noise is just added randomly rather than splitting nodes
new_output_nodes (int | tuple of ints): If passed we change the number of output nodes of this layer to be new_output_nodes
output_nodes_to_prune ([int]): list of indexes of the output nodes we want pruned e.g. [1, 3] would remove
the 1st and 3rd output node from this layer
input_nodes_to_prune ([int]): list of indexes of the input nodes we want pruned e.g. [1, 3] would remove the
1st and 3rd input node from this layer
split_output_nodes ([int]): list of indexes of nodes to split. This is for growing the layer
split_input_nodes: ([int]): list of indexes of nodes that where split in the previous layer.
split_nodes_noise_std (float): standard deviation of noise to add when splitting a node
"""
if isinstance(new_output_nodes, tuple):
new_output_nodes = new_output_nodes[self.get_resizable_dimension()]
elif new_output_nodes is not None and not isinstance(new_output_nodes, int):
raise ValueError("new_output_nodes must be tuple of int %s" % (new_output_nodes,))
if not no_splitting_or_pruning:
# choose nodes to split or prune
if new_output_nodes is not None:
if output_nodes_to_prune is None and split_output_nodes is None:
if new_output_nodes < self.get_resizable_dimension_size():
output_nodes_to_prune = self._choose_nodes_to_prune(new_output_nodes, data_set_train,
data_set_validation)
elif new_output_nodes > self.get_resizable_dimension_size():
split_output_nodes = self._choose_nodes_to_split(new_output_nodes, data_set_train,
data_set_validation)
elif self.has_resizable_dimension():
new_output_nodes = self.get_resizable_dimension_size()
if output_nodes_to_prune:
new_output_nodes -= len(output_nodes_to_prune)
if split_output_nodes:
new_output_nodes += len(split_output_nodes)
new_input_nodes = self.input_layer.output_nodes
input_nodes_changed = new_input_nodes != self._input_nodes
if self.has_resizable_dimension() and new_output_nodes is not None:
output_nodes_changed = new_output_nodes != self.get_resizable_dimension_size()
temp_output_nodes = list(self._output_nodes)
temp_output_nodes[self.get_resizable_dimension()] = new_output_nodes
self._output_nodes = tuple(temp_output_nodes)
else:
output_nodes_changed = False
self._input_nodes = new_input_nodes
for name, bound_variable in self._bound_variables.iteritems():
if input_nodes_changed and self._bound_dimensions_contains_input(bound_variable.dimensions) or \
output_nodes_changed and self._bound_dimensions_contains_output(bound_variable.dimensions):
self._forget_assign_op(name)
int_dims = self._bound_dimensions_to_ints(bound_variable.dimensions)
if isinstance(bound_variable.variable, tf.Variable):
old_values = self._session.run(bound_variable.variable)
if output_nodes_to_prune or split_output_nodes:
output_bound_axis = self._get_axis(bound_variable.dimensions,
self.OUTPUT_BOUND_VALUE, self.OUTPUT_DIM_3_BOUND_VALUE)
if output_nodes_to_prune:
old_values = np.delete(old_values, output_nodes_to_prune, output_bound_axis)
else: # split
old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
noise_std=split_nodes_noise_std)
if input_nodes_to_prune or split_input_nodes:
input_bound_axis = self._get_axis(bound_variable.dimensions,
self.INPUT_BOUND_VALUE, self.INPUT_DIM_3_BOUND_VALUE)
if input_nodes_to_prune:
old_values = np.delete(old_values, input_nodes_to_prune, input_bound_axis)
else: # split
old_values = array_extend(old_values, {input_bound_axis: split_input_nodes},
halve_extended_vectors=True)
if no_splitting_or_pruning:
new_values = self._weight_extender_func(old_values, int_dims)
else:
new_values = old_values
tf_resize(self._session, bound_variable.variable, int_dims,
new_values, self._get_assign_function(name))
else:
# this is a tensor, not a variable so has no weights
tf_resize(self._session, bound_variable.variable, int_dims)
if input_nodes_changed and self._batch_normalize_input:
if self._batch_norm_mean_train is not None:
tf_resize(self._session, self._batch_norm_mean_train, self._input_nodes)
tf_resize(self._session, self._batch_norm_var_train, self._input_nodes)
if self._batch_norm_mean_predict is not None:
tf_resize(self._session, self._batch_norm_mean_predict, self._input_nodes)
tf_resize(self._session, self._batch_norm_var_predict, self._input_nodes)
if self._normalized_train is not None:
tf_resize(self._session, self._normalized_train, (None,) + self._input_nodes)
if self._normalized_predict is not None:
tf_resize(self._session, self._normalized_predict, (None,) + self._input_nodes)
tf_resize(self.session, self._normalized_predict.op.inputs[0].op.inputs[1].op.inputs[1], self._input_nodes)
# THIS needs fixing -> self._normalized_predict.op.inputs[0].op.inputs[1].op.inputs[1]
# self._normalized_predict.op.inputs[0].op.inputs[1].op.inputs[1].op.inputs[1] returns [2] and should be [4]
# This line fixed the issue, this is all very hacky...
# self._mat_mul.op.inputs[0]._shape = TensorShape((None,) + self._input_nodes)
from tensorflow.python.framework.tensor_shape import TensorShape
if '_mat_mul_is_train_equal_' + str(True) in self.__dict__:
self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0]._shape = TensorShape(
(None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0].op.inputs[0]._shape = TensorShape(
(None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0].op.inputs[0].op.inputs[
0]._shape = TensorShape((None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0].op.inputs[0].op.inputs[0].op.inputs[
0]._shape = TensorShape((None,) + self._input_nodes)
# tf_resize(self._session, self.__dict__['_mat_mul_is_train_equal_' + str(True)], (None,) + self._input_nodes)
if '_mat_mul_is_train_equal_' + str(False) in self.__dict__:
self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0]._shape = TensorShape(
(None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0].op.inputs[0]._shape = TensorShape(
(None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0].op.inputs[0].op.inputs[
0]._shape = TensorShape((None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0].op.inputs[0].op.inputs[0].op.inputs[
0]._shape = TensorShape((None,) + self._input_nodes)
# tf_resize(self._session, self.__dict__['_mat_mul_is_train_equal_' + str(False)], (None,) + self._input_nodes)
if output_nodes_changed:
if has_lazyprop(self, 'activation_predict'):
tf_resize(self._session, self.activation_predict, (None,) + self._output_nodes)
if has_lazyprop(self, 'activation_train'):
tf_resize(self._session, self.activation_train, (None,) + self._output_nodes)
if input_nodes_changed and self.bactivate:
if has_lazyprop(self, 'bactivation_train'):
tf_resize(self._session, self.bactivation_train, (None,) + self._input_nodes)
if has_lazyprop(self, 'bactivation_predict'):
tf_resize(self._session, self.bactivation_predict, (None,) + self._input_nodes)
if self._next_layer and self._next_layer._resize_needed():
self._next_layer.resize(input_nodes_to_prune=output_nodes_to_prune, split_input_nodes=split_output_nodes,
no_splitting_or_pruning=no_splitting_or_pruning)
def test_array_split_extention_vector(self):
a = np.array([1, 2, 3])
split_extended = array_extend(a, {0: [0]})
np.testing.assert_array_almost_equal(split_extended, np.array([1, 2, 3, 1]))
def resize(self, new_output_nodes=None, output_nodes_to_prune=None, input_nodes_to_prune=None,
split_output_nodes=None,
split_input_nodes=None,
split_nodes_noise_std=.01):
"""Resize this layer by changing the number of output nodes. Will also resize any downstream layers
Args:
new_output_nodes (int): If passed we change the number of output nodes of this layer to be new_output_nodes
Otherwise we change the size to current output nodes+1
output_nodes_to_prune ([int]): list of indexes of the output nodes we want pruned e.g. [1, 3] would remove
the 1st and 3rd output node from this layer
input_nodes_to_prune ([int]): list of indexes of the input nodes we want pruned e.g. [1, 3] would remove the
1st and 3rd input node from this layer
split_output_nodes ([int]): list of indexes of nodes to split. This is for growing the layer
split_input_nodes: ([int]): list of indexes of nodes that where split in the prevous layer.
split_nodes_noise_std (float): standard deviation of noise to add when splitting a node
"""
if output_nodes_to_prune:
if split_output_nodes:
raise NotImplementedError("At the moment must either split or prune")
if not (new_output_nodes is None or new_output_nodes != self._output_nodes - len(output_nodes_to_prune)):
raise Exception("Different number of output nodes set from that left after pruning")
else:
new_output_nodes = self._output_nodes - len(output_nodes_to_prune)
elif split_output_nodes:
if not (new_output_nodes is None or new_output_nodes != self._output_nodes + len(split_output_nodes)):
raise Exception("Different number of output nodes set from that left after splitting")
else:
new_output_nodes = self._output_nodes + len(split_output_nodes)
else:
new_output_nodes = new_output_nodes or self._output_nodes
new_input_nodes = self.input_layer.output_nodes
input_nodes_changed = new_input_nodes != self._input_nodes
output_nodes_changed = new_output_nodes != self._output_nodes
self._output_nodes = new_output_nodes
self._input_nodes = new_input_nodes
for bound_variable in self._bound_variables:
if input_nodes_changed and self._bound_dimensions_contains_input(bound_variable.dimensions) or \
output_nodes_changed and self._bound_dimensions_contains_output(bound_variable.dimensions):
int_dims = self._bound_dimensions_to_ints(bound_variable.dimensions)
if isinstance(bound_variable.variable, tf.Variable):
old_values = self._session.run(bound_variable.variable)
if output_nodes_to_prune or split_output_nodes:
try:
output_bound_axis = bound_variable.dimensions.index(self.OUTPUT_BOUND_VALUE)
if output_nodes_to_prune:
old_values = np.delete(old_values, output_nodes_to_prune, output_bound_axis)
else: # split
old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
noise_std=split_nodes_noise_std)
except ValueError, e:
pass
if input_nodes_to_prune or split_input_nodes:
try:
input_bound_axis = bound_variable.dimensions.index(self.INPUT_BOUND_VALUE)
if input_nodes_to_prune:
old_values = np.delete(old_values, input_nodes_to_prune, input_bound_axis)
else: # split
old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
noise_std=split_nodes_noise_std)
except ValueError, e:
pass
new_values = self._weight_extender_func(old_values, int_dims)
tf_resize(self._session, bound_variable.variable, int_dims,
new_values)
else:
# this is a tensor so no need to provide values
tf_resize(self._session, bound_variable.variable, int_dims)