def test_tf_resize_new_values(self):
var = tf.Variable(range(20))
self.session.run(tf.initialize_variables([var]))
tf_resize(self.session, var, new_values=np.array(range(10)))
self.assertEqual(len(self.session.run(var)), 10)
def test_tf_resize_new_values(self):
var = tf.Variable(range(20))
self.session.run(tf.initialize_variables([var]))
tf_resize(self.session, var, new_values=np.array(range(10)))
self.assertEqual(len(self.session.run(var)), 10)
def test_tf_resize_grow(self):
zeros = tf.zeros((3, ))
var = tf.Variable(initial_value=zeros)
self.session.run(tf.initialize_variables([var]))
tf_resize(self.session, var, new_dimensions=(6, ))
self.assertEqual(self.session.run(var).shape, (6, ))
def test_reshape_tensor(self):
input = tf.placeholder('float', shape=(4, ))
output = tf.reshape(input, (2, 2))
print self.session.run(output, feed_dict={input: [1, 2, 3, 4]})
tf_resize(self.session, output, new_dimensions=(1, 4))
assert self.session.run(output, feed_dict={input:
[1, 2, 3,
4]}) == [[1, 2, 3, 4]]
def test_tf_resize_shrink_twice(self):
zeros = tf.zeros((6, ))
var = tf.Variable(initial_value=zeros)
self.session.run(tf.initialize_variables([var]))
tf_resize(self.session, var, new_dimensions=(4, ))
tf.train.GradientDescentOptimizer(0.1).minimize(var)
tf_resize(self.session, var, new_dimensions=(2, ))
self.assertEqual(self.session.run(var).shape, (2, ))
# this was causing an exception
tf.train.GradientDescentOptimizer(0.1).minimize(var)
def test_resize_convolution_intput_layer(self):
# resize input layer
input_var = tf.Variable(
np.random.normal(0., 1., (1, 4, 4, 1)).astype(np.float32))
weights = tf.Variable(np.ones((2, 2, 1, 32), dtype=np.float32))
output = tf.nn.relu(
tf.nn.conv2d(input_var,
weights,
strides=[1, 1, 1, 1],
padding="SAME"))
self.session.run(tf.initialize_variables([input_var, weights]))
result1 = self.session.run(output)
tf_resize(self.session, input_var, new_values=np.ones([1, 5, 5, 1]))
result3 = self.session.run(output)
print(result3)
assert result3.shape == (1, 5, 5, 32)
def test_resize_convolution_convolution_dimension(self):
input_var = tf.Variable(
np.random.normal(0., 1., (1, 4, 4, 1)).astype(np.float32))
weights = tf.Variable(np.ones((2, 2, 1, 32), dtype=np.float32))
output = tf.nn.relu(
tf.nn.conv2d(input_var,
weights,
strides=[1, 1, 1, 1],
padding="SAME"))
self.session.run(tf.initialize_variables([input_var, weights]))
result1 = self.session.run(output)
tf_resize(self.session, weights, new_values=np.ones([2, 2, 1, 16]))
result2 = self.session.run(output)
assert result2.shape == (1, 4, 4, 16)
def test_tf_resize(self):
zeros = tf.zeros((4, ))
var = tf.Variable(initial_value=zeros)
loss = tf.square(1 - tf.reduce_sum(var))
self.session.run(tf.initialize_variables([var]))
optimizer_1 = tf.train.RMSPropOptimizer(0.01)
train_1 = optimizer_1.minimize(loss)
self.session.run(
tf.initialize_variables(
list(get_tf_optimizer_variables(optimizer_1))))
self.session.run(train_1)
tf_resize(self.session, var, new_dimensions=(6, ))
optimizer_2 = tf.train.RMSPropOptimizer(0.01)
train_2 = optimizer_2.minimize(loss)
self.session.run(
tf.initialize_variables(
list(get_tf_optimizer_variables(optimizer_2))))
self.session.run(train_2)
def resize(self, new_output_nodes=None):
if new_output_nodes is None or new_output_nodes > self.output_nodes:
# promote the candidate
tf_resize(self._session, self._weights,
new_values=np.append(self.session.run(self._weights), self.session.run(self._candidate_weights),
axis=1).astype(np.float32))
tf_resize(self._session, self._back_weights,
new_values=np.append(self.session.run(self._back_weights),
self.session.run(self._candidate_back_weights), axis=0).astype(np.float32), )
tf_resize(self._session, self._bias,
new_values=np.append(self.session.run(self._bias), self.session.run(self._candidate_bias)).astype(
np.float32))
tf_resize(self._session, self._back_bias,
new_values=np.append(self.session.run(self._back_bias),
self.session.run(self._candidate_back_bias)).astype(np.float32))
super(BackWeightCandidateLayer, self).resize(new_output_nodes=new_output_nodes)
def resize(self, new_output_nodes=None,
output_nodes_to_prune=None,
input_nodes_to_prune=None,
split_output_nodes=None,
split_input_nodes=None,
data_set_train=None,
data_set_validation=None,
no_splitting_or_pruning=False,
split_nodes_noise_std=.1):
"""Resize this layer by changing the number of output nodes. Will also resize any downstream layers
Args:
data_set_validation (DataSet):Data set used for validating this network
data_set_train (DataSet): Data set used for training this network
no_splitting_or_pruning (bool): If set to true then noise is just added randomly rather than splitting nodes
new_output_nodes (int | tuple of ints): If passed we change the number of output nodes of this layer to be new_output_nodes
output_nodes_to_prune ([int]): list of indexes of the output nodes we want pruned e.g. [1, 3] would remove
the 1st and 3rd output node from this layer
input_nodes_to_prune ([int]): list of indexes of the input nodes we want pruned e.g. [1, 3] would remove the
1st and 3rd input node from this layer
split_output_nodes ([int]): list of indexes of nodes to split. This is for growing the layer
split_input_nodes: ([int]): list of indexes of nodes that where split in the previous layer.
split_nodes_noise_std (float): standard deviation of noise to add when splitting a node
"""
if isinstance(new_output_nodes, tuple):
new_output_nodes = new_output_nodes[self.get_resizable_dimension()]
elif new_output_nodes is not None and not isinstance(new_output_nodes, int):
raise ValueError("new_output_nodes must be tuple of int %s" % (new_output_nodes,))
if not no_splitting_or_pruning:
# choose nodes to split or prune
if new_output_nodes is not None:
if output_nodes_to_prune is None and split_output_nodes is None:
if new_output_nodes < self.get_resizable_dimension_size():
output_nodes_to_prune = self._choose_nodes_to_prune(new_output_nodes, data_set_train,
data_set_validation)
elif new_output_nodes > self.get_resizable_dimension_size():
split_output_nodes = self._choose_nodes_to_split(new_output_nodes, data_set_train,
data_set_validation)
elif self.has_resizable_dimension():
new_output_nodes = self.get_resizable_dimension_size()
if output_nodes_to_prune:
new_output_nodes -= len(output_nodes_to_prune)
if split_output_nodes:
new_output_nodes += len(split_output_nodes)
new_input_nodes = self.input_layer.output_nodes
input_nodes_changed = new_input_nodes != self._input_nodes
if self.has_resizable_dimension() and new_output_nodes is not None:
output_nodes_changed = new_output_nodes != self.get_resizable_dimension_size()
temp_output_nodes = list(self._output_nodes)
temp_output_nodes[self.get_resizable_dimension()] = new_output_nodes
self._output_nodes = tuple(temp_output_nodes)
else:
output_nodes_changed = False
self._input_nodes = new_input_nodes
for name, bound_variable in self._bound_variables.iteritems():
if input_nodes_changed and self._bound_dimensions_contains_input(bound_variable.dimensions) or \
output_nodes_changed and self._bound_dimensions_contains_output(bound_variable.dimensions):
self._forget_assign_op(name)
int_dims = self._bound_dimensions_to_ints(bound_variable.dimensions)
if isinstance(bound_variable.variable, tf.Variable):
old_values = self._session.run(bound_variable.variable)
if output_nodes_to_prune or split_output_nodes:
output_bound_axis = self._get_axis(bound_variable.dimensions,
self.OUTPUT_BOUND_VALUE, self.OUTPUT_DIM_3_BOUND_VALUE)
if output_nodes_to_prune:
old_values = np.delete(old_values, output_nodes_to_prune, output_bound_axis)
else: # split
old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
noise_std=split_nodes_noise_std)
if input_nodes_to_prune or split_input_nodes:
input_bound_axis = self._get_axis(bound_variable.dimensions,
self.INPUT_BOUND_VALUE, self.INPUT_DIM_3_BOUND_VALUE)
if input_nodes_to_prune:
old_values = np.delete(old_values, input_nodes_to_prune, input_bound_axis)
else: # split
old_values = array_extend(old_values, {input_bound_axis: split_input_nodes},
halve_extended_vectors=True)
if no_splitting_or_pruning:
new_values = self._weight_extender_func(old_values, int_dims)
else:
new_values = old_values
tf_resize(self._session, bound_variable.variable, int_dims,
new_values, self._get_assign_function(name))
else:
# this is a tensor, not a variable so has no weights
tf_resize(self._session, bound_variable.variable, int_dims)
if input_nodes_changed and self._batch_normalize_input:
if self._batch_norm_mean_train is not None:
tf_resize(self._session, self._batch_norm_mean_train, self._input_nodes)
tf_resize(self._session, self._batch_norm_var_train, self._input_nodes)
if self._batch_norm_mean_predict is not None:
tf_resize(self._session, self._batch_norm_mean_predict, self._input_nodes)
tf_resize(self._session, self._batch_norm_var_predict, self._input_nodes)
if self._normalized_train is not None:
tf_resize(self._session, self._normalized_train, (None,) + self._input_nodes)
if self._normalized_predict is not None:
tf_resize(self._session, self._normalized_predict, (None,) + self._input_nodes)
tf_resize(self.session, self._normalized_predict.op.inputs[0].op.inputs[1].op.inputs[1], self._input_nodes)
# THIS needs fixing -> self._normalized_predict.op.inputs[0].op.inputs[1].op.inputs[1]
# self._normalized_predict.op.inputs[0].op.inputs[1].op.inputs[1].op.inputs[1] returns [2] and should be [4]
# This line fixed the issue, this is all very hacky...
# self._mat_mul.op.inputs[0]._shape = TensorShape((None,) + self._input_nodes)
from tensorflow.python.framework.tensor_shape import TensorShape
if '_mat_mul_is_train_equal_' + str(True) in self.__dict__:
self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0]._shape = TensorShape(
(None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0].op.inputs[0]._shape = TensorShape(
(None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0].op.inputs[0].op.inputs[
0]._shape = TensorShape((None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(True)].op.inputs[0].op.inputs[0].op.inputs[0].op.inputs[
0]._shape = TensorShape((None,) + self._input_nodes)
# tf_resize(self._session, self.__dict__['_mat_mul_is_train_equal_' + str(True)], (None,) + self._input_nodes)
if '_mat_mul_is_train_equal_' + str(False) in self.__dict__:
self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0]._shape = TensorShape(
(None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0].op.inputs[0]._shape = TensorShape(
(None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0].op.inputs[0].op.inputs[
0]._shape = TensorShape((None,) + self._input_nodes)
self.__dict__['_mat_mul_is_train_equal_' + str(False)].op.inputs[0].op.inputs[0].op.inputs[0].op.inputs[
0]._shape = TensorShape((None,) + self._input_nodes)
# tf_resize(self._session, self.__dict__['_mat_mul_is_train_equal_' + str(False)], (None,) + self._input_nodes)
if output_nodes_changed:
if has_lazyprop(self, 'activation_predict'):
tf_resize(self._session, self.activation_predict, (None,) + self._output_nodes)
if has_lazyprop(self, 'activation_train'):
tf_resize(self._session, self.activation_train, (None,) + self._output_nodes)
if input_nodes_changed and self.bactivate:
if has_lazyprop(self, 'bactivation_train'):
tf_resize(self._session, self.bactivation_train, (None,) + self._input_nodes)
if has_lazyprop(self, 'bactivation_predict'):
tf_resize(self._session, self.bactivation_predict, (None,) + self._input_nodes)
if self._next_layer and self._next_layer._resize_needed():
self._next_layer.resize(input_nodes_to_prune=output_nodes_to_prune, split_input_nodes=split_output_nodes,
no_splitting_or_pruning=no_splitting_or_pruning)
def resize(self, new_output_nodes=None, output_nodes_to_prune=None, input_nodes_to_prune=None,
split_output_nodes=None,
split_input_nodes=None,
split_nodes_noise_std=.01):
"""Resize this layer by changing the number of output nodes. Will also resize any downstream layers
Args:
new_output_nodes (int): If passed we change the number of output nodes of this layer to be new_output_nodes
Otherwise we change the size to current output nodes+1
output_nodes_to_prune ([int]): list of indexes of the output nodes we want pruned e.g. [1, 3] would remove
the 1st and 3rd output node from this layer
input_nodes_to_prune ([int]): list of indexes of the input nodes we want pruned e.g. [1, 3] would remove the
1st and 3rd input node from this layer
split_output_nodes ([int]): list of indexes of nodes to split. This is for growing the layer
split_input_nodes: ([int]): list of indexes of nodes that where split in the prevous layer.
split_nodes_noise_std (float): standard deviation of noise to add when splitting a node
"""
if output_nodes_to_prune:
if split_output_nodes:
raise NotImplementedError("At the moment must either split or prune")
if not (new_output_nodes is None or new_output_nodes != self._output_nodes - len(output_nodes_to_prune)):
raise Exception("Different number of output nodes set from that left after pruning")
else:
new_output_nodes = self._output_nodes - len(output_nodes_to_prune)
elif split_output_nodes:
if not (new_output_nodes is None or new_output_nodes != self._output_nodes + len(split_output_nodes)):
raise Exception("Different number of output nodes set from that left after splitting")
else:
new_output_nodes = self._output_nodes + len(split_output_nodes)
else:
new_output_nodes = new_output_nodes or self._output_nodes
new_input_nodes = self.input_layer.output_nodes
input_nodes_changed = new_input_nodes != self._input_nodes
output_nodes_changed = new_output_nodes != self._output_nodes
self._output_nodes = new_output_nodes
self._input_nodes = new_input_nodes
for bound_variable in self._bound_variables:
if input_nodes_changed and self._bound_dimensions_contains_input(bound_variable.dimensions) or \
output_nodes_changed and self._bound_dimensions_contains_output(bound_variable.dimensions):
int_dims = self._bound_dimensions_to_ints(bound_variable.dimensions)
if isinstance(bound_variable.variable, tf.Variable):
old_values = self._session.run(bound_variable.variable)
if output_nodes_to_prune or split_output_nodes:
try:
output_bound_axis = bound_variable.dimensions.index(self.OUTPUT_BOUND_VALUE)
if output_nodes_to_prune:
old_values = np.delete(old_values, output_nodes_to_prune, output_bound_axis)
else: # split
old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
noise_std=split_nodes_noise_std)
except ValueError, e:
pass
if input_nodes_to_prune or split_input_nodes:
try:
input_bound_axis = bound_variable.dimensions.index(self.INPUT_BOUND_VALUE)
if input_nodes_to_prune:
old_values = np.delete(old_values, input_nodes_to_prune, input_bound_axis)
else: # split
old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
noise_std=split_nodes_noise_std)
except ValueError, e:
pass
new_values = self._weight_extender_func(old_values, int_dims)
tf_resize(self._session, bound_variable.variable, int_dims,
new_values)
else:
# this is a tensor so no need to provide values
tf_resize(self._session, bound_variable.variable, int_dims)
else: # split
old_values = array_extend(old_values, {output_bound_axis: split_output_nodes},
noise_std=split_nodes_noise_std)
except ValueError, e:
pass
new_values = self._weight_extender_func(old_values, int_dims)
tf_resize(self._session, bound_variable.variable, int_dims,
new_values)
else:
# this is a tensor so no need to provide values
tf_resize(self._session, bound_variable.variable, int_dims)
if output_nodes_changed:
tf_resize(self._session, self.activation_train, (None, self._output_nodes))
tf_resize(self._session, self.activation_predict, (None, self._output_nodes))
if input_nodes_changed and self.bactivate:
tf_resize(self._session, self.bactivation_train, (None, self._input_nodes))
tf_resize(self._session, self.bactivation_predict, (None, self._input_nodes))
if self._next_layer and self._next_layer.resize_needed():
self._next_layer.resize(input_nodes_to_prune=output_nodes_to_prune, split_input_nodes=split_input_nodes)
def _bound_dimensions_to_ints(self, bound_dims):
int_dims = ()
for x in bound_dims:
if isinstance(x, int):
if x == -1:
int_dims += (None,)
