def testMatMulOp(self, l1_fraction):
op = tf.get_default_graph().get_operation_by_name('fc/MatMul')
with self.cached_session():
weights = op.inputs[1].eval()
l1_reg_vector = np.mean(np.abs(weights), axis=0)
l2_reg_vector = np.sqrt(np.mean(weights**2, axis=0))
expected_reg_vector = (l1_fraction * l1_reg_vector +
(1.0 - l1_fraction) * l2_reg_vector)
# We choose the threshold at the expectation value, so that some activations
# end up above threshold and others end up below. The weights are normally
# distributed, so the L2 norm is 1.0, and the L1 norm is sqrt(2/pi).
# With a general l1_fraction, we compute a weighted average of the two:
threshold = (1.0 - l1_fraction) + l1_fraction * np.sqrt(2 / np.pi)
expected_alive = expected_reg_vector > threshold
assert_not_all_are_alive_or_dead(expected_alive)
matmul_reg = (group_lasso_regularizer.GroupLassoRegularizer(
weight_tensor=op.inputs[1],
reduce_dims=(0, ),
threshold=threshold,
l1_fraction=l1_fraction))
with self.cached_session():
actual_reg_vector = matmul_reg.regularization_vector.eval()
actual_alive = matmul_reg.alive_vector.eval()
self.assertAllClose(actual_reg_vector, expected_reg_vector)
self.assertAllEqual(actual_alive, expected_alive)
def create_regularizer(self, op_slice):
"""Create a regularizer for this conv2d OpSlice.
Args:
op_slice: op_regularizer_manager.OpSlice that is a conv2d OpSlice.
Returns:
OpRegularizer for this conv2d op.
"""
start_index = op_slice.slice.start_index
size = op_slice.slice.size
weights = op_slice.op.inputs[1] # Input 1 are the weights.
weights = tpu_util.maybe_convert_to_variable(weights)
reduce_dims = self._reduce_dims(op_slice.op)
rank = len(weights.shape.as_list())
if rank != len(reduce_dims) + 1:
raise ValueError('Rank %d incompatible with reduce_dims %s for op %s' %
(rank, reduce_dims, op_slice.op.name))
def _slice_weights():
"""Slices the weight tensor according to op_slice information."""
if rank == 2:
if reduce_dims[0] == 0:
return weights[:, start_index:start_index + size]
else:
return weights[start_index:start_index + size, :]
if rank == 3:
if 2 not in reduce_dims:
return weights[:, :, start_index:start_index + size]
if 1 not in reduce_dims:
return weights[:, start_index:start_index + size, :]
if 0 not in reduce_dims:
return weights[start_index:start_index + size, :, :]
if rank == 4:
if 3 not in reduce_dims:
return weights[:, :, :, start_index:start_index + size]
if 2 not in reduce_dims:
return weights[:, :, start_index:start_index + size, :]
if 1 not in reduce_dims:
return weights[:, start_index:start_index + size, :, :]
if 0 not in reduce_dims:
return weights[start_index:start_index + size, :, :, :]
if rank == 5:
if 4 not in reduce_dims:
return weights[:, :, :, :, start_index:start_index + size]
raise ValueError('Unsupported reduce_dim for rank 5 tensors (Conv3D)')
raise ValueError('Unsupported rank or bad reduce_dim')
weight_tensor = _slice_weights()
# If OpSlice size matches tensor size, use the entire tensor. Otherwise,
# slice the tensor accordingly.
return group_lasso_regularizer.GroupLassoRegularizer(
weight_tensor=weight_tensor,
reduce_dims=self._reduce_dims(op_slice.op),
threshold=self._threshold,
l1_fraction=self._l1_fraction)
def create_regularizer(self, op_slice):
"""Create a regularizer for this conv2d OpSlice.
Args:
op_slice: op_regularizer_manager.OpSlice that is a conv2d OpSlice.
Returns:
OpRegularizer for this conv2d op.
"""
start_index = op_slice.slice.start_index
size = op_slice.slice.size
weights = op_slice.op.inputs[1] # Input 1 are the weights.
reduce_dims = self._reduce_dims(op_slice.op)
rank = len(weights.shape.as_list())
assert rank == len(reduce_dims) + 1
def _slice_weights():
"""Slices the weight tensor according to op_slice information."""
if rank == 2:
if reduce_dims[0] == 0:
return weights[:, start_index:start_index + size]
else:
return weights[start_index:start_index + size, :]
if rank == 3:
if 2 not in reduce_dims:
return weights[:, :, start_index:start_index + size]
if 1 not in reduce_dims:
return weights[:, start_index:start_index + size, :]
if 0 not in reduce_dims:
return weights[start_index:start_index + size, :, :]
if rank == 4:
if 3 not in reduce_dims:
return weights[:, :, :, start_index:start_index + size]
if 2 not in reduce_dims:
return weights[:, :, start_index:start_index + size, :]
if 1 not in reduce_dims:
return weights[:, start_index:start_index + size, :, :]
if 0 not in reduce_dims:
return weights[start_index:start_index + size, :, :, :]
raise ValueError('Unsupported rankd or bad reduce_dim')
weight_tensor = _slice_weights()
# If OpSlice size matches tensor size, use the entire tensor. Otherwise,
# slice the tensor accordingly.
return group_lasso_regularizer.GroupLassoRegularizer(
weight_tensor=weight_tensor,
reduce_dims=self._reduce_dims(op_slice.op),
threshold=self._threshold,
l1_fraction=self._l1_fraction)
