def testMatMulOp(self, l1_fraction): op = tf.get_default_graph().get_operation_by_name('fc/MatMul') with self.cached_session(): weights = op.inputs[1].eval() l1_reg_vector = np.mean(np.abs(weights), axis=0) l2_reg_vector = np.sqrt(np.mean(weights**2, axis=0)) expected_reg_vector = (l1_fraction * l1_reg_vector + (1.0 - l1_fraction) * l2_reg_vector) # We choose the threshold at the expectation value, so that some activations # end up above threshold and others end up below. The weights are normally # distributed, so the L2 norm is 1.0, and the L1 norm is sqrt(2/pi). # With a general l1_fraction, we compute a weighted average of the two: threshold = (1.0 - l1_fraction) + l1_fraction * np.sqrt(2 / np.pi) expected_alive = expected_reg_vector > threshold assert_not_all_are_alive_or_dead(expected_alive) matmul_reg = (group_lasso_regularizer.GroupLassoRegularizer( weight_tensor=op.inputs[1], reduce_dims=(0, ), threshold=threshold, l1_fraction=l1_fraction)) with self.cached_session(): actual_reg_vector = matmul_reg.regularization_vector.eval() actual_alive = matmul_reg.alive_vector.eval() self.assertAllClose(actual_reg_vector, expected_reg_vector) self.assertAllEqual(actual_alive, expected_alive)
def create_regularizer(self, op_slice): """Create a regularizer for this conv2d OpSlice. Args: op_slice: op_regularizer_manager.OpSlice that is a conv2d OpSlice. Returns: OpRegularizer for this conv2d op. """ start_index = op_slice.slice.start_index size = op_slice.slice.size weights = op_slice.op.inputs[1] # Input 1 are the weights. weights = tpu_util.maybe_convert_to_variable(weights) reduce_dims = self._reduce_dims(op_slice.op) rank = len(weights.shape.as_list()) if rank != len(reduce_dims) + 1: raise ValueError('Rank %d incompatible with reduce_dims %s for op %s' % (rank, reduce_dims, op_slice.op.name)) def _slice_weights(): """Slices the weight tensor according to op_slice information.""" if rank == 2: if reduce_dims[0] == 0: return weights[:, start_index:start_index + size] else: return weights[start_index:start_index + size, :] if rank == 3: if 2 not in reduce_dims: return weights[:, :, start_index:start_index + size] if 1 not in reduce_dims: return weights[:, start_index:start_index + size, :] if 0 not in reduce_dims: return weights[start_index:start_index + size, :, :] if rank == 4: if 3 not in reduce_dims: return weights[:, :, :, start_index:start_index + size] if 2 not in reduce_dims: return weights[:, :, start_index:start_index + size, :] if 1 not in reduce_dims: return weights[:, start_index:start_index + size, :, :] if 0 not in reduce_dims: return weights[start_index:start_index + size, :, :, :] if rank == 5: if 4 not in reduce_dims: return weights[:, :, :, :, start_index:start_index + size] raise ValueError('Unsupported reduce_dim for rank 5 tensors (Conv3D)') raise ValueError('Unsupported rank or bad reduce_dim') weight_tensor = _slice_weights() # If OpSlice size matches tensor size, use the entire tensor. Otherwise, # slice the tensor accordingly. return group_lasso_regularizer.GroupLassoRegularizer( weight_tensor=weight_tensor, reduce_dims=self._reduce_dims(op_slice.op), threshold=self._threshold, l1_fraction=self._l1_fraction)
def create_regularizer(self, op_slice): """Create a regularizer for this conv2d OpSlice. Args: op_slice: op_regularizer_manager.OpSlice that is a conv2d OpSlice. Returns: OpRegularizer for this conv2d op. """ start_index = op_slice.slice.start_index size = op_slice.slice.size weights = op_slice.op.inputs[1] # Input 1 are the weights. reduce_dims = self._reduce_dims(op_slice.op) rank = len(weights.shape.as_list()) assert rank == len(reduce_dims) + 1 def _slice_weights(): """Slices the weight tensor according to op_slice information.""" if rank == 2: if reduce_dims[0] == 0: return weights[:, start_index:start_index + size] else: return weights[start_index:start_index + size, :] if rank == 3: if 2 not in reduce_dims: return weights[:, :, start_index:start_index + size] if 1 not in reduce_dims: return weights[:, start_index:start_index + size, :] if 0 not in reduce_dims: return weights[start_index:start_index + size, :, :] if rank == 4: if 3 not in reduce_dims: return weights[:, :, :, start_index:start_index + size] if 2 not in reduce_dims: return weights[:, :, start_index:start_index + size, :] if 1 not in reduce_dims: return weights[:, start_index:start_index + size, :, :] if 0 not in reduce_dims: return weights[start_index:start_index + size, :, :, :] raise ValueError('Unsupported rankd or bad reduce_dim') weight_tensor = _slice_weights() # If OpSlice size matches tensor size, use the entire tensor. Otherwise, # slice the tensor accordingly. return group_lasso_regularizer.GroupLassoRegularizer( weight_tensor=weight_tensor, reduce_dims=self._reduce_dims(op_slice.op), threshold=self._threshold, l1_fraction=self._l1_fraction)