def spectral_norm(weights, name=None): """Applies spectral_norm regularization to weights.""" with ops.name_scope(scope, 'spectral_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') if method == 'power': if conv: s = power_iterate_conv(weights=weights, strides=strides, padding=padding.upper(), input_shape=input_shape, output_shape=output_shape, num_iter=num_iter, weight_name=weight_name) else: s = power_iterate(weights=weights, num_iter=num_iter, weight_name=weight_name) return standard_ops.multiply(my_scale, s, name=name) else: # svd return standard_ops.multiply( my_scale, standard_ops.svd(weights, compute_uv=False)[..., 0], name=name)
def l2(weights): """Applies l2 regularization to weights.""" with ops.name_scope(scope, 'l2_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.multiply(my_scale, nn.l2_loss(weights), name=name)
def l0(weights): """Applies l2 regularization to weights.""" with ops.name_scope(scope, 'l0_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') l0_loss = l0_computation(weights, **kwargs) return standard_ops.multiply(my_scale, l0_loss, name=name)
def lp(weights): """Applies l2 regularization to weights.""" with ops.name_scope(scope, 'lp_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') reg_loss = standard_ops.reduce_sum(math_ops.pow(math_ops.abs(weights), p)) return standard_ops.multiply(my_scale, reg_loss, name=name)
def sum_reg(weights, name=None): """Applies sum regularization to weights.""" with ops.name_scope(scope, 'sum_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.multiply(my_scale, standard_ops.reduce_sum(weights), name=name)
def l1(weights, name=None): """Applies L1 regularization to weights.""" with ops.name_scope(scope, 'l1_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.multiply( my_scale, standard_ops.reduce_sum(standard_ops.abs(weights)), name=name)
def l2(weights): """Applies l2 regularization to weights.""" weights = tf.cast(weights, tf.float32) from tensorflow.python.ops import nn from tensorflow.python.ops import standard_ops from tensorflow.python.framework import ops with ops.name_scope(scope, 'l2_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.multiply(my_scale, nn.l2_loss(weights), name=name)
def var(weights, name=None): """Applies variance regularization to weights.""" with ops.name_scope(scope, 'var_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') _, var_axis0 = nn.moments(weights, axes=axes) return standard_ops.multiply( my_scale, var_axis0, name=name)
def orthogonal_sum(weights): """ Applies orthogonal regularization to weights. """ with ops.name_scope(scope, 'orthogonal_regularizer', [weights]) as name: tensor_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') norm_weights = tf.nn.l2_normalize(weights, axis=1) anchor_weights_t = tf.transpose(norm_weights) det_reg = tf.matmul(anchor_weights_t, norm_weights) identity = tf.eye(tf.shape(det_reg)[0]) det_reg = tf.subtract(det_reg, identity) det_reg = tf.reduce_sum(tf.abs(det_reg)) # Print sum value before scaling det_reg = tf.Print(det_reg, [det_reg], "Orthogonal sum for \"{}\" :".format(name)) return standard_ops.multiply(tensor_scale, det_reg, name=name)