def spectral_norm(weights, name=None):
"""Applies spectral_norm regularization to weights."""
with ops.name_scope(scope, 'spectral_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
if method == 'power':
if conv:
s = power_iterate_conv(weights=weights,
strides=strides,
padding=padding.upper(),
input_shape=input_shape,
output_shape=output_shape,
num_iter=num_iter,
weight_name=weight_name)
else:
s = power_iterate(weights=weights,
num_iter=num_iter,
weight_name=weight_name)
return standard_ops.multiply(my_scale, s, name=name)
else: # svd
return standard_ops.multiply(
my_scale,
standard_ops.svd(weights, compute_uv=False)[..., 0],
name=name)
def l2(weights):
"""Applies l2 regularization to weights."""
with ops.name_scope(scope, 'l2_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.multiply(my_scale, nn.l2_loss(weights), name=name)
def l0(weights):
"""Applies l2 regularization to weights."""
with ops.name_scope(scope, 'l0_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
l0_loss = l0_computation(weights, **kwargs)
return standard_ops.multiply(my_scale, l0_loss, name=name)
def lp(weights):
"""Applies l2 regularization to weights."""
with ops.name_scope(scope, 'lp_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
reg_loss = standard_ops.reduce_sum(math_ops.pow(math_ops.abs(weights), p))
return standard_ops.multiply(my_scale, reg_loss, name=name)
def sum_reg(weights, name=None):
"""Applies sum regularization to weights."""
with ops.name_scope(scope, 'sum_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.multiply(my_scale,
standard_ops.reduce_sum(weights),
name=name)
def l1(weights, name=None):
"""Applies L1 regularization to weights."""
with ops.name_scope(scope, 'l1_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.multiply(
my_scale,
standard_ops.reduce_sum(standard_ops.abs(weights)),
name=name)
def l2(weights):
"""Applies l2 regularization to weights."""
weights = tf.cast(weights, tf.float32)
from tensorflow.python.ops import nn
from tensorflow.python.ops import standard_ops
from tensorflow.python.framework import ops
with ops.name_scope(scope, 'l2_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.multiply(my_scale, nn.l2_loss(weights), name=name)
def var(weights, name=None):
"""Applies variance regularization to weights."""
with ops.name_scope(scope, 'var_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
_, var_axis0 = nn.moments(weights, axes=axes)
return standard_ops.multiply(
my_scale,
var_axis0,
name=name)
def orthogonal_sum(weights):
""" Applies orthogonal regularization to weights. """
with ops.name_scope(scope, 'orthogonal_regularizer', [weights]) as name:
tensor_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
norm_weights = tf.nn.l2_normalize(weights, axis=1)
anchor_weights_t = tf.transpose(norm_weights)
det_reg = tf.matmul(anchor_weights_t, norm_weights)
identity = tf.eye(tf.shape(det_reg)[0])
det_reg = tf.subtract(det_reg, identity)
det_reg = tf.reduce_sum(tf.abs(det_reg))
# Print sum value before scaling
det_reg = tf.Print(det_reg, [det_reg], "Orthogonal sum for \"{}\" :".format(name))
return standard_ops.multiply(tensor_scale, det_reg, name=name)
