def validate_size_tuple(n, s):
if is_integer(s):
# Do not change a single integer into a tuple!
# This is because we do not know the dimensionality of the
# convolution operation here, so we cannot build the size
# tuple with correct number of elements from the integer notation.
return int(s)
return validate_int_tuple_arg(n, s)
def validate_conv2d_size_tuple(arg_name, arg_value):
"""
Validate the `arg_value`, ensure it is one or two positive integers,
such that it can be used as the kernel size.
Args:
arg_name: Name of the argument.
arg_value: An integer, or a tuple of two integers.
Returns:
(int, int): The validated two integers.
"""
arg_value = validate_int_tuple_arg(arg_name, arg_value)
if len(arg_value) not in (1, 2) or any(a < 1 for a in arg_value):
raise ValueError('Invalid value for argument `{}`: expected to be '
'one or two positive integers, but got {!r}.'.format(
arg_name, arg_value))
if len(arg_value) == 1:
arg_value = arg_value * 2
return arg_value
def log_mean_exp(x, axis=None, keepdims=False, name=None):
"""
Compute :math:`\\log \\frac{1}{K} \\sum_{k=1}^K \\exp(x_k)`.
.. math::
\\begin{align*}
\\log \\frac{1}{K} \\sum_{k=1}^K \\exp(x_k)
&= \\log \\left[\\exp(x_{max}) \\frac{1}{K}
\\sum_{k=1}^K \\exp(x_k - x_{max})\\right] \\\\
&= x_{max} + \\log \\frac{1}{K}
\\sum_{k=1}^K \\exp(x_k - x_{max}) \\\\
x_{max} &= \\max x_k
\\end{align*}
Args:
x (Tensor): The input `x`.
axis (int or tuple[int]): The dimension to take average.
Default :obj:`None`, all dimensions.
keepdims (bool): Whether or not to keep the summed dimensions?
(default :obj:`False`)
Returns:
tf.Tensor: The computed value.
"""
axis = validate_int_tuple_arg('axis', axis, nullable=True)
x = tf.convert_to_tensor(x)
with tf.name_scope(name, default_name='log_mean_exp', values=[x]):
x = tf.convert_to_tensor(x)
x_max_keepdims = tf.reduce_max(x, axis=axis, keepdims=True)
if not keepdims:
x_max = tf.squeeze(x_max_keepdims, axis=axis)
else:
x_max = x_max_keepdims
mean_exp = tf.reduce_mean(tf.exp(x - x_max_keepdims),
axis=axis,
keepdims=keepdims)
return x_max + tf.log(mean_exp)
def __init__(self,
axis=-1,
value_ndims=1,
initialized=False,
scale_type='exp',
bias_regularizer=None,
bias_constraint=None,
log_scale_regularizer=None,
log_scale_constraint=None,
scale_regularizer=None,
scale_constraint=None,
trainable=True,
epsilon=1e-6,
name=None,
scope=None):
"""
Construct a new :class:`ActNorm` instance.
Args:
axis (int or Iterable[int]): The axis to apply ActNorm.
Dimensions not in `axis` will be averaged out when computing
the mean of activations. Default `-1`, the last dimension.
All items of the `axis` should be covered by `value_ndims`.
value_ndims (int): Number of value dimensions in both `x` and `y`.
`x.ndims - value_ndims == log_det.ndims` and
`y.ndims - value_ndims == log_det.ndims`.
initialized (bool): Whether or not the variables have been
initialized? If :obj:`False`, the first input `x` in the
forward pass will be used to initialize the variables.
Normally, it should take the default value, :obj:`False`.
Setting it to :obj:`True` only if you're constructing a
:class:`ActNorm` instance inside some reused variable scope.
scale_type: One of {"exp", "linear"}.
If "exp", ``y = (x + bias) * tf.exp(log_scale)``.
If "linear", ``y = (x + bias) * scale``.
Default is "exp".
bias_regularizer: The regularizer for `bias`.
bias_constraint: The constraint for `bias`.
log_scale_regularizer: The regularizer for `log_scale`.
log_scale_constraint: The constraint for `log_scale`.
scale_regularizer: The regularizer for `scale`.
scale_constraint: The constraint for `scale`.
trainable (bool): Whether or not the variables are trainable?
epsilon: Small float to avoid dividing by zero or taking
logarithm of zero.
"""
axis = validate_int_tuple_arg('axis', axis)
self._scale_type = validate_enum_arg('scale_type', scale_type,
['exp', 'linear'])
self._initialized = bool(initialized)
self._bias_regularizer = bias_regularizer
self._bias_constraint = bias_constraint
self._log_scale_regularizer = log_scale_regularizer
self._log_scale_constraint = log_scale_constraint
self._scale_regularizer = scale_regularizer
self._scale_constraint = scale_constraint
self._trainable = bool(trainable)
self._epsilon = epsilon
super(ActNorm, self).__init__(axis=axis,
value_ndims=value_ndims,
name=name,
scope=scope)
def weight_norm(kernel,
axis,
use_scale=True,
scale=None,
scale_initializer=None,
scale_regularizer=None,
scale_constraint=None,
trainable=True,
epsilon=1e-12,
name=None,
scope=None):
"""
Weight normalization proposed by (Salimans & Kingma, 2016).
Roughly speaking, the weight normalization is defined as::
kernel = scale * kernel / tf.sqrt(
tf.reduce_sum(kernel ** 2, axis=<dimensions not in `axis`>,
keepdims=True)
)
This function does not support data-dependent initialization for `scale`.
If you do need this feature, you have to turn off `scale`, and use
:func:`~tfsnippet.layers.act_norm` along with :func:`weight_norm`.
Args:
kernel: Tensor, the weight `w` to be normalized.
axis (int or tuple[int]): The axis to apply weight normalization.
See above description to know what `axis` exactly is.
use_scale (bool): Whether or not to use `scale`. Default :obj:`True`.
scale (Tensor): Instead of creating a new variable, use this tensor.
scale_initializer: The initializer for `scale`.
scale_regularizer: The regularizer for `scale`.
scale_constraint: The constraint for `scale`.
trainable (bool): Whether or not the variables are trainable?
epsilon: Small float number to avoid dividing by zero.
"""
# check the parameters
if not use_scale and scale is not None:
raise ValueError('`use_scale` is False but `scale` is specified.')
axis = validate_int_tuple_arg('axis', axis)
if not axis:
raise ValueError('`axis` cannot be empty.')
kernel = tf.convert_to_tensor(kernel)
kernel_shape = get_static_shape(kernel)
dtype = kernel.dtype.base_dtype
var_spec = ParamSpec(kernel_shape, dtype=dtype)
if scale_initializer is None:
scale_initializer = tf.ones_initializer(dtype=dtype)
if scale is not None:
scale = var_spec.validate('scale', scale)
# any dimension not specified in `axis` should be averaged out
axis = resolve_negative_axis(len(kernel_shape), axis)
reduce_axis = tuple(a for a in range(len(kernel_shape)) if a not in axis)
with tf.variable_scope(scope, default_name=name or 'weight_norm'):
# normalize the kernel
kernel = maybe_check_numerics(
tf.nn.l2_normalize(kernel, axis=reduce_axis, epsilon=epsilon),
'weight-normalized kernel')
# create the scaling variable
if use_scale:
if scale is None:
scale = model_variable('scale',
shape=kernel_shape,
dtype=dtype,
initializer=scale_initializer,
regularizer=scale_regularizer,
constraint=scale_constraint,
trainable=trainable)
scale = maybe_check_numerics(scale, 'scale')
kernel = kernel * scale
# now return the normalized weight
return kernel
def act_norm(input,
axis=-1,
initializing=False,
scale_type='exp',
bias_regularizer=None,
bias_constraint=None,
log_scale_regularizer=None,
log_scale_constraint=None,
scale_regularizer=None,
scale_constraint=None,
trainable=True,
epsilon=1e-6,
name=None,
scope=None,
value_ndims=None):
"""
ActNorm proposed by (Kingma & Dhariwal, 2018).
Examples::
import tfsnippet as spt
# apply act_norm on a dense layer
x = spt.layers.dense(x, units, activation_fn=tf.nn.relu,
normalizer_fn=functools.partial(
act_norm, initializing=initializing))
# apply act_norm on a conv2d layer
x = spt.layers.conv2d(x, out_channels, (3, 3),
channels_last=channels_last,
activation_fn=tf.nn.relu,
normalizer_fn=functools.partial(
act_norm,
axis=-1 if channels_last else -3,
value_ndims=3,
initializing=initializing,
))
Args:
input (Tensor): The input tensor.
axis (int or Iterable[int]): The axis to apply ActNorm.
Dimensions not in `axis` will be averaged out when computing
the mean of activations. Default `-1`, the last dimension.
All items of the `axis` should be covered by `value_ndims`.
initializing (bool): Whether or not to use the input `x` to initialize
the layer parameters? (default :obj:`True`)
scale_type: One of {"exp", "linear"}.
If "exp", ``y = (x + bias) * tf.exp(log_scale)``.
If "linear", ``y = (x + bias) * scale``.
Default is "exp".
bias_regularizer: The regularizer for `bias`.
bias_constraint: The constraint for `bias`.
log_scale_regularizer: The regularizer for `log_scale`.
log_scale_constraint: The constraint for `log_scale`.
scale_regularizer: The regularizer for `scale`.
scale_constraint: The constraint for `scale`.
trainable (bool): Whether or not the variables are trainable?
epsilon: Small float to avoid dividing by zero or taking
logarithm of zero.
Returns:
tf.Tensor: The output after the ActNorm has been applied.
"""
input = InputSpec(shape=['...']).validate('input', input)
rank = len(get_static_shape(input))
axis = list(validate_int_tuple_arg('axis', axis))
for i, a in enumerate(axis):
if a >= 0:
axis[i] = a - rank
value_ndims = max(-a for a in axis)
layer = ActNorm(
axis=axis,
value_ndims=value_ndims,
initialized=not initializing,
scale_type=scale_type,
bias_regularizer=bias_regularizer,
bias_constraint=bias_constraint,
log_scale_regularizer=log_scale_regularizer,
log_scale_constraint=log_scale_constraint,
scale_regularizer=scale_regularizer,
scale_constraint=scale_constraint,
trainable=trainable,
epsilon=epsilon,
name=name,
scope=scope
)
return layer.apply(input)