def validate_size_tuple(n, s): if is_integer(s): # Do not change a single integer into a tuple! # This is because we do not know the dimensionality of the # convolution operation here, so we cannot build the size # tuple with correct number of elements from the integer notation. return int(s) return validate_int_tuple_arg(n, s)
def validate_conv2d_size_tuple(arg_name, arg_value): """ Validate the `arg_value`, ensure it is one or two positive integers, such that it can be used as the kernel size. Args: arg_name: Name of the argument. arg_value: An integer, or a tuple of two integers. Returns: (int, int): The validated two integers. """ arg_value = validate_int_tuple_arg(arg_name, arg_value) if len(arg_value) not in (1, 2) or any(a < 1 for a in arg_value): raise ValueError('Invalid value for argument `{}`: expected to be ' 'one or two positive integers, but got {!r}.'.format( arg_name, arg_value)) if len(arg_value) == 1: arg_value = arg_value * 2 return arg_value
def log_mean_exp(x, axis=None, keepdims=False, name=None): """ Compute :math:`\\log \\frac{1}{K} \\sum_{k=1}^K \\exp(x_k)`. .. math:: \\begin{align*} \\log \\frac{1}{K} \\sum_{k=1}^K \\exp(x_k) &= \\log \\left[\\exp(x_{max}) \\frac{1}{K} \\sum_{k=1}^K \\exp(x_k - x_{max})\\right] \\\\ &= x_{max} + \\log \\frac{1}{K} \\sum_{k=1}^K \\exp(x_k - x_{max}) \\\\ x_{max} &= \\max x_k \\end{align*} Args: x (Tensor): The input `x`. axis (int or tuple[int]): The dimension to take average. Default :obj:`None`, all dimensions. keepdims (bool): Whether or not to keep the summed dimensions? (default :obj:`False`) Returns: tf.Tensor: The computed value. """ axis = validate_int_tuple_arg('axis', axis, nullable=True) x = tf.convert_to_tensor(x) with tf.name_scope(name, default_name='log_mean_exp', values=[x]): x = tf.convert_to_tensor(x) x_max_keepdims = tf.reduce_max(x, axis=axis, keepdims=True) if not keepdims: x_max = tf.squeeze(x_max_keepdims, axis=axis) else: x_max = x_max_keepdims mean_exp = tf.reduce_mean(tf.exp(x - x_max_keepdims), axis=axis, keepdims=keepdims) return x_max + tf.log(mean_exp)
def __init__(self, axis=-1, value_ndims=1, initialized=False, scale_type='exp', bias_regularizer=None, bias_constraint=None, log_scale_regularizer=None, log_scale_constraint=None, scale_regularizer=None, scale_constraint=None, trainable=True, epsilon=1e-6, name=None, scope=None): """ Construct a new :class:`ActNorm` instance. Args: axis (int or Iterable[int]): The axis to apply ActNorm. Dimensions not in `axis` will be averaged out when computing the mean of activations. Default `-1`, the last dimension. All items of the `axis` should be covered by `value_ndims`. value_ndims (int): Number of value dimensions in both `x` and `y`. `x.ndims - value_ndims == log_det.ndims` and `y.ndims - value_ndims == log_det.ndims`. initialized (bool): Whether or not the variables have been initialized? If :obj:`False`, the first input `x` in the forward pass will be used to initialize the variables. Normally, it should take the default value, :obj:`False`. Setting it to :obj:`True` only if you're constructing a :class:`ActNorm` instance inside some reused variable scope. scale_type: One of {"exp", "linear"}. If "exp", ``y = (x + bias) * tf.exp(log_scale)``. If "linear", ``y = (x + bias) * scale``. Default is "exp". bias_regularizer: The regularizer for `bias`. bias_constraint: The constraint for `bias`. log_scale_regularizer: The regularizer for `log_scale`. log_scale_constraint: The constraint for `log_scale`. scale_regularizer: The regularizer for `scale`. scale_constraint: The constraint for `scale`. trainable (bool): Whether or not the variables are trainable? epsilon: Small float to avoid dividing by zero or taking logarithm of zero. """ axis = validate_int_tuple_arg('axis', axis) self._scale_type = validate_enum_arg('scale_type', scale_type, ['exp', 'linear']) self._initialized = bool(initialized) self._bias_regularizer = bias_regularizer self._bias_constraint = bias_constraint self._log_scale_regularizer = log_scale_regularizer self._log_scale_constraint = log_scale_constraint self._scale_regularizer = scale_regularizer self._scale_constraint = scale_constraint self._trainable = bool(trainable) self._epsilon = epsilon super(ActNorm, self).__init__(axis=axis, value_ndims=value_ndims, name=name, scope=scope)
def weight_norm(kernel, axis, use_scale=True, scale=None, scale_initializer=None, scale_regularizer=None, scale_constraint=None, trainable=True, epsilon=1e-12, name=None, scope=None): """ Weight normalization proposed by (Salimans & Kingma, 2016). Roughly speaking, the weight normalization is defined as:: kernel = scale * kernel / tf.sqrt( tf.reduce_sum(kernel ** 2, axis=<dimensions not in `axis`>, keepdims=True) ) This function does not support data-dependent initialization for `scale`. If you do need this feature, you have to turn off `scale`, and use :func:`~tfsnippet.layers.act_norm` along with :func:`weight_norm`. Args: kernel: Tensor, the weight `w` to be normalized. axis (int or tuple[int]): The axis to apply weight normalization. See above description to know what `axis` exactly is. use_scale (bool): Whether or not to use `scale`. Default :obj:`True`. scale (Tensor): Instead of creating a new variable, use this tensor. scale_initializer: The initializer for `scale`. scale_regularizer: The regularizer for `scale`. scale_constraint: The constraint for `scale`. trainable (bool): Whether or not the variables are trainable? epsilon: Small float number to avoid dividing by zero. """ # check the parameters if not use_scale and scale is not None: raise ValueError('`use_scale` is False but `scale` is specified.') axis = validate_int_tuple_arg('axis', axis) if not axis: raise ValueError('`axis` cannot be empty.') kernel = tf.convert_to_tensor(kernel) kernel_shape = get_static_shape(kernel) dtype = kernel.dtype.base_dtype var_spec = ParamSpec(kernel_shape, dtype=dtype) if scale_initializer is None: scale_initializer = tf.ones_initializer(dtype=dtype) if scale is not None: scale = var_spec.validate('scale', scale) # any dimension not specified in `axis` should be averaged out axis = resolve_negative_axis(len(kernel_shape), axis) reduce_axis = tuple(a for a in range(len(kernel_shape)) if a not in axis) with tf.variable_scope(scope, default_name=name or 'weight_norm'): # normalize the kernel kernel = maybe_check_numerics( tf.nn.l2_normalize(kernel, axis=reduce_axis, epsilon=epsilon), 'weight-normalized kernel') # create the scaling variable if use_scale: if scale is None: scale = model_variable('scale', shape=kernel_shape, dtype=dtype, initializer=scale_initializer, regularizer=scale_regularizer, constraint=scale_constraint, trainable=trainable) scale = maybe_check_numerics(scale, 'scale') kernel = kernel * scale # now return the normalized weight return kernel
def act_norm(input, axis=-1, initializing=False, scale_type='exp', bias_regularizer=None, bias_constraint=None, log_scale_regularizer=None, log_scale_constraint=None, scale_regularizer=None, scale_constraint=None, trainable=True, epsilon=1e-6, name=None, scope=None, value_ndims=None): """ ActNorm proposed by (Kingma & Dhariwal, 2018). Examples:: import tfsnippet as spt # apply act_norm on a dense layer x = spt.layers.dense(x, units, activation_fn=tf.nn.relu, normalizer_fn=functools.partial( act_norm, initializing=initializing)) # apply act_norm on a conv2d layer x = spt.layers.conv2d(x, out_channels, (3, 3), channels_last=channels_last, activation_fn=tf.nn.relu, normalizer_fn=functools.partial( act_norm, axis=-1 if channels_last else -3, value_ndims=3, initializing=initializing, )) Args: input (Tensor): The input tensor. axis (int or Iterable[int]): The axis to apply ActNorm. Dimensions not in `axis` will be averaged out when computing the mean of activations. Default `-1`, the last dimension. All items of the `axis` should be covered by `value_ndims`. initializing (bool): Whether or not to use the input `x` to initialize the layer parameters? (default :obj:`True`) scale_type: One of {"exp", "linear"}. If "exp", ``y = (x + bias) * tf.exp(log_scale)``. If "linear", ``y = (x + bias) * scale``. Default is "exp". bias_regularizer: The regularizer for `bias`. bias_constraint: The constraint for `bias`. log_scale_regularizer: The regularizer for `log_scale`. log_scale_constraint: The constraint for `log_scale`. scale_regularizer: The regularizer for `scale`. scale_constraint: The constraint for `scale`. trainable (bool): Whether or not the variables are trainable? epsilon: Small float to avoid dividing by zero or taking logarithm of zero. Returns: tf.Tensor: The output after the ActNorm has been applied. """ input = InputSpec(shape=['...']).validate('input', input) rank = len(get_static_shape(input)) axis = list(validate_int_tuple_arg('axis', axis)) for i, a in enumerate(axis): if a >= 0: axis[i] = a - rank value_ndims = max(-a for a in axis) layer = ActNorm( axis=axis, value_ndims=value_ndims, initialized=not initializing, scale_type=scale_type, bias_regularizer=bias_regularizer, bias_constraint=bias_constraint, log_scale_regularizer=log_scale_regularizer, log_scale_constraint=log_scale_constraint, scale_regularizer=scale_regularizer, scale_constraint=scale_constraint, trainable=trainable, epsilon=epsilon, name=name, scope=scope ) return layer.apply(input)