def normal_lccdf(mu, sigma, x):
z = (x - mu) / sigma
return aet.switch(
aet.gt(z, 1.0),
aet.log(aet.erfcx(z / aet.sqrt(2.0)) / 2.0) - aet.sqr(z) / 2.0,
aet.log1p(-aet.erfc(-z / aet.sqrt(2.0)) / 2.0),
)
def normal_lcdf(mu, sigma, x):
"""Compute the log of the cumulative density function of the normal."""
z = (x - mu) / sigma
return aet.switch(
aet.lt(z, -1.0),
aet.log(aet.erfcx(-z / aet.sqrt(2.0)) / 2.0) - aet.sqr(z) / 2.0,
aet.log1p(-aet.erfc(z / aet.sqrt(2.0)) / 2.0),
)
def test_neibs_full_with_inconsistent_borders(self):
shape = (2, 3, 5, 5)
images = tt.dtensor4()
images_val = np.arange(np.prod(shape), dtype="float32").reshape(shape)
f = aesara.function([images],
tt.sqr(images2neibs(images, (2, 2), mode="full")),
mode=self.mode)
with pytest.raises(TypeError):
f(images_val)
def norm_constraint(tensor_var, max_norm, norm_axes=None, epsilon=1e-7):
"""Max weight norm constraints and gradient clipping
This takes a TensorVariable and rescales it so that incoming weight
norms are below a specified constraint value. Vectors violating the
constraint are rescaled so that they are within the allowed range.
Parameters
----------
tensor_var: TensorVariable
Aesara expression for update, gradient, or other quantity.
max_norm: scalar
This value sets the maximum allowed value of any norm in
`tensor_var`.
norm_axes: sequence (list or tuple)
The axes over which to compute the norm. This overrides the
default norm axes defined for the number of dimensions
in `tensor_var`. When this is not specified and `tensor_var` is a
matrix (2D), this is set to `(0,)`. If `tensor_var` is a 3D, 4D or
5D tensor, it is set to a tuple listing all axes but axis 0. The
former default is useful for working with dense layers, the latter
is useful for 1D, 2D and 3D convolutional layers.
(Optional)
epsilon: scalar, optional
Value used to prevent numerical instability when dividing by
very small or zero norms.
Returns
-------
TensorVariable
Input `tensor_var` with rescaling applied to weight vectors
that violate the specified constraints.
Examples
--------
>>> param = aesara.shared(
... np.random.randn(100, 200).astype(aesara.config.floatX))
>>> update = param + 100
>>> update = norm_constraint(update, 10)
>>> func = aesara.function([], [], updates=[(param, update)])
>>> # Apply constrained update
>>> _ = func()
>>> from lasagne.utils import compute_norms
>>> norms = compute_norms(param.get_value())
>>> np.isclose(np.max(norms), 10)
True
Notes
-----
When `norm_axes` is not specified, the axes over which the norm is
computed depend on the dimensionality of the input variable. If it is
2D, it is assumed to come from a dense layer, and the norm is computed
over axis 0. If it is 3D, 4D or 5D, it is assumed to come from a
convolutional layer and the norm is computed over all trailing axes
beyond axis 0. For other uses, you should explicitly specify the axes
over which to compute the norm using `norm_axes`.
"""
ndim = tensor_var.ndim
if norm_axes is not None:
sum_over = tuple(norm_axes)
elif ndim == 2: # DenseLayer
sum_over = (0, )
elif ndim in [3, 4, 5]: # Conv{1,2,3}DLayer
sum_over = tuple(range(1, ndim))
else:
raise ValueError("Unsupported tensor dimensionality {}."
"Must specify `norm_axes`".format(ndim))
dtype = np.dtype(aesara.config.floatX).type
norms = aet.sqrt(aet.sum(aet.sqr(tensor_var), axis=sum_over,
keepdims=True))
target_norms = aet.clip(norms, 0, dtype(max_norm))
constrained_output = tensor_var * (target_norms / (dtype(epsilon) + norms))
return constrained_output
