def normal_lccdf(mu, sigma, x):
z = (x - mu) / sigma
return aet.switch(
aet.gt(z, 1.0),
aet.log(aet.erfcx(z / aet.sqrt(2.0)) / 2.0) - aet.sqr(z) / 2.0,
aet.log1p(-aet.erfc(-z / aet.sqrt(2.0)) / 2.0),
)
def normal_lcdf(mu, sigma, x):
"""Compute the log of the cumulative density function of the normal."""
z = (x - mu) / sigma
return aet.switch(
aet.lt(z, -1.0),
aet.log(aet.erfcx(-z / aet.sqrt(2.0)) / 2.0) - aet.sqr(z) / 2.0,
aet.log1p(-aet.erfc(z / aet.sqrt(2.0)) / 2.0),
)
def curfft(inp, norm=None):
r"""
Performs the fast Fourier transform of a real-valued input on the GPU.
The input must be a real-valued float32 variable of dimensions (m, ..., n).
It performs FFTs of size (..., n) on m batches.
The output is a GpuArray of dimensions (m, ..., n//2+1, 2). The second to
last dimension of the output contains the n//2+1 non-trivial elements of
the real-valued FFTs. The real and imaginary parts are stored as a pair of
float32 arrays.
Parameters
----------
inp
Array of real-valued float32 of size (m, ..., n), containing m inputs of
size (..., n).
norm : {None, 'ortho', 'no_norm'}
Normalization of transform. Following numpy, default *None* normalizes
only the inverse transform by n, 'ortho' yields the unitary transform
(:math:`1/\sqrt n` forward and inverse). In addition, 'no_norm' leaves
the transform unnormalized.
"""
s = inp.shape[1:]
cond_norm = _unitary(norm)
scaling = 1
if cond_norm == "ortho":
scaling = tt.sqrt(s.prod().astype("float32"))
return curfft_op(inp, s) / scaling
def _build_prior(self, name, X, reparameterize=True, **kwargs):
mu = self.mean_func(X)
cov = stabilize(self.cov_func(X))
shape = infer_shape(X, kwargs.pop("shape", None))
if reparameterize:
chi2 = pm.ChiSquared(name + "_chi2_", self.nu)
v = pm.Normal(name + "_rotated_", mu=0.0, sigma=1.0, size=shape, **kwargs)
f = pm.Deterministic(name, (at.sqrt(self.nu) / chi2) * (mu + cholesky(cov).dot(v)))
else:
f = pm.MvStudentT(name, nu=self.nu, mu=mu, cov=cov, size=shape, **kwargs)
return f
def log_diff_normal_cdf(mu, sigma, x, y):
"""
Compute :math:`\\log(\\Phi(\frac{x - \\mu}{\\sigma}) - \\Phi(\frac{y - \\mu}{\\sigma}))` safely in log space.
Parameters
----------
mu: float
mean
sigma: float
std
x: float
y: float
must be strictly less than x.
Returns
-------
log (\\Phi(x) - \\Phi(y))
"""
x = (x - mu) / sigma / aet.sqrt(2.0)
y = (y - mu) / sigma / aet.sqrt(2.0)
# To stabilize the computation, consider these three regions:
# 1) x > y > 0 => Use erf(x) = 1 - e^{-x^2} erfcx(x) and erf(y) =1 - e^{-y^2} erfcx(y)
# 2) 0 > x > y => Use erf(x) = e^{-x^2} erfcx(-x) and erf(y) = e^{-y^2} erfcx(-y)
# 3) x > 0 > y => Naive formula log( (erf(x) - erf(y)) / 2 ) works fine.
return aet.log(0.5) + aet.switch(
aet.gt(y, 0),
-aet.square(y) + aet.log(
aet.erfcx(y) -
aet.exp(aet.square(y) - aet.square(x)) * aet.erfcx(x)),
aet.switch(
aet.lt(x, 0), # 0 > x > y
-aet.square(x) + aet.log(
aet.erfcx(-x) -
aet.exp(aet.square(x) - aet.square(y)) * aet.erfcx(-y)),
aet.log(aet.erf(x) - aet.erf(y)), # x >0 > y
),
)
def full(self, X, Xs=None):
X, Xs = self._slice(X, Xs)
rx = self.lfunc(at.as_tensor_variable(X), self.args)
if Xs is None:
rz = self.lfunc(at.as_tensor_variable(X), self.args)
r2 = self.square_dist(X, X)
else:
rz = self.lfunc(at.as_tensor_variable(Xs), self.args)
r2 = self.square_dist(X, Xs)
rx2 = at.reshape(at.square(rx), (-1, 1))
rz2 = at.reshape(at.square(rz), (1, -1))
return at.sqrt((2.0 * at.outer(rx, rz)) / (rx2 + rz2)) * at.exp(-1.0 * r2 / (rx2 + rz2))
def make_model(cls):
with pm.Model() as model:
sd_mu = np.array([1, 2, 3, 4, 5])
sd_dist = pm.LogNormal.dist(mu=sd_mu, sigma=sd_mu / 10.0, size=5)
chol_packed = pm.LKJCholeskyCov("chol_packed", eta=3, n=5, sd_dist=sd_dist)
chol = pm.expand_packed_triangular(5, chol_packed, lower=True)
cov = at.dot(chol, chol.T)
stds = at.sqrt(at.diag(cov))
pm.Deterministic("log_stds", at.log(stds))
corr = cov / stds[None, :] / stds[:, None]
corr_entries_unit = (corr[np.tril_indices(5, -1)] + 1) / 2
pm.Deterministic("corr_entries_unit", corr_entries_unit)
return model
def adagrad_window(loss_or_grads=None,
params=None,
learning_rate=0.001,
epsilon=0.1,
n_win=10):
"""Returns a function that returns parameter updates.
Instead of accumulated estimate, uses running window
Parameters
----------
loss_or_grads: symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params: list of shared variables
The variables to generate update expressions for
learning_rate: float
Learning rate.
epsilon: float
Offset to avoid zero-division in the normalizer of adagrad.
n_win: int
Number of past steps to calculate scales of parameter gradients.
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
"""
if loss_or_grads is None and params is None:
return partial(adagrad_window, **_get_call_kwargs(locals()))
elif loss_or_grads is None or params is None:
raise ValueError(
"Please provide both `loss_or_grads` and `params` to get updates")
grads = get_or_compute_grads(loss_or_grads, params)
updates = OrderedDict()
for param, grad in zip(params, grads):
i = aesara.shared(pm.floatX(0))
i_int = i.astype("int32")
value = param.get_value(borrow=True)
accu = aesara.shared(
np.zeros(value.shape + (n_win, ), dtype=value.dtype))
# Append squared gradient vector to accu_new
accu_new = aet.set_subtensor(accu[..., i_int], grad**2)
i_new = aet.switch((i + 1) < n_win, i + 1, 0)
updates[accu] = accu_new
updates[i] = i_new
accu_sum = accu_new.sum(axis=-1)
updates[param] = param - (learning_rate * grad /
aet.sqrt(accu_sum + epsilon))
return updates
def cuirfft(inp, norm=None, is_odd=False):
r"""
Performs the inverse fast Fourier Transform with real-valued output on the GPU.
The input is a variable of dimensions (m, ..., n//2+1, 2) with
type float32 representing the non-trivial elements of m
real-valued Fourier transforms of initial size (..., n). The real and
imaginary parts are stored as a pair of float32 arrays.
The output is a real-valued float32 variable of dimensions (m, ..., n)
giving the m inverse FFTs.
Parameters
----------
inp
Array of float32 of size (m, ..., n//2+1, 2), containing m inputs
with n//2+1 non-trivial elements on the last dimension and real
and imaginary parts stored as separate arrays.
norm : {None, 'ortho', 'no_norm'}
Normalization of transform. Following numpy, default *None* normalizes
only the inverse transform by n, 'ortho' yields the unitary transform
(:math:`1/\sqrt n` forward and inverse). In addition, 'no_norm' leaves
the transform unnormalized.
is_odd : {True, False}
Set to True to get a real inverse transform output with an odd last dimension
of length (N-1)*2 + 1 for an input last dimension of length N.
"""
if is_odd not in (True, False):
raise ValueError("Invalid value %s for id_odd, must be True or False" %
is_odd)
s = inp.shape[1:-1]
if is_odd:
s = tt.set_subtensor(s[-1], (s[-1] - 1) * 2 + 1)
else:
s = tt.set_subtensor(s[-1], (s[-1] - 1) * 2)
cond_norm = _unitary(norm)
scaling = 1
if cond_norm is None:
scaling = s.prod().astype("float32")
elif cond_norm == "ortho":
scaling = tt.sqrt(s.prod().astype("float32"))
return cuirfft_op(inp, s) / scaling
def logp(self, x):
"""
Calculate log-probability of EulerMaruyama distribution at specified value.
Parameters
----------
x: numeric
Value for which log-probability is calculated.
Returns
-------
TensorVariable
"""
xt = x[:-1]
f, g = self.sde_fn(x[:-1], *self.sde_pars)
mu = xt + self.dt * f
sigma = at.sqrt(self.dt) * g
return at.sum(Normal.dist(mu=mu, sigma=sigma).logp(x[1:]))
def _build_conditional(self, Xnew, pred_noise, diag):
Xs, y, sigma = self.Xs, self.y, self.sigma
# Old points
X = cartesian(*Xs)
delta = y - self.mean_func(X)
Kns = [f(x) for f, x in zip(self.cov_funcs, Xs)]
eigs_sep, Qs = zip(*map(eigh, Kns)) # Unzip
QTs = list(map(at.transpose, Qs))
eigs = kron_diag(*eigs_sep) # Combine separate eigs
if sigma is not None:
eigs += sigma**2
# New points
Km = self.cov_func(Xnew, diag=diag)
Knm = self.cov_func(X, Xnew)
Kmn = Knm.T
# Build conditional mu
alpha = kron_dot(QTs, delta)
alpha = alpha / eigs[:, None]
alpha = kron_dot(Qs, alpha)
mu = at.dot(Kmn, alpha).ravel() + self.mean_func(Xnew)
# Build conditional cov
A = kron_dot(QTs, Knm)
A = A / at.sqrt(eigs[:, None])
if diag:
Asq = at.sum(at.square(A), 0)
cov = Km - Asq
if pred_noise:
cov += sigma
else:
Asq = at.dot(A.T, A)
cov = Km - Asq
if pred_noise:
cov += sigma * at.identity_like(cov)
return mu, cov
def std(self):
return at.sqrt(at.diag(self.cov))
def std(self):
if self.batched:
return at.sqrt(batched_diag(self.cov))
else:
return at.sqrt(at.diag(self.cov))
def invprobit(x):
return 0.5 * erfc(-x / sqrt(2.0))
def std_cdf(x):
"""
Calculates the standard normal cumulative distribution function.
"""
return 0.5 + 0.5 * aet.erf(x / aet.sqrt(2.0))
def adadelta(loss_or_grads=None,
params=None,
learning_rate=1.0,
rho=0.95,
epsilon=1e-6):
r""" Adadelta updates
Scale learning rates by the ratio of accumulated gradients to accumulated
updates, see [1]_ and notes for further description.
Parameters
----------
loss_or_grads : symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params : list of shared variables
The variables to generate update expressions for
learning_rate : float or symbolic scalar
The learning rate controlling the size of update steps
rho : float or symbolic scalar
Squared gradient moving average decay factor
epsilon : float or symbolic scalar
Small value added for numerical stability
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
Notes
-----
rho should be between 0 and 1. A value of rho close to 1 will decay the
moving average slowly and a value close to 0 will decay the moving average
fast.
rho = 0.95 and epsilon=1e-6 are suggested in the paper and reported to
work for multiple datasets (MNIST, speech).
In the paper, no learning rate is considered (so learning_rate=1.0).
Probably best to keep it at this value.
epsilon is important for the very first update (so the numerator does
not become 0).
Using the step size eta and a decay factor rho the learning rate is
calculated as:
.. math::
r_t &= \\rho r_{t-1} + (1-\\rho)*g^2\\\\
\\eta_t &= \\eta \\frac{\\sqrt{s_{t-1} + \\epsilon}}
{\sqrt{r_t + \epsilon}}\\\\
s_t &= \\rho s_{t-1} + (1-\\rho)*(\\eta_t*g)^2
Optimizer can be called without both loss_or_grads and params
in that case partial function is returned
References
----------
.. [1] Zeiler, M. D. (2012):
ADADELTA: An Adaptive Learning Rate Method.
arXiv Preprint arXiv:1212.5701.
Examples
--------
>>> a = aesara.shared(1.)
>>> b = a*2
>>> updates = adadelta(b, [a], learning_rate=.01)
>>> isinstance(updates, dict)
True
>>> optimizer = adadelta(learning_rate=.01)
>>> callable(optimizer)
True
>>> updates = optimizer(b, [a])
>>> isinstance(updates, dict)
True
"""
if loss_or_grads is None and params is None:
return partial(adadelta, **_get_call_kwargs(locals()))
elif loss_or_grads is None or params is None:
raise ValueError(
"Please provide both `loss_or_grads` and `params` to get updates")
grads = get_or_compute_grads(loss_or_grads, params)
updates = OrderedDict()
# Using aesara constant to prevent upcasting of float32
one = aet.constant(1)
for param, grad in zip(params, grads):
value = param.get_value(borrow=True)
# accu: accumulate gradient magnitudes
accu = aesara.shared(np.zeros(value.shape, dtype=value.dtype),
broadcastable=param.broadcastable)
# delta_accu: accumulate update magnitudes (recursively!)
delta_accu = aesara.shared(np.zeros(value.shape, dtype=value.dtype),
broadcastable=param.broadcastable)
# update accu (as in rmsprop)
accu_new = rho * accu + (one - rho) * grad**2
updates[accu] = accu_new
# compute parameter update, using the 'old' delta_accu
update = grad * aet.sqrt(delta_accu + epsilon) / aet.sqrt(accu_new +
epsilon)
updates[param] = param - learning_rate * update
# update delta_accu (as accu, but accumulating updates)
delta_accu_new = rho * delta_accu + (one - rho) * update**2
updates[delta_accu] = delta_accu_new
return updates
def rmsprop(loss_or_grads=None,
params=None,
learning_rate=1.0,
rho=0.9,
epsilon=1e-6):
"""RMSProp updates
Scale learning rates by dividing with the moving average of the root mean
squared (RMS) gradients. See [1]_ for further description.
Parameters
----------
loss_or_grads: symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params: list of shared variables
The variables to generate update expressions for
learning_rate: float or symbolic scalar
The learning rate controlling the size of update steps
rho: float or symbolic scalar
Gradient moving average decay factor
epsilon: float or symbolic scalar
Small value added for numerical stability
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
Notes
-----
`rho` should be between 0 and 1. A value of `rho` close to 1 will decay the
moving average slowly and a value close to 0 will decay the moving average
fast.
Using the step size :math:`\\eta` and a decay factor :math:`\\rho` the
learning rate :math:`\\eta_t` is calculated as:
.. math::
r_t &= \\rho r_{t-1} + (1-\\rho)*g^2\\\\
\\eta_t &= \\frac{\\eta}{\\sqrt{r_t + \\epsilon}}
Optimizer can be called without both loss_or_grads and params
in that case partial function is returned
References
----------
.. [1] Tieleman, aet. and Hinton, G. (2012):
Neural Networks for Machine Learning, Lecture 6.5 - rmsprop.
Coursera. http://www.youtube.com/watch?v=O3sxAc4hxZU (formula @5:20)
Examples
--------
>>> a = aesara.shared(1.)
>>> b = a*2
>>> updates = rmsprop(b, [a], learning_rate=.01)
>>> isinstance(updates, dict)
True
>>> optimizer = rmsprop(learning_rate=.01)
>>> callable(optimizer)
True
>>> updates = optimizer(b, [a])
>>> isinstance(updates, dict)
True
"""
if loss_or_grads is None and params is None:
return partial(rmsprop, **_get_call_kwargs(locals()))
elif loss_or_grads is None or params is None:
raise ValueError(
"Please provide both `loss_or_grads` and `params` to get updates")
grads = get_or_compute_grads(loss_or_grads, params)
updates = OrderedDict()
# Using aesara constant to prevent upcasting of float32
one = aet.constant(1)
for param, grad in zip(params, grads):
value = param.get_value(borrow=True)
accu = aesara.shared(np.zeros(value.shape, dtype=value.dtype),
broadcastable=param.broadcastable)
accu_new = rho * accu + (one - rho) * grad**2
updates[accu] = accu_new
updates[param] = param - (learning_rate * grad /
aet.sqrt(accu_new + epsilon))
return updates
def adagrad(loss_or_grads=None, params=None, learning_rate=1.0, epsilon=1e-6):
"""Adagrad updates
Scale learning rates by dividing with the square root of accumulated
squared gradients. See [1]_ for further description.
Parameters
----------
loss_or_grads: symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params: list of shared variables
The variables to generate update expressions for
learning_rate: float or symbolic scalar
The learning rate controlling the size of update steps
epsilon: float or symbolic scalar
Small value added for numerical stability
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
Notes
-----
Using step size eta Adagrad calculates the learning rate for feature i at
time step t as:
.. math:: \\eta_{t,i} = \\frac{\\eta}
{\\sqrt{\\sum^t_{t^\\prime} g^2_{t^\\prime,i}+\\epsilon}} g_{t,i}
as such the learning rate is monotonically decreasing.
Epsilon is not included in the typical formula, see [2]_.
Optimizer can be called without both loss_or_grads and params
in that case partial function is returned
References
----------
.. [1] Duchi, J., Hazan, E., & Singer, Y. (2011):
Adaptive subgradient methods for online learning and stochastic
optimization. JMLR, 12:2121-2159.
.. [2] Chris Dyer:
Notes on AdaGrad. http://www.ark.cs.cmu.edu/cdyer/adagrad.pdf
Examples
--------
>>> a = aesara.shared(1.)
>>> b = a*2
>>> updates = adagrad(b, [a], learning_rate=.01)
>>> isinstance(updates, dict)
True
>>> optimizer = adagrad(learning_rate=.01)
>>> callable(optimizer)
True
>>> updates = optimizer(b, [a])
>>> isinstance(updates, dict)
True
"""
if loss_or_grads is None and params is None:
return partial(adagrad, **_get_call_kwargs(locals()))
elif loss_or_grads is None or params is None:
raise ValueError(
"Please provide both `loss_or_grads` and `params` to get updates")
grads = get_or_compute_grads(loss_or_grads, params)
updates = OrderedDict()
for param, grad in zip(params, grads):
value = param.get_value(borrow=True)
accu = aesara.shared(np.zeros(value.shape, dtype=value.dtype),
broadcastable=param.broadcastable)
accu_new = accu + grad**2
updates[accu] = accu_new
updates[param] = param - (learning_rate * grad /
aet.sqrt(accu_new + epsilon))
return updates
def total_norm_constraint(tensor_vars,
max_norm,
epsilon=1e-7,
return_norm=False):
"""Rescales a list of tensors based on their combined norm
If the combined norm of the input tensors exceeds the threshold then all
tensors are rescaled such that the combined norm is equal to the threshold.
Scaling the norms of the gradients is often used when training recurrent
neural networks [1]_.
Parameters
----------
tensor_vars: List of TensorVariables.
Tensors to be rescaled.
max_norm: float
Threshold value for total norm.
epsilon: scalar, optional
Value used to prevent numerical instability when dividing by
very small or zero norms.
return_norm: bool
If true the total norm is also returned.
Returns
-------
tensor_vars_scaled: list of TensorVariables
The scaled tensor variables.
norm: Aesara scalar
The combined norms of the input variables prior to rescaling,
only returned if ``return_norms=True``.
Examples
--------
>>> from lasagne.layers import InputLayer, DenseLayer
>>> import lasagne
>>> from lasagne.updates import sgd, total_norm_constraint
>>> x = aet.matrix()
>>> y = aet.ivector()
>>> l_in = InputLayer((5, 10))
>>> l1 = DenseLayer(l_in, num_units=7, nonlinearity=aet.nnet.softmax)
>>> output = lasagne.layers.get_output(l1, x)
>>> cost = aet.mean(aet.nnet.categorical_crossentropy(output, y))
>>> all_params = lasagne.layers.get_all_params(l1)
>>> all_grads = aet.grad(cost, all_params)
>>> scaled_grads = total_norm_constraint(all_grads, 5)
>>> updates = sgd(scaled_grads, all_params, learning_rate=0.1)
Notes
-----
The total norm can be used to monitor training.
References
----------
.. [1] Sutskever, I., Vinyals, O., & Le, Q. V. (2014): Sequence to sequence
learning with neural networks. In Advances in Neural Information
Processing Systems (pp. 3104-3112).
"""
norm = aet.sqrt(sum(aet.sum(tensor**2) for tensor in tensor_vars))
dtype = np.dtype(aesara.config.floatX).type
target_norm = aet.clip(norm, 0, dtype(max_norm))
multiplier = target_norm / (dtype(epsilon) + norm)
tensor_vars_scaled = [step * multiplier for step in tensor_vars]
if return_norm:
return tensor_vars_scaled, norm
else:
return tensor_vars_scaled
def euclidean_dist(self, X, Xs):
r2 = self.square_dist(X, Xs)
return at.sqrt(r2 + 1e-12)
def test_batch_normalization_train():
for axes in ("per-activation", "spatial", (1, 2, 3, 4)):
for vartype in (tensor5, tensor3, vector):
x, scale, bias, running_mean, running_var = (vartype(n) for n in (
"x", "scale", "bias", "running_mean", "running_var"))
ndim = x.ndim
eps = 5e-3 # some non-standard value to test if it's used
running_average_factor = 0.3
# remove non-existing axes
if isinstance(axes, tuple):
axes = tuple(i for i in axes if i < ndim)
if len(axes) == 0:
continue
# forward pass
(
out,
x_mean,
x_invstd,
out_running_mean,
out_running_var,
) = batchnorm.batch_normalization_train(
x,
scale,
bias,
axes,
eps,
running_average_factor,
running_mean,
running_var,
)
# reference forward pass
if axes == "per-activation":
axes2 = (0, )
elif axes == "spatial":
axes2 = (0, ) + tuple(range(2, ndim))
else:
axes2 = axes
x_mean2 = x.mean(axis=axes2, keepdims=True)
x_var2 = x.var(axis=axes2, keepdims=True)
x_invstd2 = aet.reciprocal(aet.sqrt(x_var2 + eps))
scale2 = aet.addbroadcast(scale, *axes2)
bias2 = aet.addbroadcast(bias, *axes2)
out2 = (x - x_mean2) * (scale2 * x_invstd2) + bias2
m = aet.cast(
aet.prod(x.shape) / aet.prod(scale.shape),
aesara.config.floatX)
out_running_mean2 = (running_mean * (1 - running_average_factor) +
x_mean2 * running_average_factor)
out_running_var2 = (running_var * (1 - running_average_factor) +
(m /
(m - 1)) * x_var2 * running_average_factor)
# backward pass
dy = vartype("dy")
grads = aet.grad(None, wrt=[x, scale, bias], known_grads={out: dy})
# reference backward pass
grads2 = aet.grad(None,
wrt=[x, scale, bias],
known_grads={out2: dy})
# second-order backward pass
dx = vartype("dinputs")
dscale = vartype("dscale")
dbias = vartype("dbias")
grad_grads = aet.grad(
None,
wrt=[x, dy, scale],
known_grads=OrderedDict({
grads[0]: dx,
grads[1]: dscale,
grads[2]: dbias
}),
consider_constant=[
x,
dy,
scale,
bias,
x_mean,
x_invstd,
running_mean,
running_var,
],
return_disconnected="zero",
)
# reference second-order backward pass
grad_grads2 = aet.grad(
None,
wrt=[x, dy, scale],
known_grads=OrderedDict({
grads2[0]: dx,
grads2[1]: dscale,
grads2[2]: dbias
}),
consider_constant=[
x,
dy,
scale,
bias,
x_mean2,
x_var2,
running_mean,
running_var,
],
return_disconnected="zero",
)
# compile
f = aesara.function(
[
x, scale, bias, running_mean, running_var, dy, dx, dscale,
dbias
],
[
out,
x_mean,
x_invstd,
out_running_mean,
out_running_var,
out2,
x_mean2,
x_invstd2,
out_running_mean2,
out_running_var2,
] + grads + grads2 + grad_grads + grad_grads2,
)
# check if the abstract Ops have been replaced
assert not any([
isinstance(
n.op,
(
batchnorm.AbstractBatchNormTrain,
batchnorm.AbstractBatchNormInference,
batchnorm.AbstractBatchNormTrainGrad,
),
) for n in f.maker.fgraph.toposort()
])
# run
for data_shape in ((5, 10, 30, 40, 10), (4, 3, 1, 1, 1), (2, 3, 5,
5, 5)):
data_shape = data_shape[:ndim]
param_shape = tuple(1 if d in axes2 else s
for d, s in enumerate(data_shape))
rng = np.random.default_rng(1234)
X = 4 + 3 * rng.random(data_shape).astype(aesara.config.floatX)
Dy = -1 + 2 * rng.random(data_shape).astype(
aesara.config.floatX)
Scale = rng.random(param_shape).astype(aesara.config.floatX)
Bias = rng.random(param_shape).astype(aesara.config.floatX)
Running_mean = rng.random(param_shape).astype(
aesara.config.floatX)
Running_var = rng.random(param_shape).astype(
aesara.config.floatX)
Dx = 4 + 3 * rng.random(data_shape).astype(
aesara.config.floatX)
Dscale = -1 + 2 * rng.random(param_shape).astype(
aesara.config.floatX)
Dbias = rng.random(param_shape).astype(aesara.config.floatX)
outputs = f(X, Scale, Bias, Running_mean, Running_var, Dy, Dx,
Dscale, Dbias)
# compare outputs
utt.assert_allclose(outputs[0], outputs[0 + 5]) # out
utt.assert_allclose(outputs[1], outputs[1 + 5]) # mean
utt.assert_allclose(outputs[2], outputs[2 + 5]) # invstd
utt.assert_allclose(outputs[3], outputs[3 + 5]) # running_mean
utt.assert_allclose(np.nan_to_num(outputs[4]),
np.nan_to_num(outputs[4 +
5])) # running_var
# compare gradients
utt.assert_allclose(outputs[10], outputs[10 + 3],
atol=1e-4) # dx
utt.assert_allclose(outputs[11],
outputs[11 + 3],
rtol=2e-4,
atol=1e-4) # dscale
utt.assert_allclose(outputs[12], outputs[12 + 3]) # dbias
# compare second-order gradients
utt.assert_allclose(outputs[16], outputs[16 + 3],
atol=1e-4) # ddx
utt.assert_allclose(outputs[17], outputs[17 + 3]) # ddy
utt.assert_allclose(outputs[18],
outputs[18 + 3],
rtol=3e-4,
atol=1e-4) # ddscale
def test_batch_normalization_test():
for axes in ("per-activation", "spatial", (1, 2, 3, 4)):
for vartype in (tensor5, tensor3, vector):
x, scale, bias, mean, var = (vartype(n)
for n in ("x", "scale", "bias",
"mean", "var"))
ndim = x.ndim
eps = 5e-3 # some non-standard value to test if it's used
# remove non-existing axes
if isinstance(axes, tuple):
axes = tuple(i for i in axes if i < ndim)
if len(axes) == 0:
continue
# forward pass
out = batchnorm.batch_normalization_test(x, scale, bias, mean, var,
axes, eps)
# reference forward pass
if axes == "per-activation":
axes2 = (0, )
elif axes == "spatial":
axes2 = (0, ) + tuple(range(2, ndim))
else:
axes2 = axes
scale2, bias2, mean2, var2 = (aet.addbroadcast(t, *axes2)
for t in (scale, bias, mean, var))
out2 = (x - mean2) * (scale2 / aet.sqrt(var2 + eps)) + bias2
# backward pass
dy = vartype("dy")
grads = aet.grad(None,
wrt=[x, scale, bias, mean, var],
known_grads={out: dy})
# reference backward pass
grads2 = aet.grad(None,
wrt=[x, scale, bias, mean, var],
known_grads={out2: dy})
# compile
f = aesara.function([x, scale, bias, mean, var, dy],
[out, out2] + grads + grads2)
# check if the abstract Ops have been replaced
assert not any([
isinstance(
n.op,
(
batchnorm.AbstractBatchNormTrain,
batchnorm.AbstractBatchNormInference,
batchnorm.AbstractBatchNormTrainGrad,
),
) for n in f.maker.fgraph.toposort()
])
# run
for data_shape in ((10, 20, 30, 40, 10), (4, 3, 1, 1, 1), (1, 1, 5,
5, 5)):
data_shape = data_shape[:ndim]
param_shape = tuple(1 if d in axes2 else s
for d, s in enumerate(data_shape))
rng = np.random.default_rng(1234)
X = 4 + 3 * rng.random(data_shape).astype(aesara.config.floatX)
Dy = -1 + 2 * rng.random(data_shape).astype(
aesara.config.floatX)
Scale = rng.random(param_shape).astype(aesara.config.floatX)
Bias = rng.random(param_shape).astype(aesara.config.floatX)
Mean = rng.random(param_shape).astype(aesara.config.floatX)
Var = rng.random(param_shape).astype(aesara.config.floatX)
outputs = f(X, Scale, Bias, Mean, Var, Dy)
# compare outputs
utt.assert_allclose(outputs[0], outputs[1]) # out
# compare gradients
utt.assert_allclose(outputs[2], outputs[2 + 5],
atol=4e-5) # dx
utt.assert_allclose(outputs[3], outputs[3 + 5],
atol=4e-5) # dscale
utt.assert_allclose(outputs[4], outputs[4 + 5]) # dbias
utt.assert_allclose(outputs[5], outputs[5 + 5]) # dmean
utt.assert_allclose(outputs[6],
outputs[6 + 5],
rtol=2e-3,
atol=4e-5) # dvar
def norm_constraint(tensor_var, max_norm, norm_axes=None, epsilon=1e-7):
"""Max weight norm constraints and gradient clipping
This takes a TensorVariable and rescales it so that incoming weight
norms are below a specified constraint value. Vectors violating the
constraint are rescaled so that they are within the allowed range.
Parameters
----------
tensor_var: TensorVariable
Aesara expression for update, gradient, or other quantity.
max_norm: scalar
This value sets the maximum allowed value of any norm in
`tensor_var`.
norm_axes: sequence (list or tuple)
The axes over which to compute the norm. This overrides the
default norm axes defined for the number of dimensions
in `tensor_var`. When this is not specified and `tensor_var` is a
matrix (2D), this is set to `(0,)`. If `tensor_var` is a 3D, 4D or
5D tensor, it is set to a tuple listing all axes but axis 0. The
former default is useful for working with dense layers, the latter
is useful for 1D, 2D and 3D convolutional layers.
(Optional)
epsilon: scalar, optional
Value used to prevent numerical instability when dividing by
very small or zero norms.
Returns
-------
TensorVariable
Input `tensor_var` with rescaling applied to weight vectors
that violate the specified constraints.
Examples
--------
>>> param = aesara.shared(
... np.random.randn(100, 200).astype(aesara.config.floatX))
>>> update = param + 100
>>> update = norm_constraint(update, 10)
>>> func = aesara.function([], [], updates=[(param, update)])
>>> # Apply constrained update
>>> _ = func()
>>> from lasagne.utils import compute_norms
>>> norms = compute_norms(param.get_value())
>>> np.isclose(np.max(norms), 10)
True
Notes
-----
When `norm_axes` is not specified, the axes over which the norm is
computed depend on the dimensionality of the input variable. If it is
2D, it is assumed to come from a dense layer, and the norm is computed
over axis 0. If it is 3D, 4D or 5D, it is assumed to come from a
convolutional layer and the norm is computed over all trailing axes
beyond axis 0. For other uses, you should explicitly specify the axes
over which to compute the norm using `norm_axes`.
"""
ndim = tensor_var.ndim
if norm_axes is not None:
sum_over = tuple(norm_axes)
elif ndim == 2: # DenseLayer
sum_over = (0, )
elif ndim in [3, 4, 5]: # Conv{1,2,3}DLayer
sum_over = tuple(range(1, ndim))
else:
raise ValueError("Unsupported tensor dimensionality {}."
"Must specify `norm_axes`".format(ndim))
dtype = np.dtype(aesara.config.floatX).type
norms = aet.sqrt(aet.sum(aet.sqr(tensor_var), axis=sum_over,
keepdims=True))
target_norms = aet.clip(norms, 0, dtype(max_norm))
constrained_output = tensor_var * (target_norms / (dtype(epsilon) + norms))
return constrained_output
def normal(
self,
size,
avg=0.0,
std=1.0,
ndim=None,
dtype=None,
nstreams=None,
truncate=False,
**kwargs,
):
"""
Sample a tensor of values from a normal distribution.
Parameters
----------
size : int_vector_like
Array dimensions for the output tensor.
avg : float_like, optional
The mean value for the truncated normal to sample from (defaults to 0.0).
std : float_like, optional
The standard deviation for the truncated normal to sample from (defaults to 1.0).
truncate : bool, optional
Truncates the normal distribution at 2 standard deviations if True (defaults to False).
When this flag is set, the standard deviation of the result will be less than the one specified.
ndim : int, optional
The number of dimensions for the output tensor (defaults to None).
This argument is necessary if the size argument is ambiguous on the number of dimensions.
dtype : str, optional
The data-type for the output tensor. If not specified,
the dtype is inferred from avg and std, but it is at least as precise as floatX.
kwargs
Other keyword arguments for random number generation (see uniform).
Returns
-------
samples : TensorVariable
A Aesara tensor of samples randomly drawn from a normal distribution.
"""
size = _check_size(size)
avg = undefined_grad(as_tensor_variable(avg))
std = undefined_grad(as_tensor_variable(std))
if dtype is None:
dtype = scal.upcast(config.floatX, avg.dtype, std.dtype)
avg = tensor.cast(avg, dtype=dtype)
std = tensor.cast(std, dtype=dtype)
# generate even number of uniform samples
# Do manual constant folding to lower optiimizer work.
if isinstance(size, aesara.Constant):
n_odd_samples = size.prod(dtype="int64")
else:
n_odd_samples = tensor.prod(size, dtype="int64")
n_even_samples = n_odd_samples + n_odd_samples % 2
uniform = self.uniform(
(n_even_samples, ),
low=0.0,
high=1.0,
ndim=1,
dtype=dtype,
nstreams=nstreams,
**kwargs,
)
# box-muller transform
u1 = uniform[:n_even_samples // 2]
u2 = uniform[n_even_samples // 2:]
r = tensor.sqrt(-2.0 * tensor.log(u1))
theta = np.array(2.0 * np.pi, dtype=dtype) * u2
cos_theta, sin_theta = tensor.cos(theta), tensor.sin(theta)
z0 = r * cos_theta
z1 = r * sin_theta
if truncate:
# use valid samples
to_fix0 = (z0 < -2.0) | (z0 > 2.0)
to_fix1 = (z1 < -2.0) | (z1 > 2.0)
z0_valid = z0[tensor.nonzero(~to_fix0)]
z1_valid = z1[tensor.nonzero(~to_fix1)]
# re-sample invalid samples
to_fix0 = tensor.nonzero(to_fix0)[0]
to_fix1 = tensor.nonzero(to_fix1)[0]
n_fix_samples = to_fix0.size + to_fix1.size
lower = tensor.constant(1.0 / np.e**2, dtype=dtype)
u_fix = self.uniform(
(n_fix_samples, ),
low=lower,
high=1.0,
ndim=1,
dtype=dtype,
nstreams=nstreams,
**kwargs,
)
r_fix = tensor.sqrt(-2.0 * tensor.log(u_fix))
z0_fixed = r_fix[:to_fix0.size] * cos_theta[to_fix0]
z1_fixed = r_fix[to_fix0.size:] * sin_theta[to_fix1]
# pack everything together to a useful result
norm_samples = tensor.join(0, z0_valid, z0_fixed, z1_valid,
z1_fixed)
else:
norm_samples = tensor.join(0, z0, z1)
if isinstance(n_odd_samples, aesara.Variable):
samples = norm_samples[:n_odd_samples]
elif n_odd_samples % 2 == 1:
samples = norm_samples[:-1]
else:
samples = norm_samples
samples = tensor.reshape(samples, newshape=size, ndim=ndim)
samples *= std
samples += avg
return samples
def probit(p):
return -sqrt(2.0) * erfcinv(2.0 * p)
def adam(loss_or_grads=None,
params=None,
learning_rate=0.001,
beta1=0.9,
beta2=0.999,
epsilon=1e-8):
"""Adam updates
Adam updates implemented as in [1]_.
Parameters
----------
loss_or_grads: symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params: list of shared variables
The variables to generate update expressions for
learning_rate: float
Learning rate
beta1: float
Exponential decay rate for the first moment estimates.
beta2: float
Exponential decay rate for the second moment estimates.
epsilon: float
Constant for numerical stability.
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
Notes
-----
The paper [1]_ includes an additional hyperparameter lambda. This is only
needed to prove convergence of the algorithm and has no practical use
(personal communication with the authors), it is therefore omitted here.
Optimizer can be called without both loss_or_grads and params
in that case partial function is returned
References
----------
.. [1] Kingma, Diederik, and Jimmy Ba (2014):
Adam: A Method for Stochastic Optimization.
arXiv preprint arXiv:1412.6980.
Examples
--------
>>> a = aesara.shared(1.)
>>> b = a*2
>>> updates = adam(b, [a], learning_rate=.01)
>>> isinstance(updates, dict)
True
>>> optimizer = adam(learning_rate=.01)
>>> callable(optimizer)
True
>>> updates = optimizer(b, [a])
>>> isinstance(updates, dict)
True
"""
if loss_or_grads is None and params is None:
return partial(adam, **_get_call_kwargs(locals()))
elif loss_or_grads is None or params is None:
raise ValueError(
"Please provide both `loss_or_grads` and `params` to get updates")
all_grads = get_or_compute_grads(loss_or_grads, params)
t_prev = aesara.shared(pm.aesaraf.floatX(0.0))
updates = OrderedDict()
# Using aesara constant to prevent upcasting of float32
one = aet.constant(1)
t = t_prev + 1
a_t = learning_rate * aet.sqrt(one - beta2**t) / (one - beta1**t)
for param, g_t in zip(params, all_grads):
value = param.get_value(borrow=True)
m_prev = aesara.shared(np.zeros(value.shape, dtype=value.dtype),
broadcastable=param.broadcastable)
v_prev = aesara.shared(np.zeros(value.shape, dtype=value.dtype),
broadcastable=param.broadcastable)
m_t = beta1 * m_prev + (one - beta1) * g_t
v_t = beta2 * v_prev + (one - beta2) * g_t**2
step = a_t * m_t / (aet.sqrt(v_t) + epsilon)
updates[m_prev] = m_t
updates[v_prev] = v_t
updates[param] = param - step
updates[t_prev] = t
return updates
def volatility_update(x, vol, w, a, b):
return at.sqrt(w + a * at.square(x) + b * at.square(vol))
