def adam(loss_or_grads, params, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):
"""Adam updates
Adam updates implemented as in [1]_.
Parameters
----------
loss_or_grads : symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params : list of shared variables
The variables to generate update expressions for
learning_rate : float
Learning rate
beta1 : float
Exponential decay rate for the first moment estimates.
beta2 : float
Exponential decay rate for the second moment estimates.
epsilon : float
Constant for numerical stability.
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
Notes
-----
The paper [1]_ includes an additional hyperparameter lambda. This is only
needed to prove convergence of the algorithm and has no practical use
(personal communication with the authors), it is therefore omitted here.
References
----------
.. [1] Kingma, Diederik, and Jimmy Ba (2014):
Adam: A Method for Stochastic Optimization.
arXiv preprint arXiv:1412.6980.
"""
all_grads = get_or_compute_grads(loss_or_grads, params)
t_prev = theano.shared(utils.floatX(0.0))
updates = OrderedDict()
t = t_prev + 1
a_t = learning_rate * T.sqrt(1 - beta2 ** t) / (1 - beta1 ** t)
for param, g_t in zip(params, all_grads):
value = param.get_value(borrow=True)
m_prev = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
v_prev = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
m_t = beta1 * m_prev + (1 - beta1) * g_t
v_t = beta2 * v_prev + (1 - beta2) * g_t ** 2
step = a_t * m_t / (T.sqrt(v_t) + epsilon)
updates[m_prev] = m_t
updates[v_prev] = v_t
updates[param] = param - step
updates[t_prev] = t
return updates
def adamax(loss_or_grads, params, learning_rate=0.002, beta1=0.9, beta2=0.999, epsilon=1e-8):
"""Adamax updates
Adamax updates implemented as in [1]_. This is a variant of of the Adam
algorithm based on the infinity norm.
Parameters
----------
loss_or_grads : symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params : list of shared variables
The variables to generate update expressions for
learning_rate : float
Learning rate
beta1 : float
Exponential decay rate for the first moment estimates.
beta2 : float
Exponential decay rate for the weighted infinity norm estimates.
epsilon : float
Constant for numerical stability.
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
References
----------
.. [1] Kingma, Diederik, and Jimmy Ba (2014):
Adam: A Method for Stochastic Optimization.
arXiv preprint arXiv:1412.6980.
"""
all_grads = get_or_compute_grads(loss_or_grads, params)
t_prev = theano.shared(utils.floatX(0.0))
updates = OrderedDict()
t = t_prev + 1
a_t = learning_rate / (1 - beta1 ** t)
for param, g_t in zip(params, all_grads):
value = param.get_value(borrow=True)
m_prev = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
u_prev = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
m_t = beta1 * m_prev + (1 - beta1) * g_t
u_t = T.maximum(beta2 * u_prev, abs(g_t))
step = a_t * m_t / (u_t + epsilon)
updates[m_prev] = m_t
updates[u_prev] = u_t
updates[param] = param - step
updates[t_prev] = t
return updates
def shared_empty(dim=2, dtype=None):
"""Creates empty Theano shared variable.
Shortcut to create an empty Theano shared variable with
the specified number of dimensions.
Parameters
----------
dim : int, optional
The number of dimensions for the empty variable, defaults to 2.
dtype : a numpy data-type, optional
The desired dtype for the variable. Defaults to the Theano
``floatX`` dtype.
Returns
-------
Theano shared variable
An empty Theano shared variable of dtype ``dtype`` with
`dim` dimensions.
"""
if dtype is None:
dtype = theano.config.floatX
shp = tuple([1] * dim)
return theano.shared(np.zeros(shp, dtype=dtype))
def adagrad(loss_or_grads, params, learning_rate=1.0, epsilon=1e-6):
"""Adagrad updates
Scale learning rates by dividing with the square root of accumulated
squared gradients. See [1]_ for further description.
Parameters
----------
loss_or_grads : symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params : list of shared variables
The variables to generate update expressions for
learning_rate : float or symbolic scalar
The learning rate controlling the size of update steps
epsilon : float or symbolic scalar
Small value added for numerical stability
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
Notes
-----
Using step size eta Adagrad calculates the learning rate for feature i at
time step t as:
.. math:: \\eta_{t,i} = \\frac{\\eta}
{\\sqrt{\\sum^t_{t^\\prime} g^2_{t^\\prime,i}+\\epsilon}} g_{t,i}
as such the learning rate is monotonically decreasing.
Epsilon is not included in the typical formula, see [2]_.
References
----------
.. [1] Duchi, J., Hazan, E., & Singer, Y. (2011):
Adaptive subgradient methods for online learning and stochastic
optimization. JMLR, 12:2121-2159.
.. [2] Chris Dyer:
Notes on AdaGrad. http://www.ark.cs.cmu.edu/cdyer/adagrad.pdf
"""
grads = get_or_compute_grads(loss_or_grads, params)
updates = OrderedDict()
for param, grad in zip(params, grads):
value = theano.compat.get_value(param, borrow=True)
accu = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=theano.compat.broadcastable(param))
accu_new = accu + grad ** 2
updates[accu] = accu_new
updates[param] = param - (learning_rate * grad / T.sqrt(accu_new + epsilon))
return updates
def rmsprop(loss_or_grads, params, learning_rate=1.0, rho=0.9, epsilon=1e-6):
"""RMSProp updates
Scale learning rates by dividing with the moving average of the root mean
squared (RMS) gradients. See [1]_ for further description.
Parameters
----------
loss_or_grads : symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params : list of shared variables
The variables to generate update expressions for
learning_rate : float or symbolic scalar
The learning rate controlling the size of update steps
rho : float or symbolic scalar
Gradient moving average decay factor
epsilon : float or symbolic scalar
Small value added for numerical stability
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
Notes
-----
`rho` should be between 0 and 1. A value of `rho` close to 1 will decay the
moving average slowly and a value close to 0 will decay the moving average
fast.
Using the step size :math:`\\eta` and a decay factor :math:`\\rho` the
learning rate :math:`\\eta_t` is calculated as:
.. math::
r_t &= \\rho r_{t-1} + (1-\\rho)*g^2\\\\
\\eta_t &= \\frac{\\eta}{\\sqrt{r_t + \\epsilon}}
References
----------
.. [1] Tieleman, T. and Hinton, G. (2012):
Neural Networks for Machine Learning, Lecture 6.5 - rmsprop.
Coursera. http://www.youtube.com/watch?v=O3sxAc4hxZU (formula @5:20)
"""
grads = get_or_compute_grads(loss_or_grads, params)
updates = OrderedDict()
for param, grad in zip(params, grads):
value = param.get_value(borrow=True)
accu = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
accu_new = rho * accu + (1 - rho) * grad ** 2
updates[accu] = accu_new
updates[param] = param - (learning_rate * grad / T.sqrt(accu_new + epsilon))
return updates
def apply_nesterov_momentum(updates, params=None, momentum=0.9):
"""Returns a modified update dictionary including Nesterov momentum
Generates update expressions of the form:
* ``velocity := momentum * velocity + updates[param] - param``
* ``param := param + momentum * velocity + updates[param] - param``
Parameters
----------
updates : OrderedDict
A dictionary mapping parameters to update expressions
params : iterable of shared variables, optional
The variables to apply momentum to. If omitted, will apply
momentum to all `updates.keys()`.
momentum : float or symbolic scalar, optional
The amount of momentum to apply. Higher momentum results in
smoothing over more update steps. Defaults to 0.9.
Returns
-------
OrderedDict
A copy of `updates` with momentum updates for all `params`.
Notes
-----
Higher momentum also results in larger update steps. To counter that,
you can optionally scale your learning rate by `1 - momentum`.
The classic formulation of Nesterov momentum (or Nesterov accelerated
gradient) requires the gradient to be evaluated at the predicted next
position in parameter space. Here, we use the formulation described at
https://github.com/lisa-lab/pylearn2/pull/136#issuecomment-10381617,
which allows the gradient to be evaluated at the current parameters.
See Also
--------
nesterov_momentum : Shortcut applying Nesterov momentum to SGD updates
"""
if params is None:
params = updates.keys()
updates = OrderedDict(updates)
for param in params:
value = theano.compat.get_value(param, borrow=True)
velocity = theano.shared(
np.zeros(value.shape, dtype=value.dtype), broadcastable=getattr(param, "broadcastable", None)
)
x = momentum * velocity + updates[param] - param
updates[velocity] = x
updates[param] = momentum * x + updates[param]
return updates
def apply_momentum(updates, params=None, momentum=0.9):
"""Returns a modified update dictionary including momentum
Generates update expressions of the form:
* ``velocity := momentum * velocity + updates[param] - param``
* ``param := param + velocity``
Parameters
----------
updates : OrderedDict
A dictionary mapping parameters to update expressions
params : iterable of shared variables, optional
The variables to apply momentum to. If omitted, will apply
momentum to all `updates.keys()`.
momentum : float or symbolic scalar, optional
The amount of momentum to apply. Higher momentum results in
smoothing over more update steps. Defaults to 0.9.
Returns
-------
OrderedDict
A copy of `updates` with momentum updates for all `params`.
Notes
-----
Higher momentum also results in larger update steps. To counter that,
you can optionally scale your learning rate by `1 - momentum`.
See Also
--------
momentum : Shortcut applying momentum to SGD updates
"""
if params is None:
params = updates.keys()
updates = OrderedDict(updates)
for param in params:
value = param.get_value(borrow=True)
velocity = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
x = momentum * velocity + updates[param]
updates[velocity] = x - param
updates[param] = x
return updates
def adadelta(loss_or_grads, params, learning_rate=1.0, rho=0.95, epsilon=1e-6):
""" Adadelta updates
Scale learning rates by a the ratio of accumulated gradients to accumulated
step sizes, see [1]_ and notes for further description.
Parameters
----------
loss_or_grads : symbolic expression or list of expressions
A scalar loss expression, or a list of gradient expressions
params : list of shared variables
The variables to generate update expressions for
learning_rate : float or symbolic scalar
The learning rate controlling the size of update steps
rho : float or symbolic scalar
Squared gradient moving average decay factor
epsilon : float or symbolic scalar
Small value added for numerical stability
Returns
-------
OrderedDict
A dictionary mapping each parameter to its update expression
Notes
-----
rho should be between 0 and 1. A value of rho close to 1 will decay the
moving average slowly and a value close to 0 will decay the moving average
fast.
rho = 0.95 and epsilon=1e-6 are suggested in the paper and reported to
work for multiple datasets (MNIST, speech).
In the paper, no learning rate is considered (so learning_rate=1.0).
Probably best to keep it at this value.
epsilon is important for the very first update (so the numerator does
not become 0).
Using the step size eta and a decay factor rho the learning rate is
calculated as:
.. math::
r_t &= \\rho r_{t-1} + (1-\\rho)*g^2\\\\
\\eta_t &= \\eta \\frac{\\sqrt{s_{t-1} + \\epsilon}}
{\sqrt{r_t + \epsilon}}\\\\
s_t &= \\rho s_{t-1} + (1-\\rho)*(\\eta_t*g)^2
References
----------
.. [1] Zeiler, M. D. (2012):
ADADELTA: An Adaptive Learning Rate Method.
arXiv Preprint arXiv:1212.5701.
"""
grads = get_or_compute_grads(loss_or_grads, params)
updates = OrderedDict()
for param, grad in zip(params, grads):
value = param.get_value(borrow=True)
# accu: accumulate gradient magnitudes
accu = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
# delta_accu: accumulate update magnitudes (recursively!)
delta_accu = theano.shared(np.zeros(value.shape, dtype=value.dtype), broadcastable=param.broadcastable)
# update accu (as in rmsprop)
accu_new = rho * accu + (1 - rho) * grad ** 2
updates[accu] = accu_new
# compute parameter update, using the 'old' delta_accu
update = grad * T.sqrt(delta_accu + epsilon) / T.sqrt(accu_new + epsilon)
updates[param] = param - learning_rate * update
# update delta_accu (as accu, but accumulating updates)
delta_accu_new = rho * delta_accu + (1 - rho) * update ** 2
updates[delta_accu] = delta_accu_new
return updates
def create_param(spec, shape, name=None):
"""
Helper method to create Theano shared variables for layer parameters
and to initialize them.
Parameters
----------
spec : numpy array, Theano expression, or callable
Either of the following:
* a numpy array with the initial parameter values
* a Theano expression or shared variable representing the parameters
* a function or callable that takes the desired shape of
the parameter array as its single argument and returns
a numpy array.
shape : iterable of int
a tuple or other iterable of integers representing the desired
shape of the parameter array.
name : string, optional
If a new variable is created, the name to give to the parameter
variable. This is ignored if `spec` is already a Theano expression
or shared variable.
Returns
-------
Theano shared variable or Theano expression
A Theano shared variable or expression representing layer parameters.
If a numpy array was provided, a shared variable is initialized to
contain this array. If a shared variable or expression was provided,
it is simply returned. If a callable was provided, it is called, and
its output is used to initialize a shared variable.
Notes
-----
This function is called by :meth:`Layer.add_param()` in the constructor
of most :class:`Layer` subclasses. This enables those layers to
support initialization with numpy arrays, existing Theano shared variables
or expressions, and callables for generating initial parameter values.
"""
shape = tuple(shape) # convert to tuple if needed
if any(d <= 0 for d in shape):
raise ValueError((
"Cannot create param with a non-positive shape dimension. "
"Tried to create param with shape=%r, name=%r") % (shape, name))
if theano.compat.is_variable(spec):
# We cannot check the shape here, Theano expressions (even shared
# variables) do not have a fixed compile-time shape. We can check the
# dimensionality though.
# Note that we cannot assign a name here. We could assign to the
# `name` attribute of the variable, but the user may have already
# named the variable and we don't want to override this.
if spec.ndim != len(shape):
raise RuntimeError("parameter variable has %d dimensions, "
"should be %d" % (spec.ndim, len(shape)))
return spec
elif isinstance(spec, np.ndarray):
if spec.shape != shape:
raise RuntimeError("parameter array has shape %s, should be "
"%s" % (spec.shape, shape))
return theano.shared(spec, name=name)
elif hasattr(spec, '__call__'):
arr = spec(shape)
try:
arr = floatX(arr)
except Exception:
raise RuntimeError("cannot initialize parameters: the "
"provided callable did not return an "
"array-like value")
if arr.shape != shape:
raise RuntimeError("cannot initialize parameters: the "
"provided callable did not return a value "
"with the correct shape")
return theano.shared(arr, name=name)
else:
raise RuntimeError("cannot initialize parameters: 'spec' is not "
"a numpy array, a Theano expression, or a "
"callable")
