def jacobian(expression, wrt, consider_constant=None,
disconnected_inputs='raise'):
'''
similar implementation as in theano.gradient, but ignore not empty updates
(because when you use it in lasagna there is should be some update and it is ok)
'''
from theano.tensor import arange
# Check inputs have the right format
assert isinstance(expression, Variable), \
"tensor.jacobian expects a Variable as `expression`"
assert expression.ndim < 2, \
("tensor.jacobian expects a 1 dimensional variable as "
"`expression`. If not use flatten to make it a vector")
using_list = isinstance(wrt, list)
using_tuple = isinstance(wrt, tuple)
if isinstance(wrt, (list, tuple)):
wrt = list(wrt)
else:
wrt = [wrt]
if expression.ndim == 0:
# expression is just a scalar, use grad
return format_as(using_list, using_tuple,
grad(expression,
wrt,
consider_constant=consider_constant,
disconnected_inputs=disconnected_inputs))
def inner_function(*args):
idx = args[0]
expr = args[1]
rvals = []
for inp in args[2:]:
rval = grad(expr[idx],
inp,
consider_constant=consider_constant,
disconnected_inputs=disconnected_inputs)
rvals.append(rval)
return rvals
# Computing the gradients does not affect the random seeds on any random
# generator used n expression (because during computing gradients we are
# just backtracking over old values. (rp Jan 2012 - if anyone has a
# counter example please show me)
jacobs, updates = theano.scan(inner_function,
sequences=arange(expression.shape[0]),
non_sequences=[expression] + wrt)
#the only difference from theano implementation -- no assertion for updates
# assert not updates, \
# ("Scan has returned a list of updates. This should not "
# "happen! Report this to theano-users (also include the "
# "script that generated the error)")
return format_as(using_list, using_tuple, jacobs)
def hessian_(cost, wrt, consider_constant=None,
disconnected_inputs='raise'):
"""
:type cost: Scalar (0-dimensional) Variable.
:type wrt: Vector (1-dimensional tensor) 'Variable' or list of
vectors (1-dimensional tensors) Variables
:param consider_constant: a list of expressions not to backpropagate
through
:type disconnected_inputs: string
:param disconnected_inputs: Defines the behaviour if some of the variables
in ``wrt`` are not part of the computational graph computing ``cost``
(or if all links are non-differentiable). The possible values are:
- 'ignore': considers that the gradient on these parameters is zero.
- 'warn': consider the gradient zero, and print a warning.
- 'raise': raise an exception.
:return: either a instance of Variable or list/tuple of Variables
(depending upon `wrt`) repressenting the Hessian of the `cost`
with respect to (elements of) `wrt`. If an element of `wrt` is not
differentiable with respect to the output, then a zero
variable is returned. The return value is of same type
as `wrt`: a list/tuple or TensorVariable in all cases.
"""
from theano.tensor import arange
# Check inputs have the right format
assert isinstance(cost, Variable), \
"tensor.hessian expects a Variable as `cost`"
assert cost.ndim == 0, \
"tensor.hessian expects a 0 dimensional variable as `cost`"
using_list = isinstance(wrt, list)
using_tuple = isinstance(wrt, tuple)
if isinstance(wrt, (list, tuple)):
wrt = list(wrt)
else:
wrt = [wrt]
expr = grad(cost, wrt, consider_constant=consider_constant,
disconnected_inputs=disconnected_inputs)
# It is possible that the inputs are disconnected from expr,
# even if they are connected to cost.
# This should not be an error.
hess, updates = scan(lambda i, y: grad(
y[i],
wrt,
consider_constant=consider_constant,
disconnected_inputs='ignore'),
sequences=arange(len(expr)),
non_sequences=[expr])
assert not updates, \
("Scan has returned a list of updates. This should not "
"happen! Report this to theano-users (also include the "
"script that generated the error)")
return format_as(using_list, using_tuple, stack(hess))
def hessian_(cost, wrt, consider_constant=None,
disconnected_inputs='raise'):
"""
:type cost: Scalar (0-dimensional) Variable.
:type wrt: Vector (1-dimensional tensor) 'Variable' or list of
vectors (1-dimensional tensors) Variables
:param consider_constant: a list of expressions not to backpropagate
through
:type disconnected_inputs: string
:param disconnected_inputs: Defines the behaviour if some of the variables
in ``wrt`` are not part of the computational graph computing ``cost``
(or if all links are non-differentiable). The possible values are:
- 'ignore': considers that the gradient on these parameters is zero.
- 'warn': consider the gradient zero, and print a warning.
- 'raise': raise an exception.
:return: either a instance of Variable or list/tuple of Variables
(depending upon `wrt`) repressenting the Hessian of the `cost`
with respect to (elements of) `wrt`. If an element of `wrt` is not
differentiable with respect to the output, then a zero
variable is returned. The return value is of same type
as `wrt`: a list/tuple or TensorVariable in all cases.
"""
from theano.tensor import arange
# Check inputs have the right format
assert isinstance(cost, Variable), \
"tensor.hessian expects a Variable as `cost`"
assert cost.ndim == 0, \
"tensor.hessian expects a 0 dimensional variable as `cost`"
using_list = isinstance(wrt, list)
using_tuple = isinstance(wrt, tuple)
if isinstance(wrt, (list, tuple)):
wrt = list(wrt)
else:
wrt = [wrt]
expr = grad(cost, wrt, consider_constant=consider_constant,
disconnected_inputs=disconnected_inputs)
# It is possible that the inputs are disconnected from expr,
# even if they are connected to cost.
# This should not be an error.
hess, updates = scan(lambda i, y: grad(
y[i],
wrt,
consider_constant=consider_constant,
disconnected_inputs='ignore'),
sequences=arange(len(expr)),
non_sequences=[expr])
assert not updates, \
("Scan has returned a list of updates. This should not "
"happen! Report this to theano-users (also include the "
"script that generated the error)")
return format_as(using_list, using_tuple, stack(hess)[0])
def flatten_hessian(cost, wrt, consider_constant=None, disconnected_inputs="raise", block_diagonal=True):
"""
:type cost: Scalar (0-dimensional) Variable.
:type wrt: Vector (1-dimensional tensor) 'Variable' or list of
vectors (1-dimensional tensors) Variables
:param consider_constant: a list of expressions not to backpropagate
through
:type disconnected_inputs: string
:param disconnected_inputs: Defines the behaviour if some of the variables
in ``wrt`` are not part of the computational graph computing ``cost``
(or if all links are non-differentiable). The possible values are:
- 'ignore': considers that the gradient on these parameters is zero.
- 'warn': consider the gradient zero, and print a warning.
- 'raise': raise an exception.
:return: either a instance of Variable or list/tuple of Variables
(depending upon `wrt`) repressenting the Hessian of the `cost`
with respect to (elements of) `wrt`. If an element of `wrt` is not
differentiable with respect to the output, then a zero
variable is returned. The return value is of same type
as `wrt`: a list/tuple or TensorVariable in all cases.
"""
import theano
from theano.tensor import arange
# Check inputs have the right format
import theano.tensor as TT
from theano import Variable
from theano import grad
assert isinstance(cost, Variable), "tensor.hessian expects a Variable as `cost`"
assert cost.ndim == 0, "tensor.hessian expects a 0 dimensional variable as `cost`"
using_list = isinstance(wrt, list)
using_tuple = isinstance(wrt, tuple)
if isinstance(wrt, (list, tuple)):
wrt = list(wrt)
else:
wrt = [wrt]
hessians = []
if not block_diagonal:
expr = TT.concatenate(
[
grad(
cost, input, consider_constant=consider_constant, disconnected_inputs=disconnected_inputs
).flatten()
for input in wrt
]
)
for input in wrt:
assert isinstance(input, Variable), "tensor.hessian expects a (list of) Variable as `wrt`"
# assert input.ndim == 1, \
# "tensor.hessian expects a (list of) 1 dimensional variable " \
# "as `wrt`"
if block_diagonal:
expr = grad(
cost, input, consider_constant=consider_constant, disconnected_inputs=disconnected_inputs
).flatten()
# It is possible that the inputs are disconnected from expr,
# even if they are connected to cost.
# This should not be an error.
hess, updates = theano.scan(
lambda i, y, x: grad(y[i], x, consider_constant=consider_constant, disconnected_inputs="ignore").flatten(),
sequences=arange(expr.shape[0]),
non_sequences=[expr, input],
)
assert not updates, (
"Scan has returned a list of updates. This should not "
"happen! Report this to theano-users (also include the "
"script that generated the error)"
)
hessians.append(hess)
if block_diagonal:
from theano.gradient import format_as
return format_as(using_list, using_tuple, hessians)
else:
return TT.concatenate(hessians, axis=1)
def flatten_hessian(cost,
wrt,
consider_constant=None,
disconnected_inputs='raise',
block_diagonal=True):
"""Flatten hessian.
:type cost: Scalar (0-dimensional) Variable.
:type wrt: Vector (1-dimensional tensor) 'Variable' or list of
vectors (1-dimensional tensors) Variables
:param consider_constant: a list of expressions not to backpropagate
through
:type disconnected_inputs: string
:param disconnected_inputs: Defines the behaviour if some of the variables
in ``wrt`` are not part of the computational graph computing ``cost``
(or if all links are non-differentiable). The possible values are:
- 'ignore': considers that the gradient on these parameters is zero.
- 'warn': consider the gradient zero, and print a warning.
- 'raise': raise an exception.
:return: either a instance of Variable or list/tuple of Variables
(depending upon `wrt`) repressenting the Hessian of the `cost`
with respect to (elements of) `wrt`. If an element of `wrt` is not
differentiable with respect to the output, then a zero
variable is returned. The return value is of same type
as `wrt`: a list/tuple or TensorVariable in all cases.
"""
assert isinstance(cost, Variable), \
"tensor.hessian expects a Variable as `cost`"
assert cost.ndim == 0, \
"tensor.hessian expects a 0 dimensional variable as `cost`"
using_list = isinstance(wrt, list)
using_tuple = isinstance(wrt, tuple)
if isinstance(wrt, (list, tuple)):
wrt = list(wrt)
else:
wrt = [wrt]
hessians = []
if not block_diagonal:
expr = TT.concatenate([
grad(cost,
input,
consider_constant=consider_constant,
disconnected_inputs=disconnected_inputs).flatten()
for input in wrt
])
for input in wrt:
assert isinstance(input, Variable), \
"tensor.hessian expects a (list of) Variable as `wrt`"
# assert input.ndim == 1, \
# "tensor.hessian expects a (list of) 1 dimensional variable " \
# "as `wrt`"
if block_diagonal:
expr = grad(cost,
input,
consider_constant=consider_constant,
disconnected_inputs=disconnected_inputs).flatten()
# It is possible that the inputs are disconnected from expr,
# even if they are connected to cost.
# This should not be an error.
hess, updates = theano.scan(
lambda i, y, x: grad(y[i],
x,
consider_constant=consider_constant,
disconnected_inputs='ignore').flatten(),
sequences=arange(expr.shape[0]),
non_sequences=[expr, input])
assert not updates, \
("Scan has returned a list of updates. This should not "
"happen! Report this to theano-users (also include the "
"script that generated the error)")
hessians.append(hess)
if block_diagonal:
return format_as(using_list, using_tuple, hessians)
else:
return TT.concatenate(hessians, axis=1)
