def setUp(self):
n = self.n = 15
self.X = X = tf.Variable(tf.zeros([n], dtype=float64))
self.cost = tf.exp(tf.reduce_sum(X**2))
Y = self.Y = rnd.randn(n) * 1e-3
A = self.A = rnd.randn(n) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y**2))
self.correct_grad = 2 * Y * np.exp(np.sum(Y**2))
# ... and hess
# First form hessian matrix H
# Convert Y and A into matrices (column vectors)
Ymat = np.matrix(Y)
Amat = np.matrix(A)
diag = np.eye(n)
H = np.exp(np.sum(Y**2)) * (4 * Ymat.T.dot(Ymat) + 2 * diag)
# Then 'right multiply' H by A
self.correct_hess = np.array(Amat.dot(H)).squeeze()
self.backend = TensorflowBackend()
def setUp(self):
n1 = self.n1 = 3
n2 = self.n2 = 4
n3 = self.n3 = 5
self.X = X = tf.Variable(tf.zeros([n1, n2, n3]))
self.cost = tf.exp(tf.reduce_sum(X**2))
Y = self.Y = rnd.randn(n1, n2, n3).astype(float32) * 1e-3
A = self.A = rnd.randn(n1, n2, n3).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y**2))
self.correct_grad = 2 * Y * np.exp(np.sum(Y**2))
# ... and hess
# First form hessian tensor H (6th order)
Y1 = Y.reshape(n1, n2, n3, 1, 1, 1)
Y2 = Y.reshape(1, 1, 1, n1, n2, n3)
# Create an n1 x n2 x n3 x n1 x n2 x n3 diagonal tensor
diag = np.eye(n1 * n2 * n3).reshape(n1, n2, n3, n1, n2, n3)
H = np.exp(np.sum(Y**2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = A.reshape(1, 1, 1, n1, n2, n3)
self.correct_hess = np.sum(H * Atensor, axis=(3, 4, 5))
self.backend = TensorflowBackend()
def setUp(self):
m = self.m = 10
n = self.n = 15
self.X = X = tf.Variable(tf.zeros([m, n]))
self.cost = tf.exp(tf.reduce_sum(X**2))
Y = self.Y = rnd.randn(m, n).astype(float32) * 1e-3
A = self.A = rnd.randn(m, n).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y**2))
self.correct_grad = 2 * Y * np.exp(np.sum(Y**2))
# ... and hess
# First form hessian tensor H (4th order)
Y1 = Y.reshape(m, n, 1, 1)
Y2 = Y.reshape(1, 1, m, n)
# Create an m x n x m x n array with diag[i,j,k,l] == 1 iff
# (i == k and j == l), this is a 'diagonal' tensor.
diag = np.eye(m * n).reshape(m, n, m, n)
H = np.exp(np.sum(Y**2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = A.reshape(1, 1, m, n)
self.correct_hess = np.sum(H * Atensor, axis=(2, 3))
self.backend = TensorflowBackend()
class TestTensor3(unittest.TestCase):
def setUp(self):
n1 = self.n1 = 3
n2 = self.n2 = 4
n3 = self.n3 = 5
self.X = X = tf.Variable(tf.zeros([n1, n2, n3]))
self.cost = tf.exp(tf.reduce_sum(X**2))
Y = self.Y = rnd.randn(n1, n2, n3).astype(float32) * 1e-3
A = self.A = rnd.randn(n1, n2, n3).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y**2))
self.correct_grad = 2 * Y * np.exp(np.sum(Y**2))
# ... and hess
# First form hessian tensor H (6th order)
Y1 = Y.reshape(n1, n2, n3, 1, 1, 1)
Y2 = Y.reshape(1, 1, 1, n1, n2, n3)
# Create an n1 x n2 x n3 x n1 x n2 x n3 diagonal tensor
diag = np.eye(n1 * n2 * n3).reshape(n1, n2, n3, n1, n2, n3)
H = np.exp(np.sum(Y**2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = A.reshape(1, 1, 1, n1, n2, n3)
self.correct_hess = np.sum(H * Atensor, axis=(3, 4, 5))
self.backend = TensorflowBackend()
def test_compile(self):
cost_compiled = self.backend.compile_function(self.cost, self.X)
np_testing.assert_allclose(self.correct_cost,
cost_compiled(self.Y),
rtol=1e-4)
def test_grad(self):
grad = self.backend.compute_gradient(self.cost, self.X)
np_testing.assert_allclose(self.correct_grad, grad(self.Y), rtol=1e-4)
def test_hessian(self):
hess = self.backend.compute_hessian(self.cost, self.X)
# Now test hess
np_testing.assert_allclose(self.correct_hess,
hess(self.Y, self.A),
rtol=1e-4)
class TestMatrix(unittest.TestCase):
def setUp(self):
m = self.m = 10
n = self.n = 15
self.X = X = tf.Variable(tf.zeros([m, n]))
self.cost = tf.exp(tf.reduce_sum(X**2))
Y = self.Y = rnd.randn(m, n).astype(float32) * 1e-3
A = self.A = rnd.randn(m, n).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y**2))
self.correct_grad = 2 * Y * np.exp(np.sum(Y**2))
# ... and hess
# First form hessian tensor H (4th order)
Y1 = Y.reshape(m, n, 1, 1)
Y2 = Y.reshape(1, 1, m, n)
# Create an m x n x m x n array with diag[i,j,k,l] == 1 iff
# (i == k and j == l), this is a 'diagonal' tensor.
diag = np.eye(m * n).reshape(m, n, m, n)
H = np.exp(np.sum(Y**2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = A.reshape(1, 1, m, n)
self.correct_hess = np.sum(H * Atensor, axis=(2, 3))
self.backend = TensorflowBackend()
def test_compile(self):
cost_compiled = self.backend.compile_function(self.cost, self.X)
np_testing.assert_allclose(self.correct_cost,
cost_compiled(self.Y),
rtol=1e-4)
def test_grad(self):
grad = self.backend.compute_gradient(self.cost, self.X)
np_testing.assert_allclose(self.correct_grad, grad(self.Y), rtol=1e-4)
def test_hessian(self):
hess = self.backend.compute_hessian(self.cost, self.X)
# Now test hess
np_testing.assert_allclose(self.correct_hess,
hess(self.Y, self.A),
rtol=1e-4)
class TestTensor3(unittest.TestCase):
def setUp(self):
self.X = X = tf.Variable(tf.zeros([0]))
self.cost = tf.exp(tf.reduce_sum(X**2))
n1 = self.n1 = 3
n2 = self.n2 = 4
n3 = self.n3 = 5
Y = self.Y = rnd.randn(n1, n2, n3).astype(float32) * 1e-3
A = self.A = rnd.randn(n1, n2, n3).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y ** 2))
self.correct_grad = correct_grad = 2 * Y * np.exp(np.sum(Y ** 2))
# ... and hess
# First form hessian tensor H (6th order)
Y1 = Y.reshape(n1, n2, n3, 1, 1, 1)
Y2 = Y.reshape(1, 1, 1, n1, n2, n3)
# Create an n1 x n2 x n3 x n1 x n2 x n3 diagonal tensor
diag = np.eye(n1 * n2 * n3).reshape(n1, n2, n3, n1, n2, n3)
H = np.exp(np.sum(Y ** 2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = A.reshape(1, 1, 1, n1, n2, n3)
self.correct_hess = np.sum(H * Atensor, axis=(3, 4, 5))
self.backend = TensorflowBackend()
def test_compile(self):
cost_compiled = self.backend.compile_function(self.cost, self.X)
np_testing.assert_allclose(self.correct_cost, cost_compiled(self.Y),
rtol=1e-4)
def test_grad(self):
grad = self.backend.compute_gradient(self.cost, self.X)
np_testing.assert_allclose(self.correct_grad, grad(self.Y), rtol=1e-4)
def test_hessian(self):
hess = self.backend.compute_hessian(self.cost, self.X)
# Now test hess
np_testing.assert_allclose(self.correct_hess, hess(self.Y, self.A),
rtol=1e-4)
class TestMatrix(unittest.TestCase):
def setUp(self):
self.X = X = tf.Variable(tf.zeros([0]))
self.cost = tf.exp(tf.reduce_sum(X**2))
m = self.m = 10
n = self.n = 15
Y = self.Y = rnd.randn(m, n).astype(float32) * 1e-3
A = self.A = rnd.randn(m, n).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y ** 2))
self.correct_grad = correct_grad = 2 * Y * np.exp(np.sum(Y ** 2))
# ... and hess
# First form hessian tensor H (4th order)
Y1 = Y.reshape(m, n, 1, 1)
Y2 = Y.reshape(1, 1, m, n)
# Create an m x n x m x n array with diag[i,j,k,l] == 1 iff
# (i == k and j == l), this is a 'diagonal' tensor.
diag = np.eye(m * n).reshape(m, n, m, n)
H = np.exp(np.sum(Y ** 2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = A.reshape(1, 1, m, n)
self.correct_hess = np.sum(H * Atensor, axis=(2, 3))
self.backend = TensorflowBackend()
def test_compile(self):
cost_compiled = self.backend.compile_function(self.cost, self.X)
np_testing.assert_allclose(self.correct_cost, cost_compiled(self.Y),
rtol=1e-4)
def test_grad(self):
grad = self.backend.compute_gradient(self.cost, self.X)
np_testing.assert_allclose(self.correct_grad, grad(self.Y),
rtol=1e-4)
def test_hessian(self):
hess = self.backend.compute_hessian(self.cost, self.X)
# Now test hess
np_testing.assert_allclose(self.correct_hess, hess(self.Y, self.A),
rtol=1e-4)
def setUp(self):
self.X = X = tf.Variable(tf.zeros([0]))
self.cost = tf.exp(tf.reduce_sum(X**2))
m = self.m = 10
n = self.n = 15
Y = self.Y = rnd.randn(m, n).astype(float32) * 1e-3
A = self.A = rnd.randn(m, n).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y ** 2))
self.correct_grad = correct_grad = 2 * Y * np.exp(np.sum(Y ** 2))
# ... and hess
# First form hessian tensor H (4th order)
Y1 = Y.reshape(m, n, 1, 1)
Y2 = Y.reshape(1, 1, m, n)
# Create an m x n x m x n array with diag[i,j,k,l] == 1 iff
# (i == k and j == l), this is a 'diagonal' tensor.
diag = np.eye(m * n).reshape(m, n, m, n)
H = np.exp(np.sum(Y ** 2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = A.reshape(1, 1, m, n)
self.correct_hess = np.sum(H * Atensor, axis=(2, 3))
self.backend = TensorflowBackend()
def setUp(self):
self.X = X = tf.Variable(tf.zeros([0]))
self.cost = tf.exp(tf.reduce_sum(X**2))
n = self.n = 15
Y = self.Y = rnd.randn(n).astype(float32) * 1e-3
A = self.A = rnd.randn(n).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y ** 2))
self.correct_grad = correct_grad = 2 * Y * np.exp(np.sum(Y ** 2))
# ... and hess
# First form hessian matrix H
# Convert Y and A into matrices (column vectors)
Ymat = np.matrix(Y)
Amat = np.matrix(A)
diag = np.eye(n)
H = np.exp(np.sum(Y ** 2)) * (4 * Ymat.T.dot(Ymat) + 2 * diag)
# Then 'right multiply' H by A
self.correct_hess = np.array(Amat.dot(H)).squeeze()
self.backend = TensorflowBackend()
def setUp(self):
self.X = X = tf.Variable(tf.zeros([0]))
self.cost = tf.exp(tf.reduce_sum(X**2))
n1 = self.n1 = 3
n2 = self.n2 = 4
n3 = self.n3 = 5
Y = self.Y = rnd.randn(n1, n2, n3).astype(float32) * 1e-3
A = self.A = rnd.randn(n1, n2, n3).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y ** 2))
self.correct_grad = correct_grad = 2 * Y * np.exp(np.sum(Y ** 2))
# ... and hess
# First form hessian tensor H (6th order)
Y1 = Y.reshape(n1, n2, n3, 1, 1, 1)
Y2 = Y.reshape(1, 1, 1, n1, n2, n3)
# Create an n1 x n2 x n3 x n1 x n2 x n3 diagonal tensor
diag = np.eye(n1 * n2 * n3).reshape(n1, n2, n3, n1, n2, n3)
H = np.exp(np.sum(Y ** 2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = A.reshape(1, 1, 1, n1, n2, n3)
self.correct_hess = np.sum(H * Atensor, axis=(3, 4, 5))
self.backend = TensorflowBackend()
class TestVector(unittest.TestCase):
def setUp(self):
n = self.n = 15
self.X = X = tf.Variable(tf.zeros([n], dtype=float64))
self.cost = tf.exp(tf.reduce_sum(X**2))
Y = self.Y = rnd.randn(n) * 1e-3
A = self.A = rnd.randn(n) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y**2))
self.correct_grad = 2 * Y * np.exp(np.sum(Y**2))
# ... and hess
# First form hessian matrix H
# Convert Y and A into matrices (column vectors)
Ymat = np.matrix(Y)
Amat = np.matrix(A)
diag = np.eye(n)
H = np.exp(np.sum(Y**2)) * (4 * Ymat.T.dot(Ymat) + 2 * diag)
# Then 'right multiply' H by A
self.correct_hess = np.array(Amat.dot(H)).squeeze()
self.backend = TensorflowBackend()
def test_compile(self):
cost_compiled = self.backend.compile_function(self.cost, self.X)
np_testing.assert_allclose(self.correct_cost,
cost_compiled(self.Y),
rtol=1e-4)
def test_grad(self):
grad = self.backend.compute_gradient(self.cost, self.X)
np_testing.assert_allclose(self.correct_grad, grad(self.Y), rtol=1e-4)
def test_hessian(self):
hess = self.backend.compute_hessian(self.cost, self.X)
# Now test hess
np_testing.assert_allclose(self.correct_hess,
hess(self.Y, self.A),
rtol=1e-4)
class TestVector(unittest.TestCase):
def setUp(self):
self.X = X = tf.Variable(tf.zeros([0]))
self.cost = tf.exp(tf.reduce_sum(X**2))
n = self.n = 15
Y = self.Y = rnd.randn(n).astype(float32) * 1e-3
A = self.A = rnd.randn(n).astype(float32) * 1e-3
# Calculate correct cost and grad...
self.correct_cost = np.exp(np.sum(Y ** 2))
self.correct_grad = correct_grad = 2 * Y * np.exp(np.sum(Y ** 2))
# ... and hess
# First form hessian matrix H
# Convert Y and A into matrices (column vectors)
Ymat = np.matrix(Y)
Amat = np.matrix(A)
diag = np.eye(n)
H = np.exp(np.sum(Y ** 2)) * (4 * Ymat.T.dot(Ymat) + 2 * diag)
# Then 'right multiply' H by A
self.correct_hess = np.array(Amat.dot(H)).squeeze()
self.backend = TensorflowBackend()
def test_compile(self):
cost_compiled = self.backend.compile_function(self.cost, self.X)
np_testing.assert_allclose(self.correct_cost, cost_compiled(self.Y),
rtol=1e-4)
def test_grad(self):
grad = self.backend.compute_gradient(self.cost, self.X)
np_testing.assert_allclose(self.correct_grad, grad(self.Y), rtol=1e-4)
def test_hessian(self):
hess = self.backend.compute_hessian(self.cost, self.X)
# Now test hess
np_testing.assert_allclose(self.correct_hess, hess(self.Y, self.A),
rtol=1e-4)
def __init__(self,
manifold,
cost,
accuracy=None,
cost_dropout=None,
accuracy_dropout=None,
train_summary=None,
test_summary=None,
egrad=None,
ehess=None,
grad=None,
hess=None,
arg=None,
data=[],
precon=None,
verbosity=2,
logdir=None):
self.manifold = manifold
# We keep a reference to the original cost function in case we want to
# call the `prepare` method twice (for instance, after switching from
# a first- to second-order method).
self._cost = None
self._original_cost = cost
self._train_scalars_and_summary = None
self._original_train_scalars_and_summary = \
[cost, accuracy, cost_dropout, accuracy_dropout, train_summary]
self._test_scalars_and_summary = None
self._original_test_scalars_and_summary = \
[cost, accuracy, cost_dropout, accuracy_dropout, test_summary]
self._argument = None
self._egrad = egrad
self._ehess = ehess
self._grad = grad
self._hess = hess
self._arg = arg if isinstance(arg, list) else [arg]
self._data = data
self._backend = None
self._logdir = logdir
if precon is None:
def precon(x, d):
return d
self.precon = precon
self.verbosity = verbosity
self._backends = filter(
lambda b: b.is_available(),
[TheanoBackend(),
AutogradBackend(),
TensorflowBackend()])
self._backend = None
def __init__(self,
manifold,
cost,
egrad=None,
ehess=None,
grad=None,
hess=None,
arg=None,
precon=None,
verbosity=2):
self.manifold = manifold
# We keep a reference to the original cost function in case we want to
# call the `prepare` method twice (for instance, after switching from
# a first- to second-order method).
self._cost = None
self._original_cost = cost
self._egrad = egrad
self._ehess = ehess
self._grad = grad
self._hess = hess
self._arg = arg
self._backend = None
if precon is None:
def precon(x, d):
return d
self.precon = precon
self.verbosity = verbosity
self._backends = list(
filter(lambda b: b.is_available(), [
TheanoBackend(),
PytorchBackend(),
AutogradBackend(),
TensorflowBackend()
]))
self._backend = None
def setUp(self):
x = tf.Variable(tf.zeros([0]))
y = tf.Variable(tf.zeros([0]))
z = tf.Variable(tf.zeros([0]))
f = (tf.exp(tf.reduce_sum(x**2)) + tf.exp(tf.reduce_sum(y**2)) +
tf.exp(tf.reduce_sum(z**2)))
self.cost = f
self.arg = [x, y, z]
n1 = self.n1 = 3
n2 = self.n2 = 4
n3 = self.n3 = 5
n4 = self.n4 = 6
n5 = self.n5 = 7
n6 = self.n6 = 8
self.y = y = (rnd.randn(n1).astype(float32) * 1e-3,
rnd.randn(n2, n3).astype(float32) * 1e-3,
rnd.randn(n4, n5, n6).astype(float32) * 1e-3)
self.a = a = (rnd.randn(n1).astype(float32) * 1e-3,
rnd.randn(n2, n3).astype(float32) * 1e-3,
rnd.randn(n4, n5, n6).astype(float32) * 1e-3)
self.correct_cost = (np.exp(np.sum(y[0]**2)) +
np.exp(np.sum(y[1]**2)) +
np.exp(np.sum(y[2]**2)))
# CALCULATE CORRECT GRAD
g1 = 2 * y[0] * np.exp(np.sum(y[0] ** 2))
g2 = 2 * y[1] * np.exp(np.sum(y[1] ** 2))
g3 = 2 * y[2] * np.exp(np.sum(y[2] ** 2))
self.correct_grad = (g1, g2, g3)
# CALCULATE CORRECT HESS
# 1. VECTOR
Ymat = np.matrix(y[0])
Amat = np.matrix(a[0])
diag = np.eye(n1)
H = np.exp(np.sum(y[0] ** 2)) * (4 * Ymat.T.dot(Ymat) + 2 * diag)
# Then 'left multiply' H by A
h1 = np.array(Amat.dot(H)).flatten()
# 2. MATRIX
# First form hessian tensor H (4th order)
Y1 = y[1].reshape(n2, n3, 1, 1)
Y2 = y[1].reshape(1, 1, n2, n3)
# Create an m x n x m x n array with diag[i,j,k,l] == 1 iff
# (i == k and j == l), this is a 'diagonal' tensor.
diag = np.eye(n2 * n3).reshape(n2, n3, n2, n3)
H = np.exp(np.sum(y[1] ** 2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = a[1].reshape(1, 1, n2, n3)
h2 = np.sum(H * Atensor, axis=(2, 3))
# 3. Tensor3
# First form hessian tensor H (6th order)
Y1 = y[2].reshape(n4, n5, n6, 1, 1, 1)
Y2 = y[2].reshape(1, 1, 1, n4, n5, n6)
# Create an n1 x n2 x n3 x n1 x n2 x n3 diagonal tensor
diag = np.eye(n4 * n5 * n6).reshape(n4, n5, n6, n4, n5, n6)
H = np.exp(np.sum(y[2] ** 2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = a[2].reshape(1, 1, 1, n4, n5, n6)
h3 = np.sum(H * Atensor, axis=(3, 4, 5))
self.correct_hess = (h1, h2, h3)
self.backend = TensorflowBackend()
class TestMixed(unittest.TestCase):
# Test autograd on a tuple containing vector, matrix and tensor3.
def setUp(self):
x = tf.Variable(tf.zeros([0]))
y = tf.Variable(tf.zeros([0]))
z = tf.Variable(tf.zeros([0]))
f = (tf.exp(tf.reduce_sum(x**2)) + tf.exp(tf.reduce_sum(y**2)) +
tf.exp(tf.reduce_sum(z**2)))
self.cost = f
self.arg = [x, y, z]
n1 = self.n1 = 3
n2 = self.n2 = 4
n3 = self.n3 = 5
n4 = self.n4 = 6
n5 = self.n5 = 7
n6 = self.n6 = 8
self.y = y = (rnd.randn(n1).astype(float32) * 1e-3,
rnd.randn(n2, n3).astype(float32) * 1e-3,
rnd.randn(n4, n5, n6).astype(float32) * 1e-3)
self.a = a = (rnd.randn(n1).astype(float32) * 1e-3,
rnd.randn(n2, n3).astype(float32) * 1e-3,
rnd.randn(n4, n5, n6).astype(float32) * 1e-3)
self.correct_cost = (np.exp(np.sum(y[0]**2)) +
np.exp(np.sum(y[1]**2)) +
np.exp(np.sum(y[2]**2)))
# CALCULATE CORRECT GRAD
g1 = 2 * y[0] * np.exp(np.sum(y[0] ** 2))
g2 = 2 * y[1] * np.exp(np.sum(y[1] ** 2))
g3 = 2 * y[2] * np.exp(np.sum(y[2] ** 2))
self.correct_grad = (g1, g2, g3)
# CALCULATE CORRECT HESS
# 1. VECTOR
Ymat = np.matrix(y[0])
Amat = np.matrix(a[0])
diag = np.eye(n1)
H = np.exp(np.sum(y[0] ** 2)) * (4 * Ymat.T.dot(Ymat) + 2 * diag)
# Then 'left multiply' H by A
h1 = np.array(Amat.dot(H)).flatten()
# 2. MATRIX
# First form hessian tensor H (4th order)
Y1 = y[1].reshape(n2, n3, 1, 1)
Y2 = y[1].reshape(1, 1, n2, n3)
# Create an m x n x m x n array with diag[i,j,k,l] == 1 iff
# (i == k and j == l), this is a 'diagonal' tensor.
diag = np.eye(n2 * n3).reshape(n2, n3, n2, n3)
H = np.exp(np.sum(y[1] ** 2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = a[1].reshape(1, 1, n2, n3)
h2 = np.sum(H * Atensor, axis=(2, 3))
# 3. Tensor3
# First form hessian tensor H (6th order)
Y1 = y[2].reshape(n4, n5, n6, 1, 1, 1)
Y2 = y[2].reshape(1, 1, 1, n4, n5, n6)
# Create an n1 x n2 x n3 x n1 x n2 x n3 diagonal tensor
diag = np.eye(n4 * n5 * n6).reshape(n4, n5, n6, n4, n5, n6)
H = np.exp(np.sum(y[2] ** 2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = a[2].reshape(1, 1, 1, n4, n5, n6)
h3 = np.sum(H * Atensor, axis=(3, 4, 5))
self.correct_hess = (h1, h2, h3)
self.backend = TensorflowBackend()
def test_compile(self):
cost_compiled = self.backend.compile_function(self.cost, self.arg)
np_testing.assert_allclose(self.correct_cost, cost_compiled(self.y))
def test_grad(self):
grad = self.backend.compute_gradient(self.cost, self.arg)
for k in range(len(grad(self.y))):
np_testing.assert_allclose(self.correct_grad[k], grad(self.y)[k],
rtol=1e-4)
def test_hessian(self):
hess = self.backend.compute_hessian(self.cost, self.arg)
# Now test hess
for k in range(len(hess(self.y, self.a))):
np_testing.assert_allclose(self.correct_hess[k],
hess(self.y, self.a)[k], rtol=1e-4)
class TestMixed(unittest.TestCase):
# Test autograd on a tuple containing vector, matrix and tensor3.
def setUp(self):
n1 = self.n1 = 3
n2 = self.n2 = 4
n3 = self.n3 = 5
n4 = self.n4 = 6
n5 = self.n5 = 7
n6 = self.n6 = 8
x = tf.Variable(tf.zeros([n1]))
y = tf.Variable(tf.zeros([n2, n3]))
z = tf.Variable(tf.zeros([n4, n5, n6]))
f = (tf.exp(tf.reduce_sum(x**2)) + tf.exp(tf.reduce_sum(y**2)) +
tf.exp(tf.reduce_sum(z**2)))
self.cost = f
self.arg = [x, y, z]
self.y = y = (rnd.randn(n1).astype(float32) * 1e-3,
rnd.randn(n2, n3).astype(float32) * 1e-3,
rnd.randn(n4, n5, n6).astype(float32) * 1e-3)
self.a = a = (rnd.randn(n1).astype(float32) * 1e-3,
rnd.randn(n2, n3).astype(float32) * 1e-3,
rnd.randn(n4, n5, n6).astype(float32) * 1e-3)
self.correct_cost = (np.exp(np.sum(y[0]**2)) +
np.exp(np.sum(y[1]**2)) + np.exp(np.sum(y[2]**2)))
# CALCULATE CORRECT GRAD
g1 = 2 * y[0] * np.exp(np.sum(y[0]**2))
g2 = 2 * y[1] * np.exp(np.sum(y[1]**2))
g3 = 2 * y[2] * np.exp(np.sum(y[2]**2))
self.correct_grad = (g1, g2, g3)
# CALCULATE CORRECT HESS
# 1. VECTOR
Ymat = np.matrix(y[0])
Amat = np.matrix(a[0])
diag = np.eye(n1)
H = np.exp(np.sum(y[0]**2)) * (4 * Ymat.T.dot(Ymat) + 2 * diag)
# Then 'left multiply' H by A
h1 = np.array(Amat.dot(H)).flatten()
# 2. MATRIX
# First form hessian tensor H (4th order)
Y1 = y[1].reshape(n2, n3, 1, 1)
Y2 = y[1].reshape(1, 1, n2, n3)
# Create an m x n x m x n array with diag[i,j,k,l] == 1 iff
# (i == k and j == l), this is a 'diagonal' tensor.
diag = np.eye(n2 * n3).reshape(n2, n3, n2, n3)
H = np.exp(np.sum(y[1]**2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = a[1].reshape(1, 1, n2, n3)
h2 = np.sum(H * Atensor, axis=(2, 3))
# 3. Tensor3
# First form hessian tensor H (6th order)
Y1 = y[2].reshape(n4, n5, n6, 1, 1, 1)
Y2 = y[2].reshape(1, 1, 1, n4, n5, n6)
# Create an n1 x n2 x n3 x n1 x n2 x n3 diagonal tensor
diag = np.eye(n4 * n5 * n6).reshape(n4, n5, n6, n4, n5, n6)
H = np.exp(np.sum(y[2]**2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = a[2].reshape(1, 1, 1, n4, n5, n6)
h3 = np.sum(H * Atensor, axis=(3, 4, 5))
self.correct_hess = (h1, h2, h3)
self.backend = TensorflowBackend()
def test_compile(self):
cost_compiled = self.backend.compile_function(self.cost, self.arg)
np_testing.assert_allclose(self.correct_cost, cost_compiled(self.y))
def test_grad(self):
grad = self.backend.compute_gradient(self.cost, self.arg)
for k in range(len(grad(self.y))):
np_testing.assert_allclose(self.correct_grad[k],
grad(self.y)[k],
rtol=1e-4)
def test_hessian(self):
hess = self.backend.compute_hessian(self.cost, self.arg)
# Now test hess
for k in range(len(hess(self.y, self.a))):
np_testing.assert_allclose(self.correct_hess[k],
hess(self.y, self.a)[k],
rtol=1e-4)
def setUp(self):
n1 = self.n1 = 3
n2 = self.n2 = 4
n3 = self.n3 = 5
n4 = self.n4 = 6
n5 = self.n5 = 7
n6 = self.n6 = 8
x = tf.Variable(tf.zeros([n1]))
y = tf.Variable(tf.zeros([n2, n3]))
z = tf.Variable(tf.zeros([n4, n5, n6]))
f = (tf.exp(tf.reduce_sum(x**2)) + tf.exp(tf.reduce_sum(y**2)) +
tf.exp(tf.reduce_sum(z**2)))
self.cost = f
self.arg = [x, y, z]
self.y = y = (rnd.randn(n1).astype(float32) * 1e-3,
rnd.randn(n2, n3).astype(float32) * 1e-3,
rnd.randn(n4, n5, n6).astype(float32) * 1e-3)
self.a = a = (rnd.randn(n1).astype(float32) * 1e-3,
rnd.randn(n2, n3).astype(float32) * 1e-3,
rnd.randn(n4, n5, n6).astype(float32) * 1e-3)
self.correct_cost = (np.exp(np.sum(y[0]**2)) +
np.exp(np.sum(y[1]**2)) + np.exp(np.sum(y[2]**2)))
# CALCULATE CORRECT GRAD
g1 = 2 * y[0] * np.exp(np.sum(y[0]**2))
g2 = 2 * y[1] * np.exp(np.sum(y[1]**2))
g3 = 2 * y[2] * np.exp(np.sum(y[2]**2))
self.correct_grad = (g1, g2, g3)
# CALCULATE CORRECT HESS
# 1. VECTOR
Ymat = np.matrix(y[0])
Amat = np.matrix(a[0])
diag = np.eye(n1)
H = np.exp(np.sum(y[0]**2)) * (4 * Ymat.T.dot(Ymat) + 2 * diag)
# Then 'left multiply' H by A
h1 = np.array(Amat.dot(H)).flatten()
# 2. MATRIX
# First form hessian tensor H (4th order)
Y1 = y[1].reshape(n2, n3, 1, 1)
Y2 = y[1].reshape(1, 1, n2, n3)
# Create an m x n x m x n array with diag[i,j,k,l] == 1 iff
# (i == k and j == l), this is a 'diagonal' tensor.
diag = np.eye(n2 * n3).reshape(n2, n3, n2, n3)
H = np.exp(np.sum(y[1]**2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = a[1].reshape(1, 1, n2, n3)
h2 = np.sum(H * Atensor, axis=(2, 3))
# 3. Tensor3
# First form hessian tensor H (6th order)
Y1 = y[2].reshape(n4, n5, n6, 1, 1, 1)
Y2 = y[2].reshape(1, 1, 1, n4, n5, n6)
# Create an n1 x n2 x n3 x n1 x n2 x n3 diagonal tensor
diag = np.eye(n4 * n5 * n6).reshape(n4, n5, n6, n4, n5, n6)
H = np.exp(np.sum(y[2]**2)) * (4 * Y1 * Y2 + 2 * diag)
# Then 'right multiply' H by A
Atensor = a[2].reshape(1, 1, 1, n4, n5, n6)
h3 = np.sum(H * Atensor, axis=(3, 4, 5))
self.correct_hess = (h1, h2, h3)
self.backend = TensorflowBackend()