def test_linearcg():
vv = theano.shared(v, name='v')
gg = theano.shared(g, name='g')
hh = theano.shared(h, name='h')
dw = T.dot(v.T,g) / M
dv = T.dot(g.T,h) / M
da = T.mean(v, axis=0)
db = T.mean(g, axis=0)
dc = T.mean(h, axis=0)
newgrads = lincg.linear_cg(
lambda xw, xv, xa, xb, xc: natural.compute_Lx(vv,gg,hh,xw,xv,xa,xb,xc),
[dw, dv, da, db, dc],
rtol=1e-5,
maxiter = 30,
damp = 0.,
floatX = floatX,
profile=0)
f = theano.function([], newgrads)
[new_dw, new_dv, new_da, new_db, new_dc] = f()
numpy.testing.assert_almost_equal(Linv_x_w, new_dw, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_v, new_dv, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_a, new_da, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_b, new_db, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_c, new_dc, decimal=1)
def test_minres_with_xinit():
rng = numpy.random.RandomState(123412)
vv = theano.shared(v, name='v')
gg = theano.shared(g, name='g')
hh = theano.shared(h, name='h')
dw = T.dot(v.T,g) / M
dv = T.dot(g.T,h) / M
da = T.mean(v, axis=0)
db = T.mean(g, axis=0)
dc = T.mean(h, axis=0)
xinit = [ rng.rand(N0,N1),
rng.rand(N1,N2),
rng.rand(N0),
rng.rand(N1),
rng.rand(N2)]
xinit = [xi.astype(floatX) for xi in xinit]
newgrads = minres.minres(
lambda xw, xv, xa, xb, xc: natural.compute_Lx(vv,gg,hh,xw,xv,xa,xb,xc),
[dw, dv, da, db, dc],
rtol=1e-5,
damp = 0.,
maxiter = 10000,
xinit = xinit,
profile=0)[0]
f = theano.function([], newgrads)
[new_dw, new_dv, new_da, new_db, new_dc] = f()
numpy.testing.assert_almost_equal(Linv_x_w, new_dw, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_v, new_dv, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_a, new_da, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_b, new_db, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_c, new_dc, decimal=1)
def test_minres_with_jacobi():
vv = theano.shared(v, name='v')
gg = theano.shared(g, name='g')
hh = theano.shared(h, name='h')
dw = T.dot(v.T,g) / M
dv = T.dot(g.T,h) / M
da = T.mean(v, axis=0)
db = T.mean(g, axis=0)
dc = T.mean(h, axis=0)
Ldiag_terms = natural.generic_compute_L_diag([vv,gg,hh])
Ms = [Ldiag_term + 0.1 for Ldiag_term in Ldiag_terms]
newgrads = minres.minres(
lambda xw, xv, xa, xb, xc: natural.compute_Lx(vv,gg,hh,xw,xv,xa,xb,xc),
[dw, dv, da, db, dc],
rtol=1e-5,
damp = 0.,
maxiter = 10000,
Ms = Ms,
profile=0)[0]
f = theano.function([], newgrads)
[new_dw, new_dv, new_da, new_db, new_dc] = f()
numpy.testing.assert_almost_equal(Linv_x_w, new_dw, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_v, new_dv, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_a, new_da, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_b, new_db, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_c, new_dc, decimal=1)
def test_linearcg():
vv = theano.shared(v, name='v')
gg = theano.shared(g, name='g')
hh = theano.shared(h, name='h')
dw = T.dot(v.T, g) / M
dv = T.dot(g.T, h) / M
da = T.mean(v, axis=0)
db = T.mean(g, axis=0)
dc = T.mean(h, axis=0)
newgrads = lincg.linear_cg(lambda xw, xv, xa, xb, xc: natural.compute_Lx(
vv, gg, hh, xw, xv, xa, xb, xc), [dw, dv, da, db, dc],
rtol=1e-5,
maxiter=30,
damp=0.,
floatX=floatX,
profile=0)
f = theano.function([], newgrads)
[new_dw, new_dv, new_da, new_db, new_dc] = f()
numpy.testing.assert_almost_equal(Linv_x_w, new_dw, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_v, new_dv, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_a, new_da, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_b, new_db, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_c, new_dc, decimal=1)
def test_compute_Lx():
## now compare against theano version
vv = T.matrix()
gg = T.matrix()
hh = T.matrix()
aa = T.vector()
bb = T.vector()
cc = T.vector()
xxw_mat = T.matrix()
xxv_mat = T.matrix()
xxw = T.vector()
xxv = T.vector()
xxa = T.vector()
xxb = T.vector()
xxc = T.vector()
# test compute_Lx
LLx = natural.compute_Lx(vv, gg, hh, xxw_mat, xxv_mat, xxa, xxb, xxc)
t1 = time.time()
f = theano.function([vv, gg, hh, xxw_mat, xxv_mat, xxa, xxb, xxc], LLx)
print 'Elapsed: ', time.time() - t1
rvals = f(v, g, h, xw_mat, xv_mat, xa, xb, xc)
numpy.testing.assert_almost_equal(Lx_w, rvals[0], decimal=3)
numpy.testing.assert_almost_equal(Lx_v, rvals[1], decimal=3)
numpy.testing.assert_almost_equal(Lx_a, rvals[2], decimal=3)
numpy.testing.assert_almost_equal(Lx_b, rvals[3], decimal=3)
numpy.testing.assert_almost_equal(Lx_c, rvals[4], decimal=3)
def test_minres_with_jacobi():
vv = theano.shared(v, name='v')
gg = theano.shared(g, name='g')
hh = theano.shared(h, name='h')
dw = T.dot(v.T, g) / M
dv = T.dot(g.T, h) / M
da = T.mean(v, axis=0)
db = T.mean(g, axis=0)
dc = T.mean(h, axis=0)
Ldiag_terms = natural.generic_compute_L_diag([vv, gg, hh])
Ms = [Ldiag_term + 0.1 for Ldiag_term in Ldiag_terms]
newgrads = minres.minres(lambda xw, xv, xa, xb, xc: natural.compute_Lx(
vv, gg, hh, xw, xv, xa, xb, xc), [dw, dv, da, db, dc],
rtol=1e-5,
damp=0.,
maxiter=10000,
Ms=Ms,
profile=0)[0]
f = theano.function([], newgrads)
[new_dw, new_dv, new_da, new_db, new_dc] = f()
numpy.testing.assert_almost_equal(Linv_x_w, new_dw, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_v, new_dv, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_a, new_da, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_b, new_db, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_c, new_dc, decimal=1)
def test_compute_Lx():
## now compare against theano version
vv = T.matrix()
gg = T.matrix()
hh = T.matrix()
aa = T.vector()
bb = T.vector()
cc = T.vector()
xxw_mat = T.matrix()
xxv_mat = T.matrix()
xxw = T.vector()
xxv = T.vector()
xxa = T.vector()
xxb = T.vector()
xxc = T.vector()
# test compute_Lx
LLx = natural.compute_Lx(vv, gg, hh, xxw_mat, xxv_mat, xxa, xxb, xxc)
t1 = time.time()
f = theano.function([vv, gg, hh, xxw_mat, xxv_mat, xxa, xxb, xxc], LLx)
print 'Elapsed: ', time.time() - t1
rvals = f(v, g, h, xw_mat, xv_mat, xa, xb, xc)
numpy.testing.assert_almost_equal(Lx_w, rvals[0], decimal=3)
numpy.testing.assert_almost_equal(Lx_v, rvals[1], decimal=3)
numpy.testing.assert_almost_equal(Lx_a, rvals[2], decimal=3)
numpy.testing.assert_almost_equal(Lx_b, rvals[3], decimal=3)
numpy.testing.assert_almost_equal(Lx_c, rvals[4], decimal=3)
def test_compute_Lx():
## baseline result ##
Lx = numpy.dot(L, vals['x'])
Lx_w = Lx[:N0*N1].reshape(N0,N1)
Lx_v = Lx[N0*N1 : N0*N1 + N1*N2].reshape(N1,N2)
Lx_a = Lx[N0*N1 + N1*N2 : N0*N1 + N1*N2 + N0]
Lx_b = Lx[N0*N1 + N1*N2 + N0 : N0*N1 + N1*N2 + N0 + N1]
Lx_c = Lx[-N2:]
# natural.compute_Lx implementation
symb_inputs = [symb['v'], symb['g'], symb['h'],
symb['x_W'], symb['x_V'],
symb['x_a'], symb['x_b'], symb['x_c']]
Lx = natural.compute_Lx(*symb_inputs)
t1 = time.time()
f = theano.function(symb_inputs, Lx)
print 'natural.compute_Lx elapsed: ', time.time() - t1
rvals = f(vals['v'], vals['g'], vals['h'],
vals['x_W'], vals['x_V'],
vals['x_a'], vals['x_b'], vals['x_c'])
numpy.testing.assert_almost_equal(Lx_w, rvals[0], decimal=3)
numpy.testing.assert_almost_equal(Lx_v, rvals[1], decimal=3)
numpy.testing.assert_almost_equal(Lx_a, rvals[2], decimal=3)
numpy.testing.assert_almost_equal(Lx_b, rvals[3], decimal=3)
numpy.testing.assert_almost_equal(Lx_c, rvals[4], decimal=3)
# fisher.compute_Lx implementation
energies = - T.sum(T.dot(symb['v'], symb['W']) * symb['g'], axis=1) \
- T.sum(T.dot(symb['g'], symb['V']) * symb['h'], axis=1) \
- T.dot(symb['v'], symb['a']) \
- T.dot(symb['g'], symb['b']) \
- T.dot(symb['h'], symb['c'])
symb_params = [symb['W'], symb['V'], symb['a'], symb['b'], symb['c']]
symb_x = [symb['x_W'], symb['x_V'], symb['x_a'], symb['x_b'], symb['x_c']]
LLx = fisher.compute_Lx(energies, symb_params, symb_x)
f_inputs = [symb['v'], symb['g'], symb['h']] + symb_params + symb_x
f = theano.function(f_inputs, LLx)
t1 = time.time()
rvals = f(vals['v'], vals['g'], vals['h'],
vals['W'], vals['V'], vals['a'], vals['b'], vals['c'],
vals['x_W'], vals['x_V'], vals['x_a'], vals['x_b'], vals['x_c'])
### compare both implementation ###
print 'fisher.compute_Lx elapsed: ', time.time() - t1
numpy.testing.assert_almost_equal(Lx_w, rvals[0], decimal=3)
numpy.testing.assert_almost_equal(Lx_v, rvals[1], decimal=3)
numpy.testing.assert_almost_equal(Lx_a, rvals[2], decimal=3)
numpy.testing.assert_almost_equal(Lx_b, rvals[3], decimal=3)
numpy.testing.assert_almost_equal(Lx_c, rvals[4], decimal=3)
def test_compute_Lx():
## baseline result ##
Lx = numpy.dot(L, vals['x'])
Lx_w = Lx[:N0 * N1].reshape(N0, N1)
Lx_v = Lx[N0 * N1:N0 * N1 + N1 * N2].reshape(N1, N2)
Lx_a = Lx[N0 * N1 + N1 * N2:N0 * N1 + N1 * N2 + N0]
Lx_b = Lx[N0 * N1 + N1 * N2 + N0:N0 * N1 + N1 * N2 + N0 + N1]
Lx_c = Lx[-N2:]
# natural.compute_Lx implementation
symb_inputs = [
symb['v'], symb['g'], symb['h'], symb['x_W'], symb['x_V'], symb['x_a'],
symb['x_b'], symb['x_c']
]
Lx = natural.compute_Lx(*symb_inputs)
t1 = time.time()
f = theano.function(symb_inputs, Lx)
print 'natural.compute_Lx elapsed: ', time.time() - t1
rvals = f(vals['v'], vals['g'], vals['h'], vals['x_W'], vals['x_V'],
vals['x_a'], vals['x_b'], vals['x_c'])
numpy.testing.assert_almost_equal(Lx_w, rvals[0], decimal=3)
numpy.testing.assert_almost_equal(Lx_v, rvals[1], decimal=3)
numpy.testing.assert_almost_equal(Lx_a, rvals[2], decimal=3)
numpy.testing.assert_almost_equal(Lx_b, rvals[3], decimal=3)
numpy.testing.assert_almost_equal(Lx_c, rvals[4], decimal=3)
# fisher.compute_Lx implementation
energies = - T.sum(T.dot(symb['v'], symb['W']) * symb['g'], axis=1) \
- T.sum(T.dot(symb['g'], symb['V']) * symb['h'], axis=1) \
- T.dot(symb['v'], symb['a']) \
- T.dot(symb['g'], symb['b']) \
- T.dot(symb['h'], symb['c'])
symb_params = [symb['W'], symb['V'], symb['a'], symb['b'], symb['c']]
symb_x = [symb['x_W'], symb['x_V'], symb['x_a'], symb['x_b'], symb['x_c']]
LLx = fisher.compute_Lx(energies, symb_params, symb_x)
f_inputs = [symb['v'], symb['g'], symb['h']] + symb_params + symb_x
f = theano.function(f_inputs, LLx)
t1 = time.time()
rvals = f(vals['v'], vals['g'], vals['h'], vals['W'], vals['V'], vals['a'],
vals['b'], vals['c'], vals['x_W'], vals['x_V'], vals['x_a'],
vals['x_b'], vals['x_c'])
### compare both implementation ###
print 'fisher.compute_Lx elapsed: ', time.time() - t1
numpy.testing.assert_almost_equal(Lx_w, rvals[0], decimal=3)
numpy.testing.assert_almost_equal(Lx_v, rvals[1], decimal=3)
numpy.testing.assert_almost_equal(Lx_a, rvals[2], decimal=3)
numpy.testing.assert_almost_equal(Lx_b, rvals[3], decimal=3)
numpy.testing.assert_almost_equal(Lx_c, rvals[4], decimal=3)
def test_minres_with_xinit():
rng = numpy.random.RandomState(123412)
vv = theano.shared(v, name='v')
gg = theano.shared(g, name='g')
hh = theano.shared(h, name='h')
dw = T.dot(v.T, g) / M
dv = T.dot(g.T, h) / M
da = T.mean(v, axis=0)
db = T.mean(g, axis=0)
dc = T.mean(h, axis=0)
xinit = [
rng.rand(N0, N1),
rng.rand(N1, N2),
rng.rand(N0),
rng.rand(N1),
rng.rand(N2)
]
xinit = [xi.astype(floatX) for xi in xinit]
newgrads = minres.minres(lambda xw, xv, xa, xb, xc: natural.compute_Lx(
vv, gg, hh, xw, xv, xa, xb, xc), [dw, dv, da, db, dc],
rtol=1e-5,
damp=0.,
maxiter=10000,
xinit=xinit,
profile=0)[0]
f = theano.function([], newgrads)
[new_dw, new_dv, new_da, new_db, new_dc] = f()
numpy.testing.assert_almost_equal(Linv_x_w, new_dw, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_v, new_dv, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_a, new_da, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_b, new_db, decimal=1)
numpy.testing.assert_almost_equal(Linv_x_c, new_dc, decimal=1)