def bottle_neck(self, hidden_state, lstm_cells, x_data, train):
x = chainer.Variable(x_data, volatile=not train)
a1 = self.l1_a(F.concat((x, hidden_state["h1"])))
x1 = self.l1_x(F.concat((x, hidden_state["h1"], lstm_cells["c1"])))
c1, h1 = peephole_lstm(lstm_cells["c1"], a1, x1)
h1 = gradient_clip(h1, 10.0)
a2 = self.l2_a(F.concat((x, hidden_state["h2"], h1)))
x2 = self.l2_x(F.concat((x, hidden_state["h2"], h1, lstm_cells["c2"])))
c2, h2 = peephole_lstm(lstm_cells["c2"], a2, x2)
h2 = gradient_clip(h2, 10.0)
a3 = self.l2_a(F.concat((x, hidden_state["h3"], h2)))
x3 = self.l2_x(F.concat((x, hidden_state["h3"], h2, lstm_cells["c3"])))
c3, h3 = peephole_lstm(lstm_cells["c3"], a3, x3)
h3 = gradient_clip(h3, 10.0)
y = self.l4(F.concat((h1, h2, h3)))
y = gradient_clip(y, 100.0)
n = int((y.data.shape[1] - 1) / 6)
gi, b = gauss_bernoulli_params(n, y)
hidden_state = {"h1": h1, "h2": h2, "h3": h3}
if train:
lstm_cells = {"c1": c1, "c2": c2, "c3": c3}
return gi, b, hidden_state, lstm_cells
def check_backward(self, c_prev_data, a_data, x_data, c_grad, h_grad):
c_prev = chainer.Variable(c_prev_data)
a = chainer.Variable(a_data)
x = chainer.Variable(x_data)
c, h = peephole_lstm(c_prev, a, x)
c.grad = c_grad
h.grad = h_grad
c.backward()
func = c.creator
f = lambda: func.forward((c_prev.data, a.data, x.data))
gc_prev, ga, gx = gradient_check.numerical_grad(f, (c_prev.data, a.data, x.data), (c_grad, h_grad), eps=1e-2)
gradient_check.assert_allclose(gc_prev, c_prev.grad)
gradient_check.assert_allclose(ga, a.grad)
gradient_check.assert_allclose(gx, x.grad)
def check_forward(self, c_prev_data, a_data, x_data):
c_prev = chainer.Variable(c_prev_data)
a = chainer.Variable(a_data)
x = chainer.Variable(x_data)
c, h = peephole_lstm(c_prev, a, x)
self.assertEqual(c.data.dtype, numpy.float32)
self.assertEqual(h.data.dtype, numpy.float32)
# Compute expected out
a_in = self.a
i_in = self.x[:, [0, 3]]
f_in = self.x[:, [1, 4]]
o_in = self.x[:, [2, 5]]
c_expect = _sigmoid(i_in) * numpy.tanh(a_in) + _sigmoid(f_in) * self.c_prev
h_expect = _sigmoid(o_in) * numpy.tanh(c_expect)
gradient_check.assert_allclose(c_expect, c.data)
gradient_check.assert_allclose(h_expect, h.data)
