def forward(self, x):
if self.h is None:
h_new = F.tanh(self.x2h(x))
else:
h_new = F.tanh(self.x2h(x) + self.h2h(self.h))
self.h = h_new
return h_new
def __call__(self, x):
if self.h is None:
h_new = F.tanh(self.x2h(x))
else:
h_new = F.tanh(self.x2h(x) + self.h2h(self.h))
self.h = h_new
return h_new
def __call__(self, x):
if self.h is None:
N, D = x.shape
H, H = self.h2f.W.shape
self.h = np.zeros((N, H), np.float32)
self.c = np.zeros((N, H), np.float32)
f = F.sigmoid(self.x2f(x) + self.h2f(self.h))
i = F.sigmoid(self.x2i(x) + self.h2i(self.h))
o = F.sigmoid(self.x2o(x) + self.h2o(self.h))
u = F.tanh(self.x2u(x) + self.h2u(self.h))
c = (f * self.c) + (i * u)
h = o * F.tanh(c)
self.h, self.c = h, c
return h
def test_tanh_backward_twice(self):
x = Variable(np.array(1.0))
y = F.tanh(x)
y.backward(create_graph=True)
gx = x.grad
x.cleargrad()
gx.backward(create_graph=False)
result = x.grad.data
assert np.allclose(result, -0.63970001)
def forward(self, x):
if self.h is None:
f = F.sigmoid(self.x2f(x))
i = F.sigmoid(self.x2i(x))
o = F.sigmoid(self.x2o(x))
u = F.tanh(self.x2u(x))
else:
f = F.sigmoid(self.x2f(x) + self.h2f(self.h))
i = F.sigmoid(self.x2i(x) + self.h2i(self.h))
o = F.sigmoid(self.x2o(x) + self.h2o(self.h))
u = F.tanh(self.x2u(x) + self.h2u(self.h))
if self.c is None:
c_new = (i * u)
else:
c_new = (f * self.c) + (i * u)
h_new = o * F.tanh(c_new)
self.h, self.c = h_new, c_new
return h_new
def __call__(self, x):
if self.h is None:
f = F.sigmoid(self.x2f(x))
i = F.sigmoid(self.x2i(x))
o = F.sigmoid(self.x2o(x))
u = F.tanh(self.x2u(x))
else:
f = F.sigmoid(self.x2f(x) + self.h2f(self.h))
i = F.sigmoid(self.x2i(x) + self.h2i(self.h))
o = F.sigmoid(self.x2o(x) + self.h2o(self.h))
u = F.tanh(self.x2u(x) + self.h2u(self.h))
if self.c is None:
c = (i * u)
else:
c = (f * self.c) + (i * u)
h = o * F.tanh(c)
self.h, self.c = h, c
return h
def test_tanh(self):
x = Variable(np.array(1.0))
y = F.tanh(x)
x.name = 'x'
y.name = 'y'
y.backward(create_graph=True)
iters = 0
for i in range(iters):
gx = x.grad
x.cleargrad()
gx.backward(create_graph=True)
gx = x.grad
gx.name = 'gx' + str(iters + 1)
txt = get_dot_graph(gx)
with open('test.dot', 'w') as f:
f.write(txt)
if '__file__' in globals():
import os
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import numpy as np
from dezero.utils import plot_dot_graph
from dezero import Variable
import dezero.functions as F
x = Variable(np.array(np.array(1.0)))
y = F.tanh(x)
x.name = 'x'
y.name = 'y'
y.backward(create_graph=True)
iters = 5
for i in range(iters):
gx = x.grad
x.cleargrad()
gx.backward(create_graph=True)
gx = x.grad
gx.name = 'gx' + str(iters + 1)
plot_dot_graph(gx, verbose=False, to_file='../save/tanh.png')
import numpy as np
from dezero import Variable
from dezero.utils import plot_dot_graph
import dezero.functions as f
iterss = 0
if __name__ == '__main__':
x = Variable(np.array(1))
y = f.tanh(x)
x.name = 'x'
y.name = 'y'
y.backward(create_graph=True)
for iters in [0, 1, 2, 3, 4, 5, 6]:
for i in range(iters):
gx: Variable = x.grad
x.clear_grad()
gx.backward(create_graph=True)
gx: Variable = x.grad
gx.name = f'gx{iters + 1}'
plot_dot_graph(gx, verbose=False, to_file=f'tanh_diff_{iters + 1}.png')
def test_backward(self):
x = Variable(np.array(0.5))
y = tanh(x)
y.backward()
self.assertAlmostEqual(x.grad.data, np.array(0.7864477330213905))
def test_forward(self):
x = Variable(np.array(0.5))
y = tanh(x)
self.assertAlmostEqual(y.data, np.array(0.4621171572))
def test_tanh_backward_once(self):
x = Variable(np.array(np.pi / 4))
y = F.tanh(x)
y.backward()
result = x.grad.data
assert np.allclose(result, 0.56993396)
def test_tanh_forward(self):
x = Variable(np.array(np.pi / 4))
y = F.tanh(x)
assert np.allclose(y.data, 0.6557942)