def __init__(self, *args):
self.n_unit = len(args)
super().__init__()
for i in range(self.n_unit - 1):
self.parameter[f"w_encode{i}"] = nn.asarray(np.random.randn(args[i], args[i + 1]))
self.parameter[f"b_encode{i}"] = nn.asarray(np.zeros(args[i + 1]))
self.parameter[f"w_decode{i}"] = nn.asarray(np.random.randn(args[i + 1], args[i]))
self.parameter[f"b_decode{i}"] = nn.asarray(np.zeros(args[i]))
def test_add_scalar(self):
npa = np.random.randn(5, 6)
npb = 2
a = nn.asarray(npa)
b = nn.asarray(npb)
c = a + b
self.assertTrue(np.allclose(c.value, npa + npb))
npg = np.random.randn(5, 6)
c.backward(npg)
self.assertTrue(np.allclose(a.grad, npg))
self.assertTrue(np.allclose(b.grad, np.sum(npg)))
def test_matmul(self):
npa = np.random.randn(4, 6)
npb = np.random.randn(6, 3)
a = nn.asarray(npa)
b = nn.asarray(npb)
c = a @ b
self.assertTrue(np.allclose(c.value, npa @ npb))
npg = np.random.randn(4, 3)
c.backward(npg)
self.assertTrue(np.allclose(a.grad, npg @ npb.T))
self.assertTrue(np.allclose(b.grad, npa.T @ npg))
def test_add(self):
npa = np.random.randn(4, 5)
npb = np.random.randn(4, 5)
a = nn.asarray(npa)
b = nn.asarray(npb)
c = a + b
self.assertTrue(np.allclose(c.value, npa + npb))
npg = np.random.randn(4, 5)
c.backward(npg)
self.assertTrue(np.allclose(a.grad, npg))
self.assertTrue(np.allclose(b.grad, npg))
def test_multiply(self):
npx = np.random.randn(5, 6)
npy = np.random.randn(5, 6)
x = nn.asarray(npx)
y = nn.asarray(npy)
z = x * y
self.assertTrue(np.allclose(z.value, npx * npy))
npg = np.random.randn(5, 6)
z.backward(npg)
self.assertTrue(np.allclose(x.grad, npg * npy))
self.assertTrue(np.allclose(y.grad, npg * npx))
def test_add_bias(self):
npa = np.random.randn(4, 3)
npb = np.random.randn(3)
a = nn.asarray(npa)
b = nn.asarray(npb)
c = a + b
self.assertTrue(np.allclose(c.value, npa + npb))
npg = np.random.randn(4, 3)
c.backward(npg)
self.assertTrue(np.allclose(a.grad, npg))
self.assertTrue(np.allclose(b.grad, npg.sum(axis=0)))
def test_sigmoid_cross_entropy(self):
npx = np.random.randn(10, 3)
npy = np.tanh(npx * 0.5) * 0.5 + 0.5
npt = np.random.uniform(0, 1, (10, 3))
x = nn.asarray(npx)
t = nn.asarray(npt)
loss = nn.loss.sigmoid_cross_entropy(x, t)
self.assertTrue(
np.allclose(loss.value,
-npt * np.log(npy) - (1 - npt) * np.log(1 - npy)))
npg = np.random.randn(10, 3)
loss.backward(npg)
self.assertTrue(np.allclose(x.grad, npg * (npy - npt)))
self.assertTrue(np.allclose(t.grad, -npg * npx))
def test_tanh(self):
npx = np.random.randn(4, 7)
x = nn.asarray(npx)
y = nn.tanh(x)
self.assertTrue(np.allclose(y.value, np.tanh(npx)))
npg = np.random.randn(4, 7)
y.backward(npg)
self.assertTrue(np.allclose(x.grad, npg * (1 - y.value**2)))
def test_negative(self):
npx = np.random.randn(8, 9)
x = nn.asarray(npx)
y = -x
self.assertTrue(np.allclose(y.value, -npx))
npg = np.random.randn(8, 9)
y.backward(npg)
self.assertTrue(np.allclose(x.grad, -npg))
def test_log(self):
npx = np.random.uniform(0, 10, (4, 5))
x = nn.asarray(npx)
y = nn.log(x)
self.assertTrue(np.allclose(y.value, np.log(npx)))
npg = np.random.randn(4, 5)
y.backward(npg)
self.assertTrue(np.allclose(x.grad, npg / npx))
def test_backward(self):
npx = np.random.randn(1, 5)
x = nn.asarray(npx)
nn.softmax(x).backward()
grad1 = np.copy(x.grad)
x.cleargrad()
nn.exp(nn.log_softmax(x)).backward()
grad2 = np.copy(x.grad)
self.assertTrue(np.allclose(grad1, grad2))
def test_sigmoid(self):
npx = np.random.randn(3, 5)
x = nn.asarray(npx)
y = nn.sigmoid(x)
self.assertTrue(np.allclose(y.value, np.tanh(npx * 0.5) * 0.5 + 0.5))
npg = np.random.randn(3, 5)
y.backward(npg)
self.assertTrue(np.allclose(x.grad, npg * y.value * (1 - y.value)))
def test_max_pooling2d(self):
img = np.array([
[2, 3, 4, 1],
[2, 5, 1, 2],
[3, 5, 1, 3],
[3, 7, 8, 2]
]).astype(np.float)
img = img[None, :, :, None]
expected = np.array([[5, 4], [7, 8]])
actual = nn.max_pooling2d(img, 2, 2).value.squeeze()
self.assertTrue((expected == actual).all(), actual)
expected = np.array([
[0, 0, 1, 0],
[0, 1, 0, 0],
[0, 0, 0, 0],
[0, 1, 1, 0]
])
img = nn.asarray(img)
nn.max_pooling2d(img, 2, 2).backward(np.ones((1, 2, 2, 1)))
actual = img.grad.squeeze()
self.assertTrue((expected == actual).all())