def test_forward_case_2(self):
layer = TanhLayer()
layer.initialize_parameters()
x = np.array([[-3.0, 4.0]])
actual = layer.forward(x)
x_torch = self.numpy_to_torch(x)
expect_torch = torch.tanh(x_torch)
expect = self.torch_to_numpy(expect_torch)
self.assertEquals(expect.shape, actual.shape)
self.assertClose(expect, actual)
def simple_nn(images, labels, test_images, test_labels):
learning_rate = 0.1
batch_size = 100
num_batches = int(labels.shape[0] / batch_size)
epochs = 10
model = Model()
model.add_layers([
LinearLayer(28 * 28, 256),
TanhLayer(),
LinearLayer(256, 10),
SoftmaxWithLossLayer(),
])
model.initialize_params(StandardNormalInitializer(0.01))
optimizer = Adam()
for epoch in range(epochs):
print("Epoch {}".format(epoch))
for batch in range(num_batches):
batch_mask = np.random.choice(labels.shape[0], batch_size)
X_train = images[batch_mask, :]
y_train = labels[batch_mask, :]
# gradient
grads = model.gradient(X_train, y_train)
# calculate loss
if batch % 100 == 0:
loss = model.loss(X_train, y_train)
print("Batch {}: Loss = {}".format(batch, loss))
# update
model.update_paramters(optimizer)
# predict
a2 = model.predict(test_images)
print(
np.sum(np.argmax(a2, axis=1) == np.argmax(test_labels, axis=1)) /
test_labels.shape[0])
def test_backward_cast_2(self):
layer = TanhLayer()
layer.initialize_parameters()
x = np.array([[-3.0, 4.0]])
y = layer.forward(x)
dy = np.ones(y.shape)
dx_actual = layer.backward(dy)
x_torch = self.numpy_to_torch(x, requires_grad=True)
y_torch = torch.tanh(x_torch)
dy_torch = torch.ones(y_torch.shape)
y_torch.backward(gradient=dy_torch)
dx_torch = x_torch.grad
dx_expect = self.torch_to_numpy(dx_torch)
self.assertEquals(dx_actual.shape, dx_expect.shape)
self.assertClose(dx_expect, dx_actual)