def test_Sequential(self):
np.random.seed(42)
torch.manual_seed(42)
batch_size, n_in = 2, 4
for _ in range(100):
# layers initialization
alpha = 0.9
torch_layer = torch.nn.BatchNorm1d(n_in,
eps=BatchNormalization.EPS,
momentum=1. - alpha,
affine=True)
torch_layer.bias.data = torch.from_numpy(
np.random.random(n_in).astype(np.float32))
custom_layer = Sequential()
bn_layer = BatchNormalization(alpha)
bn_layer.moving_mean = torch_layer.running_mean.numpy().copy()
bn_layer.moving_variance = torch_layer.running_var.numpy().copy()
custom_layer.add(bn_layer)
scaling_layer = ChannelwiseScaling(n_in)
scaling_layer.gamma = torch_layer.weight.data.numpy()
scaling_layer.beta = torch_layer.bias.data.numpy()
custom_layer.add(scaling_layer)
custom_layer.train()
layer_input = np.random.uniform(-5, 5, (batch_size, n_in)).astype(
np.float32)
next_layer_grad = np.random.uniform(
-5, 5, (batch_size, n_in)).astype(np.float32)
# 1. check layer output
custom_layer_output = custom_layer.updateOutput(layer_input)
layer_input_var = Variable(torch.from_numpy(layer_input),
requires_grad=True)
torch_layer_output_var = torch_layer(layer_input_var)
self.assertTrue(
np.allclose(torch_layer_output_var.data.numpy(),
custom_layer_output,
atol=1e-6))
# 2. check layer input grad
custom_layer_grad = custom_layer.backward(layer_input,
next_layer_grad)
torch_layer_output_var.backward(torch.from_numpy(next_layer_grad))
torch_layer_grad_var = layer_input_var.grad
self.assertTrue(
np.allclose(torch_layer_grad_var.data.numpy(),
custom_layer_grad,
atol=1e-5))
# 3. check layer parameters grad
weight_grad, bias_grad = custom_layer.getGradParameters()[1]
torch_weight_grad = torch_layer.weight.grad.data.numpy()
torch_bias_grad = torch_layer.bias.grad.data.numpy()
self.assertTrue(
np.allclose(torch_weight_grad, weight_grad, atol=1e-6))
self.assertTrue(np.allclose(torch_bias_grad, bias_grad, atol=1e-6))
for i in range(n_epoch):
# print(f"EPOCH: {i}")
for x_batch, y_batch in get_batches(train_x, train_y, batch_size):
net.zeroGradParameters()
# Forward
predictions = net.forward(x_batch)
loss = criterion.forward(predictions, y_batch)
# Backward
dp = criterion.backward(predictions, y_batch)
net.backward(x_batch, dp)
# Update weights
adam_optimizer(net.getParameters(), net.getGradParameters(),
optimizer_config, optimizer_state)
loss_history.append(loss)
acc = 100 * accuracy_score(predictions.argmax(axis=-1),
y_batch.argmax(axis=-1))
acc_history.append(acc)
print(f"EPOCH: {i} loss: {loss} accuracy: {acc}")
################################
#### Visualize training statistics
plt.figure(figsize=(12, 5))
plt.subplot(121)
plt.title("Training loss")
plt.xlabel("#iteration")
for i in range(n_epoch):
print(f"EPOCH: {i}")
for x_batch, y_batch in get_batches(X, Y, batch_size):
net.zeroGradParameters()
# Forward
predictions = net.forward(x_batch)
loss = criterion.forward(predictions, y_batch)
# Backward
dp = criterion.backward(predictions, y_batch)
net.backward(x_batch, dp)
# Update weights
sgd_momentum(net.getParameters(), net.getGradParameters(),
optimizer_config, optimizer_state)
loss_history.append(loss)
acc_history.append(100 * accuracy_score(predictions.argmax(axis=-1),
y_batch.argmax(axis=-1)))
################################
#### Visualize training statistics
plt.figure(figsize=(12, 5))
plt.subplot(121)
plt.title("Training loss")
plt.xlabel("#iteration")
plt.ylabel("loss")
plt.plot(loss_history, 'b')