def test_Sequential(self): np.random.seed(42) torch.manual_seed(42) batch_size, n_in = 2, 4 for _ in range(100): # layers initialization alpha = 0.9 torch_layer = torch.nn.BatchNorm1d(n_in, eps=BatchNormalization.EPS, momentum=1. - alpha, affine=True) torch_layer.bias.data = torch.from_numpy( np.random.random(n_in).astype(np.float32)) custom_layer = Sequential() bn_layer = BatchNormalization(alpha) bn_layer.moving_mean = torch_layer.running_mean.numpy().copy() bn_layer.moving_variance = torch_layer.running_var.numpy().copy() custom_layer.add(bn_layer) scaling_layer = ChannelwiseScaling(n_in) scaling_layer.gamma = torch_layer.weight.data.numpy() scaling_layer.beta = torch_layer.bias.data.numpy() custom_layer.add(scaling_layer) custom_layer.train() layer_input = np.random.uniform(-5, 5, (batch_size, n_in)).astype( np.float32) next_layer_grad = np.random.uniform( -5, 5, (batch_size, n_in)).astype(np.float32) # 1. check layer output custom_layer_output = custom_layer.updateOutput(layer_input) layer_input_var = Variable(torch.from_numpy(layer_input), requires_grad=True) torch_layer_output_var = torch_layer(layer_input_var) self.assertTrue( np.allclose(torch_layer_output_var.data.numpy(), custom_layer_output, atol=1e-6)) # 2. check layer input grad custom_layer_grad = custom_layer.backward(layer_input, next_layer_grad) torch_layer_output_var.backward(torch.from_numpy(next_layer_grad)) torch_layer_grad_var = layer_input_var.grad self.assertTrue( np.allclose(torch_layer_grad_var.data.numpy(), custom_layer_grad, atol=1e-5)) # 3. check layer parameters grad weight_grad, bias_grad = custom_layer.getGradParameters()[1] torch_weight_grad = torch_layer.weight.grad.data.numpy() torch_bias_grad = torch_layer.bias.grad.data.numpy() self.assertTrue( np.allclose(torch_weight_grad, weight_grad, atol=1e-6)) self.assertTrue(np.allclose(torch_bias_grad, bias_grad, atol=1e-6))
for i in range(n_epoch): # print(f"EPOCH: {i}") for x_batch, y_batch in get_batches(train_x, train_y, batch_size): net.zeroGradParameters() # Forward predictions = net.forward(x_batch) loss = criterion.forward(predictions, y_batch) # Backward dp = criterion.backward(predictions, y_batch) net.backward(x_batch, dp) # Update weights adam_optimizer(net.getParameters(), net.getGradParameters(), optimizer_config, optimizer_state) loss_history.append(loss) acc = 100 * accuracy_score(predictions.argmax(axis=-1), y_batch.argmax(axis=-1)) acc_history.append(acc) print(f"EPOCH: {i} loss: {loss} accuracy: {acc}") ################################ #### Visualize training statistics plt.figure(figsize=(12, 5)) plt.subplot(121) plt.title("Training loss") plt.xlabel("#iteration")
for i in range(n_epoch): print(f"EPOCH: {i}") for x_batch, y_batch in get_batches(X, Y, batch_size): net.zeroGradParameters() # Forward predictions = net.forward(x_batch) loss = criterion.forward(predictions, y_batch) # Backward dp = criterion.backward(predictions, y_batch) net.backward(x_batch, dp) # Update weights sgd_momentum(net.getParameters(), net.getGradParameters(), optimizer_config, optimizer_state) loss_history.append(loss) acc_history.append(100 * accuracy_score(predictions.argmax(axis=-1), y_batch.argmax(axis=-1))) ################################ #### Visualize training statistics plt.figure(figsize=(12, 5)) plt.subplot(121) plt.title("Training loss") plt.xlabel("#iteration") plt.ylabel("loss") plt.plot(loss_history, 'b')