def test_Sequential(self): np.random.seed(42) torch.manual_seed(42) batch_size, n_in = 2, 4 for _ in range(100): # layers initialization alpha = 0.9 torch_layer = torch.nn.BatchNorm1d(n_in, eps=BatchNormalization.EPS, momentum=1. - alpha, affine=True) torch_layer.bias.data = torch.from_numpy( np.random.random(n_in).astype(np.float32)) custom_layer = Sequential() bn_layer = BatchNormalization(alpha) bn_layer.moving_mean = torch_layer.running_mean.numpy().copy() bn_layer.moving_variance = torch_layer.running_var.numpy().copy() custom_layer.add(bn_layer) scaling_layer = ChannelwiseScaling(n_in) scaling_layer.gamma = torch_layer.weight.data.numpy() scaling_layer.beta = torch_layer.bias.data.numpy() custom_layer.add(scaling_layer) custom_layer.train() layer_input = np.random.uniform(-5, 5, (batch_size, n_in)).astype( np.float32) next_layer_grad = np.random.uniform( -5, 5, (batch_size, n_in)).astype(np.float32) # 1. check layer output custom_layer_output = custom_layer.updateOutput(layer_input) layer_input_var = Variable(torch.from_numpy(layer_input), requires_grad=True) torch_layer_output_var = torch_layer(layer_input_var) self.assertTrue( np.allclose(torch_layer_output_var.data.numpy(), custom_layer_output, atol=1e-6)) # 2. check layer input grad custom_layer_grad = custom_layer.backward(layer_input, next_layer_grad) torch_layer_output_var.backward(torch.from_numpy(next_layer_grad)) torch_layer_grad_var = layer_input_var.grad self.assertTrue( np.allclose(torch_layer_grad_var.data.numpy(), custom_layer_grad, atol=1e-5)) # 3. check layer parameters grad weight_grad, bias_grad = custom_layer.getGradParameters()[1] torch_weight_grad = torch_layer.weight.grad.data.numpy() torch_bias_grad = torch_layer.bias.grad.data.numpy() self.assertTrue( np.allclose(torch_weight_grad, weight_grad, atol=1e-6)) self.assertTrue(np.allclose(torch_bias_grad, bias_grad, atol=1e-6))
loss_history = [] acc_history = [] for i in range(n_epoch): # print(f"EPOCH: {i}") for x_batch, y_batch in get_batches(train_x, train_y, batch_size): net.zeroGradParameters() # Forward predictions = net.forward(x_batch) loss = criterion.forward(predictions, y_batch) # Backward dp = criterion.backward(predictions, y_batch) net.backward(x_batch, dp) # Update weights adam_optimizer(net.getParameters(), net.getGradParameters(), optimizer_config, optimizer_state) loss_history.append(loss) acc = 100 * accuracy_score(predictions.argmax(axis=-1), y_batch.argmax(axis=-1)) acc_history.append(acc) print(f"EPOCH: {i} loss: {loss} accuracy: {acc}") ################################ #### Visualize training statistics plt.figure(figsize=(12, 5))
# ----- Implementation of the architecture ----- architecture = Sequential(Linear(2, 25, ReLU()), Linear(25, 25, ReLU()), Linear(25, 25, ReLU()), Linear(25, 2, Sigma())) # ----- Training ----- round = 1 prev_loss = math.inf prev_prev_loss = math.inf errors = [] for epoch in range(epochs): for batch_start in range(0, nb_samples, batch_size): features = train_features[batch_start:batch_start + batch_size, :] labels = train_labels[batch_start:batch_start + batch_size] tr_loss, tr_error = architecture.forward(train_features, train_labels) architecture.backward() architecture.update(eta) loss, error = architecture.forward(test_features, test_labels) print(' --- Epoch ', round, ' Test Loss: ', loss.item(), '---', ' Errors: ', error) if prev_loss < tr_loss: eta *= 0.999 if prev_prev_loss < tr_loss: eta *= 0.998 errors.append(error) prev_prev_loss = prev_loss prev_loss = tr_loss round += 1