def test_mse_loss(self): """Testing class :class:`lib.backprop_functions.MSELossFunction`.""" # Simple test of the forward method only. a = torch.from_numpy(np.array([[2., 3.]])) t = torch.from_numpy(np.array([[1., 1.]])) target_mse = 0.5 * (1**2 + 2**2) # 2.5 our_mse = bf.mse_loss(a, t).detach().numpy() self.assertAlmostEqual(our_mse, target_mse, 5, 'MSE value not correctly computed.') # Compare forward and backward method to PyTorch its implementation. mse_ours = bf.mse_loss(self.A2, self.T) mse_torch = self._pytorch_mse(self.A2, self.T) self.assertAlmostEqual(mse_ours.detach().numpy(), mse_torch.detach().numpy(), 5, 'MSE value not correctly computed.') # Check if gradient computation is correct (i.e., backward path) if self.A2.grad is not None: self.A2.grad.zero_() mse_ours.backward() our_grad_A = self.A2.grad.clone() self.A2.grad.zero_() mse_torch.backward() torch_grad_A = self.A2.grad.clone() grad_error = torch.sum((our_grad_A - torch_grad_A)**2).detach().numpy() self.assertAlmostEqual( grad_error, 0., 5, 'MSE loss gradients not ' + 'correctly computed.')
def train(args, device, train_loader, net): """Train the given network on the given (regression) dataset. Args: args (argparse.Namespace): The command-line arguments. device: The PyTorch device to be used. train_loader (torch.utils.data.DataLoader): The data handler for training data. net: The (student) neural network. """ print('Training network ...') net.train() optimizer = torch.optim.SGD(net.parameters(), lr=args.lr, momentum=args.momentum) for e in range(args.epochs): for i, (inputs, targets) in enumerate(train_loader): inputs, targets = inputs.to(device), targets.to(device) predictions = net.forward(inputs) optimizer.zero_grad() loss = bf.mse_loss(predictions, targets) loss.backward() optimizer.step() if (e + 1) % 10 == 0: print('Epoch {} -- loss = {}.'.format(e + 1, loss)) print('Training network ... Done')
def train(args, device, train_loader, net): """Train the given network on the given (regression) dataset. Args: args (argparse.Namespace): The command-line arguments. device: The PyTorch device to be used. train_loader (torch.utils.data.DataLoader): The data handler for training data. net: The (student) neural network. """ print('Training network ...') net.train() optimizer = torch.optim.SGD(net.parameters(), lr=args.lr, momentum=args.momentum) if args.plot_matrix_angles: angles_transpose = torch.empty(args.epochs, net.depth, requires_grad=False) angles_pinv = torch.empty(args.epochs, net.depth, requires_grad=False) angles_contr = torch.empty(args.epochs, net.depth, requires_grad=False) for e in range(args.epochs): for i, (inputs, targets) in enumerate(train_loader): inputs, targets = inputs.to(device), targets.to(device) predictions = net.forward(inputs) optimizer.zero_grad() loss = bf.mse_loss(predictions, targets) loss.backward() optimizer.step() # Save the feedback alignment angles if args.plot_matrix_angles: for i in range(net.depth): W = net.linear_layers[i + 1].weights B = net.linear_layers[i + 1].feedbackweights W_pinv = torch.pinverse(W) angles_transpose[e, i] = utils.compute_matrix_angle(W.t(), B) angles_pinv[e, i] = utils.compute_matrix_angle(W_pinv, B) angles_contr[e, i] = utils.compute_matrix_angle(W_pinv, W.t()) if (e + 1) % 10 == 0: print('Epoch {} -- loss = {}.'.format(e + 1, loss)) print('Training network ... Done') if args.plot_matrix_angles: print('Plotting matrix angles ...') utils.plot_angles(angles_transpose, r'angle between $W_i^T$ and $B$', r'$B_{%s} \angle W_{%s}^{T} [\circ]$') utils.plot_angles(angles_pinv, r'angles between $W_i^{\dagger}$ and $B$', r'$B_{%s} \angle W_{%s}^{\dagger} [\circ]$')
def test(device, test_loader, net): """test a train network by computing the MSE on the test set. Args: (....): See docstring of function :func:`train`. test_loader (torch.utils.data.DataLoader): The data handler for test data. Returns: (float): The mean-squared error for the test set ``test_loader`` when using the network ``net``. Note, the ``Function`` :func:`lib.backprop_functions.mse_loss` is used to compute the MSE value. """ ####################################################################### ### NOTE, the function `mse_loss` divides by the current batch size. In ### order to compute the MSE across several mini-batches, one needs to ### correct for this behavior. ####################################################################### net.eval() with torch.no_grad(): num_samples = 0 mse = 0. for i, (inputs, targets) in enumerate(test_loader): inputs, targets = inputs.to(device), targets.to(device) batch_size = int(inputs.shape[0]) num_samples += batch_size predictions = net.forward(inputs) mse += batch_size * bf.mse_loss(predictions, targets) mse /= num_samples print('Test MSE: {}'.format(mse)) return float(mse.cpu().detach().numpy())
def test_mse_loss_targets(self): """Testing if gradients of targets in class :class:`lib.backprop_functions.MSELossFunction` are correctly implemented. """ # Ensure that the gradients with respect to the targets are also # correctly computed. mse_ours = bf.mse_loss(self.T, self.A2) mse_torch = self._pytorch_mse(self.T, self.A2) if self.A2.grad is not None: self.A2.grad.zero_() mse_ours.backward() our_grad_A = self.A2.grad.clone() self.A2.grad.zero_() mse_torch.backward() torch_grad_A = self.A2.grad.clone() grad_error = torch.sum((our_grad_A - torch_grad_A) ** 2).detach(). \ numpy() self.assertAlmostEqual( grad_error, 0., 5, 'MSE loss gradients for targets not correctly computed.')