def test_early_stopping3(self): """ The purpose of this test is to ensure that the early stopping is not activated, when the configuration for EarlyStopping is set to None. Even though we modify the val_acc as before, after epoch 4, EarlyStopping is not configured and as a result, we train for all 10 epochs """ optimizer_cfg = DefaultOptimizerConfig() optimizer_cfg.training_cfg.device = torch.device( 'cuda') # trick the device so that no warnings are triggered # upon instantiation of the DefaultOptimizer optimizer = DefaultOptimizer(optimizer_cfg) optimizer.device = Mock() optimizer.optimizer_cfg.training_cfg.epochs = 10 optimizer.optimizer_cfg.training_cfg.early_stopping = None model = Mock(spec=nn.Module) model.parameters = Mock() dataset = Mock(spec=torch.utils.data.BatchSampler) dataset.__len__ = Mock( return_value=2 ) # otherwise the pytorch DataLoader object will be unhappy will the # default parameters # patch disables import torch.optim, so we can skip creating models to test the optimizer with patch('trojai.modelgen.default_optimizer.torch.optim.Adam') as patched_optimizer, \ patch('trojai.modelgen.default_optimizer.train_val_dataset_split', return_value=(dataset, dataset)) as patched_train_val_split: # this function overrides the return value of train_epoch, so that we can simulate # when early-stopping is supposed to occur, and def train_epoch_side_effect(net, train_loader, val_loader, epoch, progress_bar_disable=True): # these variables are not consequential for the early-stopping code, so we just set them to # constants train_acc_noop = 1.0 train_loss_noop = 1.0 ts = EpochTrainStatistics(train_acc_noop, train_loss_noop) val_acc_noop = 1.0 if epoch < 2: val_loss = 10.0 - epoch # we keep the loss decreasing until the first 4 epochs # This prevents the early-stopping code from being activated, # since the loss is decreasing every epoch vs = EpochValidationStatistics(val_acc_noop, val_loss) return ts, vs else: val_loss = float( epoch) # we fix the loss from here on within eps, # we expect it to quit in 5 epochs vs = EpochValidationStatistics(val_acc_noop, val_loss) return ts, vs optimizer.train_epoch = Mock(side_effect=train_epoch_side_effect) _, _, num_epochs_trained = optimizer.train(model, dataset) # the early stopping should *not* have been run, b/c we set it to None, so we should # have trained for the full 10 epochs self.assertEqual(num_epochs_trained, optimizer.optimizer_cfg.training_cfg.epochs)
def test_str(self): training_cfg = TrainingConfig(device='cpu') cfg = DefaultOptimizerConfig(training_cfg) optimizer_obj = DefaultOptimizer(cfg) optimizer_string = str(optimizer_obj) correct_string = "{'batch_size':32, 'num_epochs':10, " \ "'device':'cpu', 'lr':1.00000e-04, 'loss_function':'cross_entropy_loss', 'optimizer':'adam'}" self.assertEqual(optimizer_string, correct_string)
def test_static_accuracy(self): """ Test the accuracy computation built into the optimizer, given some data. This function tests the accuracy of a given chunk of data, with no previous data totals, and thus only tests "static" accuracy, not "running" accuracy """ cfg = DefaultOptimizerConfig() batch_size = cfg.training_cfg.batch_size num_outputs = 5 # now, modify a subset of the network output and make that the "real" output step = 0.05 batch_acc_vec = np.arange(0, 1 + step, step) for batch_acc in batch_acc_vec: random_mat = self.rso.rand(batch_size, num_outputs) row_sum = random_mat.sum(axis=1) # normalize the random_mat such that every row adds up to 1 # broadcast so we can divide every element in matrix by the row's sum fake_network_output = random_mat / row_sum[:, None] # shape: [batch_size x n_output] network_output = np.argmax(fake_network_output, axis=1) # the hard-decision prediction true_output = network_output.copy() num_indices_to_modify = int(batch_size * (1 - batch_acc)) indices_to_modify = self.rso.choice(range(batch_size), num_indices_to_modify, replace=False) # create the "true" output such that the target accuracy matches the desired value for ii in indices_to_modify: true_output[ii] = (true_output[ii] + 1) % num_outputs expected_balanced_acc = balanced_accuracy_score( true_output, network_output) * 100 # convert datatypes to what is expected during operation network_output_pt = torch.tensor(fake_network_output, dtype=torch.float) true_output_pt = torch.tensor(true_output, dtype=torch.long) # now compute the accuracy actual_acc, n_total, n_correct = \ _running_eval_acc(network_output_pt, true_output_pt, n_total=None, n_correct=None) self.assertAlmostEqual(actual_acc, expected_balanced_acc)
def test_accuracy(self): cfg = DefaultOptimizerConfig() optimizer_obj = DefaultOptimizer(cfg) batch_size = cfg.training_cfg.batch_size num_outputs = 5 random_mat = self.rso.rand(batch_size, num_outputs) row_sum = random_mat.sum(axis=1) # normalize the random_mat such that every row adds up to 1 # broadcast so we can divide every element in matrix by the row's sum fake_network_output = random_mat / row_sum[:, None] network_output = np.argmax(fake_network_output, axis=1) # now, modify a subset of the netowrk output and make that the "real" output true_output = network_output.copy() target_accuracy = 0.8 num_indices_to_modify = int(batch_size * (1 - target_accuracy)) num_indices_unmodified = batch_size - num_indices_to_modify indices_to_modify = self.rso.choice(range(batch_size), num_indices_to_modify, replace=False) expected_accuracy = float(num_indices_unmodified) / float( batch_size) * 100 for ii in indices_to_modify: true_output[ii] = true_output[ii] + 1 # convert datatypes to what is expected during operation network_output_pt = torch.tensor(fake_network_output, dtype=torch.float) true_output_pt = torch.tensor(true_output, dtype=torch.long) # now compute the accuracy actual_acc, n_total, n_correct = \ _eval_acc(network_output_pt, true_output_pt, n_total=0, n_correct=0) self.assertAlmostEqual(actual_acc, expected_accuracy)
def test_early_stopping3(self): """ The purpose of this test is to ensure that the early stopping is not activated - for the case where the EarlyStopping criterion of the number of epochs over which the validation accuracy must be lower than the threshold is not reached before end of training. """ optimizer_cfg = DefaultOptimizerConfig() optimizer_cfg.training_cfg.device = torch.device( 'cuda') # trick the device so that no warnings are triggered # upon instantiation of the DefaultOptimizer optimizer = DefaultOptimizer(optimizer_cfg) optimizer.optimizer_cfg.training_cfg.epochs = 10 optimizer.optimizer_cfg.training_cfg.early_stopping = EarlyStoppingConfig( ) model = Mock(spec=nn.Module) model.parameters = Mock() dataset = Mock(spec=torch.utils.data.Dataset) # patch disables import torch.optim, so we can skip creating models to test the optimizer with patch('trojai.modelgen.default_optimizer.torch.optim.Adam') as patched_optimizer, \ patch('trojai.modelgen.default_optimizer.train_val_dataset_split', return_value=([], [])) as patched_train_val_split: # this function overrides the return value of train_epoch, so that we can simulate # when early-stopping is supposed to occur, and def train_epoch_side_effect(net, train_loader, val_loader, epoch, compute_batch_stats, progress_bar_disable=True): # these variables are not consequential for the early-stopping code, so we just set them to # constants batch_num_no_op = 999 batch_train_acc_noop = 1 batch_train_loss_noop = 1 batch_val_loss_noop = 1 if epoch < 9: batch_val_acc = epoch # we just set the accuracy to an integer that increases over every epoch. # This prevents the early-stopping code from being activated, # since the accuracy is changing by a drastic amount every epoch return [ BatchStatistics(batch_num_no_op, batch_train_acc_noop, batch_train_loss_noop, batch_val_acc, batch_val_loss_noop) ] else: batch_val_acc = optimizer.optimizer_cfg.training_cfg.early_stopping.val_acc_eps return [ BatchStatistics(batch_num_no_op, batch_train_acc_noop, batch_train_loss_noop, batch_val_acc, batch_val_loss_noop) ] optimizer.train_epoch = Mock(side_effect=train_epoch_side_effect) _, _, num_epochs_trained = optimizer.train(model, dataset) # now put in the test condition # the test condition here that no warnings are triggered upon instantiation. # this means that early stopping code should not have been run, and we should # have trained for the full 10 epochs self.assertEqual(num_epochs_trained, optimizer.optimizer_cfg.training_cfg.epochs)