def test_early_stopping3(self):
"""
The purpose of this test is to ensure that the early stopping is not activated, when the configuration for
EarlyStopping is set to None. Even though we modify the val_acc as before, after epoch 4, EarlyStopping is
not configured and as a result, we train for all 10 epochs
"""
optimizer_cfg = DefaultOptimizerConfig()
optimizer_cfg.training_cfg.device = torch.device(
'cuda') # trick the device so that no warnings are triggered
# upon instantiation of the DefaultOptimizer
optimizer = DefaultOptimizer(optimizer_cfg)
optimizer.device = Mock()
optimizer.optimizer_cfg.training_cfg.epochs = 10
optimizer.optimizer_cfg.training_cfg.early_stopping = None
model = Mock(spec=nn.Module)
model.parameters = Mock()
dataset = Mock(spec=torch.utils.data.BatchSampler)
dataset.__len__ = Mock(
return_value=2
) # otherwise the pytorch DataLoader object will be unhappy will the
# default parameters
# patch disables import torch.optim, so we can skip creating models to test the optimizer
with patch('trojai.modelgen.default_optimizer.torch.optim.Adam') as patched_optimizer, \
patch('trojai.modelgen.default_optimizer.train_val_dataset_split',
return_value=(dataset, dataset)) as patched_train_val_split:
# this function overrides the return value of train_epoch, so that we can simulate
# when early-stopping is supposed to occur, and
def train_epoch_side_effect(net,
train_loader,
val_loader,
epoch,
progress_bar_disable=True):
# these variables are not consequential for the early-stopping code, so we just set them to
# constants
train_acc_noop = 1.0
train_loss_noop = 1.0
ts = EpochTrainStatistics(train_acc_noop, train_loss_noop)
val_acc_noop = 1.0
if epoch < 2:
val_loss = 10.0 - epoch # we keep the loss decreasing until the first 4 epochs
# This prevents the early-stopping code from being activated,
# since the loss is decreasing every epoch
vs = EpochValidationStatistics(val_acc_noop, val_loss)
return ts, vs
else:
val_loss = float(
epoch) # we fix the loss from here on within eps,
# we expect it to quit in 5 epochs
vs = EpochValidationStatistics(val_acc_noop, val_loss)
return ts, vs
optimizer.train_epoch = Mock(side_effect=train_epoch_side_effect)
_, _, num_epochs_trained = optimizer.train(model, dataset)
# the early stopping should *not* have been run, b/c we set it to None, so we should
# have trained for the full 10 epochs
self.assertEqual(num_epochs_trained,
optimizer.optimizer_cfg.training_cfg.epochs)
def test_str(self):
training_cfg = TrainingConfig(device='cpu')
cfg = DefaultOptimizerConfig(training_cfg)
optimizer_obj = DefaultOptimizer(cfg)
optimizer_string = str(optimizer_obj)
correct_string = "{'batch_size':32, 'num_epochs':10, " \
"'device':'cpu', 'lr':1.00000e-04, 'loss_function':'cross_entropy_loss', 'optimizer':'adam'}"
self.assertEqual(optimizer_string, correct_string)
def test_static_accuracy(self):
"""
Test the accuracy computation built into the optimizer, given some data.
This function tests the accuracy of a given chunk of data, with no previous data totals,
and thus only tests "static" accuracy, not "running" accuracy
"""
cfg = DefaultOptimizerConfig()
batch_size = cfg.training_cfg.batch_size
num_outputs = 5
# now, modify a subset of the network output and make that the "real" output
step = 0.05
batch_acc_vec = np.arange(0, 1 + step, step)
for batch_acc in batch_acc_vec:
random_mat = self.rso.rand(batch_size, num_outputs)
row_sum = random_mat.sum(axis=1)
# normalize the random_mat such that every row adds up to 1
# broadcast so we can divide every element in matrix by the row's sum
fake_network_output = random_mat / row_sum[:,
None] # shape: [batch_size x n_output]
network_output = np.argmax(fake_network_output,
axis=1) # the hard-decision prediction
true_output = network_output.copy()
num_indices_to_modify = int(batch_size * (1 - batch_acc))
indices_to_modify = self.rso.choice(range(batch_size),
num_indices_to_modify,
replace=False)
# create the "true" output such that the target accuracy matches the desired value
for ii in indices_to_modify:
true_output[ii] = (true_output[ii] + 1) % num_outputs
expected_balanced_acc = balanced_accuracy_score(
true_output, network_output) * 100
# convert datatypes to what is expected during operation
network_output_pt = torch.tensor(fake_network_output,
dtype=torch.float)
true_output_pt = torch.tensor(true_output, dtype=torch.long)
# now compute the accuracy
actual_acc, n_total, n_correct = \
_running_eval_acc(network_output_pt, true_output_pt, n_total=None, n_correct=None)
self.assertAlmostEqual(actual_acc, expected_balanced_acc)
def test_accuracy(self):
cfg = DefaultOptimizerConfig()
optimizer_obj = DefaultOptimizer(cfg)
batch_size = cfg.training_cfg.batch_size
num_outputs = 5
random_mat = self.rso.rand(batch_size, num_outputs)
row_sum = random_mat.sum(axis=1)
# normalize the random_mat such that every row adds up to 1
# broadcast so we can divide every element in matrix by the row's sum
fake_network_output = random_mat / row_sum[:, None]
network_output = np.argmax(fake_network_output, axis=1)
# now, modify a subset of the netowrk output and make that the "real" output
true_output = network_output.copy()
target_accuracy = 0.8
num_indices_to_modify = int(batch_size * (1 - target_accuracy))
num_indices_unmodified = batch_size - num_indices_to_modify
indices_to_modify = self.rso.choice(range(batch_size),
num_indices_to_modify,
replace=False)
expected_accuracy = float(num_indices_unmodified) / float(
batch_size) * 100
for ii in indices_to_modify:
true_output[ii] = true_output[ii] + 1
# convert datatypes to what is expected during operation
network_output_pt = torch.tensor(fake_network_output,
dtype=torch.float)
true_output_pt = torch.tensor(true_output, dtype=torch.long)
# now compute the accuracy
actual_acc, n_total, n_correct = \
_eval_acc(network_output_pt, true_output_pt, n_total=0, n_correct=0)
self.assertAlmostEqual(actual_acc, expected_accuracy)
def test_early_stopping3(self):
"""
The purpose of this test is to ensure that the early stopping is not activated - for the case where the
EarlyStopping criterion of the number of epochs over which the validation accuracy must be lower than the
threshold is not reached before end of training.
"""
optimizer_cfg = DefaultOptimizerConfig()
optimizer_cfg.training_cfg.device = torch.device(
'cuda') # trick the device so that no warnings are triggered
# upon instantiation of the DefaultOptimizer
optimizer = DefaultOptimizer(optimizer_cfg)
optimizer.optimizer_cfg.training_cfg.epochs = 10
optimizer.optimizer_cfg.training_cfg.early_stopping = EarlyStoppingConfig(
)
model = Mock(spec=nn.Module)
model.parameters = Mock()
dataset = Mock(spec=torch.utils.data.Dataset)
# patch disables import torch.optim, so we can skip creating models to test the optimizer
with patch('trojai.modelgen.default_optimizer.torch.optim.Adam') as patched_optimizer, \
patch('trojai.modelgen.default_optimizer.train_val_dataset_split',
return_value=([], [])) as patched_train_val_split:
# this function overrides the return value of train_epoch, so that we can simulate
# when early-stopping is supposed to occur, and
def train_epoch_side_effect(net,
train_loader,
val_loader,
epoch,
compute_batch_stats,
progress_bar_disable=True):
# these variables are not consequential for the early-stopping code, so we just set them to
# constants
batch_num_no_op = 999
batch_train_acc_noop = 1
batch_train_loss_noop = 1
batch_val_loss_noop = 1
if epoch < 9:
batch_val_acc = epoch # we just set the accuracy to an integer that increases over every epoch.
# This prevents the early-stopping code from being activated,
# since the accuracy is changing by a drastic amount every epoch
return [
BatchStatistics(batch_num_no_op, batch_train_acc_noop,
batch_train_loss_noop, batch_val_acc,
batch_val_loss_noop)
]
else:
batch_val_acc = optimizer.optimizer_cfg.training_cfg.early_stopping.val_acc_eps
return [
BatchStatistics(batch_num_no_op, batch_train_acc_noop,
batch_train_loss_noop, batch_val_acc,
batch_val_loss_noop)
]
optimizer.train_epoch = Mock(side_effect=train_epoch_side_effect)
_, _, num_epochs_trained = optimizer.train(model, dataset)
# now put in the test condition
# the test condition here that no warnings are triggered upon instantiation.
# this means that early stopping code should not have been run, and we should
# have trained for the full 10 epochs
self.assertEqual(num_epochs_trained,
optimizer.optimizer_cfg.training_cfg.epochs)