def test_different_data_order_on_different_epochs(self):
del self.hparams.training_hparams.gamma
del self.hparams.training_hparams.milestone_steps
del self.hparams.training_hparams.warmup_steps
init = {
k: v.clone().detach()
for k, v in self.model.state_dict().items()
}
# Train the model once and get the state.
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(1, 0, len(self.train_loader)),
end_step=Step.from_epoch(1, 1, len(self.train_loader)))
state1 = TestTrain.get_state(self.model)
# Train the model again and get the state.
self.model.load_state_dict(init)
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(2, 0, len(self.train_loader)),
end_step=Step.from_epoch(2, 1, len(self.train_loader)))
state2 = TestTrain.get_state(self.model)
# Ensure that the model states are NOT the same.
for k in state1:
self.assertFalse(np.array_equal(state1[k], state2[k]))
def test_checkpointing(self):
callback_step_count = 0
def callback(output_location, step, model, optimizer, logger):
nonlocal callback_step_count
callback_step_count += 1
# Train to epoch 1, iteration 1.
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=self.callbacks,
end_step=Step.from_epoch(1, 1, len(self.train_loader)))
# Add a step-counting callback.
self.callbacks.append(callback)
# Train to epoch 1, iteration 1 again. Checkpointing should ensure we
# only train for one step.
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=self.callbacks,
end_step=Step.from_epoch(1, 1, len(self.train_loader)))
self.assertEqual(callback_step_count, 2)
def test_nonrepeatable_data_order_without_seed(self):
del self.hparams.training_hparams.data_order_seed
init = {
k: v.clone().detach()
for k, v in self.model.state_dict().items()
}
# Train the model once and get the state.
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(1, 0, len(self.train_loader)),
end_step=Step.from_epoch(1, 1, len(self.train_loader)))
state1 = TestTrain.get_state(self.model)
# Train the model again and get the state.
self.model.load_state_dict(init)
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(1, 0, len(self.train_loader)),
end_step=Step.from_epoch(1, 1, len(self.train_loader)))
state2 = TestTrain.get_state(self.model)
# Ensure that the model states are NOT the same.
for k in state1:
self.assertFalse(np.array_equal(state1[k], state2[k]))
def test_last_step(self):
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=self.callbacks,
start_step=Step.from_epoch(2, 11, len(self.train_loader)),
end_step=Step.from_epoch(3, 0, len(self.train_loader)))
end_state = TestStandardCallbacks.get_state(self.model)
# Check that final state has been saved.
end_loc = paths.model(self.root,
Step.from_epoch(3, 0, len(self.train_loader)))
self.assertTrue(os.path.exists(end_loc))
# Check that the final state that is saved matches the final state of the network.
self.model.load_state_dict(torch.load(end_loc))
saved_state = TestStandardCallbacks.get_state(self.model)
self.assertStateEqual(end_state, saved_state)
# Check that the logger has the right number of entries.
self.assertTrue(os.path.exists(paths.logger(self.root)))
logger = MetricLogger.create_from_file(self.root)
self.assertEqual(len(logger.get_data('train_loss')), 1)
self.assertEqual(len(logger.get_data('test_loss')), 1)
self.assertEqual(len(logger.get_data('train_accuracy')), 1)
self.assertEqual(len(logger.get_data('test_accuracy')), 1)
# Check that the checkpoint file exists.
self.assertTrue(os.path.exists(paths.checkpoint(self.root)))
def test_create_restore_delete(self):
# Create the hyperparameters and objects to save.
hp = models.registry.get_default_hparams('cifar_resnet_20')
model = models.registry.get(hp.model_hparams)
optimizer = optimizers.get_optimizer(hp.training_hparams, model)
dataloader = datasets.registry.get(hp.dataset_hparams)
step = Step.from_epoch(13, 27, 400)
# Run one step of SGD.
examples, labels = next(iter(dataloader))
optimizer.zero_grad()
model.train()
model.loss_criterion(model(examples), labels).backward()
optimizer.step()
# Create a fake logger.
logger = MetricLogger()
logger.add('test_accuracy', Step.from_epoch(0, 0, 400), 0.1)
logger.add('test_accuracy', Step.from_epoch(10, 0, 400), 0.5)
logger.add('test_accuracy', Step.from_epoch(100, 0, 400), 0.8)
# Save a checkpoint.
checkpointing.save_checkpoint_callback(self.root, step, model, optimizer, logger)
self.assertTrue(os.path.exists(paths.checkpoint(self.root)))
# Create new models.
model2 = models.registry.get(hp.model_hparams)
optimizer2 = optimizers.get_optimizer(hp.training_hparams, model)
# Ensure the new model has different weights.
sd1, sd2 = model.state_dict(), model2.state_dict()
for k in model.prunable_layer_names:
self.assertFalse(np.array_equal(sd1[k].numpy(), sd2[k].numpy()))
self.assertIn('momentum_buffer', optimizer.state[optimizer.param_groups[0]['params'][0]])
self.assertNotIn('momentum_buffer', optimizer2.state[optimizer.param_groups[0]['params'][0]])
# Restore the checkpointt.
step2, logger2 = checkpointing.restore_checkpoint(self.root, model2, optimizer2, 400)
self.assertTrue(os.path.exists(paths.checkpoint(self.root)))
self.assertEqual(step, step2)
self.assertEqual(str(logger), str(logger2))
# Ensure the new model is now the same.
sd1, sd2 = model.state_dict(), model2.state_dict()
self.assertEqual(set(sd1.keys()), set(sd2.keys()))
for k in sd1:
self.assertTrue(np.array_equal(sd1[k].numpy(), sd2[k].numpy()))
# Ensure the new optimizer is now the same.
mom1 = optimizer.state[optimizer.param_groups[0]['params'][0]]['momentum_buffer']
mom2 = optimizer2.state[optimizer.param_groups[0]['params'][0]]['momentum_buffer']
self.assertTrue(np.array_equal(mom1.numpy(), mom2.numpy()))
def test_from_epoch(self):
self.assertStepEquals(Step.from_epoch(0, 0, 1), 0, 0, 0)
self.assertStepEquals(Step.from_epoch(5, 0, 1), 5, 5, 0)
self.assertStepEquals(Step.from_epoch(100, 0, 1), 100, 100, 0)
self.assertStepEquals(Step.from_epoch(100, 20, 1), 120, 120, 0)
self.assertStepEquals(Step.from_epoch(0, 0, 100), 0, 0, 0)
self.assertStepEquals(Step.from_epoch(0, 50, 100), 50, 0, 50)
self.assertStepEquals(Step.from_epoch(1, 0, 100), 100, 1, 0)
self.assertStepEquals(Step.from_epoch(1, 50, 100), 150, 1, 50)
self.assertStepEquals(Step.from_epoch(100, 30, 100), 10030, 100, 30)
self.assertStepEquals(Step.from_epoch(100, 1000, 100), 11000, 110, 0)
def test_train_two_epoch_late_start(self):
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(0, 5, len(self.train_loader)),
end_step=Step.from_epoch(2, 5, len(self.train_loader)))
self.assertEqual(self.step_counter, 25)
self.assertEqual(self.ep, 2)
self.assertEqual(self.it, 5)
self.assertEqual(self.lr, 0.01)
def restore_checkpoint(output_location, model, optimizer,
iterations_per_epoch):
checkpoint_location = paths.checkpoint(output_location)
if not get_platform().exists(checkpoint_location):
return None, None
checkpoint = get_platform().load_model(checkpoint_location,
map_location=torch.device('cpu'))
# Handle DataParallel.
module_in_name = get_platform().is_parallel
if module_in_name and not all(
k.startswith('module.') for k in checkpoint['model_state_dict']):
checkpoint['model_state_dict'] = {
'module.' + k: v
for k, v in checkpoint['model_state_dict'].items()
}
elif all(k.startswith('module.')
for k in checkpoint['model_state_dict']) and not module_in_name:
checkpoint['model_state_dict'] = {
k[len('module.'):]: v
for k, v in checkpoint['model_state_dict'].items()
}
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
step = Step.from_epoch(checkpoint['ep'], checkpoint['it'],
iterations_per_epoch)
logger = MetricLogger.create_from_string(checkpoint['logger'])
return step, logger
def test_first_step(self):
init_state = TestStandardCallbacks.get_state(self.model)
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=self.callbacks,
end_step=Step.from_epoch(0, 1, len(self.train_loader)))
# Check that the initial state has been saved.
model_state_loc = paths.model(self.root,
Step.zero(len(self.train_loader)))
self.assertTrue(os.path.exists(model_state_loc))
# Check that the model state at init reflects the saved state.
self.model.load_state_dict(torch.load(model_state_loc))
saved_state = TestStandardCallbacks.get_state(self.model)
self.assertStateEqual(init_state, saved_state)
# Check that the checkpoint file exists.
self.assertTrue(os.path.exists(paths.checkpoint(self.root)))
# Check that the logger file doesn't exist.
self.assertFalse(os.path.exists(paths.logger(self.root)))
def test_train_zero_steps_late_start(self):
before = TestTrain.get_state(self.model)
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(0, 5, len(self.train_loader)),
end_step=Step.from_epoch(0, 5, len(self.train_loader)))
after = TestTrain.get_state(self.model)
for k in before:
self.assertTrue(np.array_equal(before[k], after[k]))
self.assertEqual(self.step_counter, 0)
self.assertEqual(self.ep, 0)
self.assertEqual(self.it, 0)
def test_train_in_full_later_start(self):
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(1, 5, len(self.train_loader)))
self.assertEqual(self.step_counter, 20)
self.assertEqual(self.ep, 3)
self.assertEqual(self.it, 0)
self.assertEqual(self.lr, 0.01)
def test_train_more_than_two_epochs(self):
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
end_step=Step.from_epoch(2, 1, len(self.train_loader)))
self.assertEqual(self.step_counter, 26)
self.assertEqual(self.ep, 2)
self.assertEqual(self.it, 1)
self.assertEqual(self.lr, 0.01)
def test_end_to_end(self):
init_loc = paths.model(self.root, Step.zero(len(self.train_loader)))
end_loc = paths.model(self.root,
Step.from_epoch(3, 0, len(self.train_loader)))
init_state = TestStandardCallbacks.get_state(self.model)
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=self.callbacks,
start_step=Step.from_epoch(0, 0, len(self.train_loader)),
end_step=Step.from_epoch(3, 0, len(self.train_loader)))
end_state = TestStandardCallbacks.get_state(self.model)
# Check that final state has been saved.
self.assertTrue(os.path.exists(init_loc))
self.assertTrue(os.path.exists(end_loc))
# Check that the checkpoint file still exists.
self.assertTrue(os.path.exists(paths.checkpoint(self.root)))
# Check that the initial and final states match those that were saved.
self.model.load_state_dict(torch.load(init_loc))
saved_state = TestStandardCallbacks.get_state(self.model)
self.assertStateEqual(init_state, saved_state)
self.model.load_state_dict(torch.load(end_loc))
saved_state = TestStandardCallbacks.get_state(self.model)
self.assertStateEqual(end_state, saved_state)
# Check that the logger has the right number of entries.
self.assertTrue(os.path.exists(paths.logger(self.root)))
logger = MetricLogger.create_from_file(self.root)
self.assertEqual(len(logger.get_data('train_loss')), 4)
self.assertEqual(len(logger.get_data('test_loss')), 4)
self.assertEqual(len(logger.get_data('train_accuracy')), 4)
self.assertEqual(len(logger.get_data('test_accuracy')), 4)
def test_train_one_epoch(self):
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
end_step=Step.from_epoch(1, 0, len(self.train_loader)))
self.assertEqual(self.step_counter,
13) # Same as len(self.train_loader) + 1
self.assertEqual(self.ep, 1)
self.assertEqual(self.it, 0)
self.assertEqual(self.lr, 0.1)
def train(training_hparams: hparams.TrainingHparams,
model: Model,
train_loader: DataLoader,
output_location: str,
callbacks: typing.List[typing.Callable] = [],
start_step: Step = None,
end_step: Step = None):
"""The main training loop for this framework.
Args:
* training_hparams: The training hyperparameters whose schema is specified in hparams.py.
* model: The model to train. Must be a models.base.Model
* train_loader: The training data. Must be a datasets.base.DataLoader
* output_location: The string path where all outputs should be stored.
* callbacks: A list of functions that are called before each training step and once more
after the last training step. Each function takes five arguments: the current step,
the output location, the model, the optimizer, and the logger.
Callbacks are used for running the test set, saving the logger, saving the state of the
model, etc. The provide hooks into the training loop for customization so that the
training loop itself can remain simple.
* start_step: The step at which the training data and learning rate schedule should begin.
Defaults to step 0.
* end_step: The step at which training should cease. Otherwise, training will go for the
full `training_hparams.training_steps` steps.
"""
# Create the output location if it doesn't already exist.
if not get_platform().exists(output_location) and get_platform(
).is_primary_process:
get_platform().makedirs(output_location)
# Get the optimizer and learning rate schedule.
model.to(get_platform().torch_device)
optimizer = optimizers.get_optimizer(training_hparams, model)
step_optimizer = optimizer
lr_schedule = optimizers.get_lr_schedule(training_hparams, optimizer,
train_loader.iterations_per_epoch)
# Adapt for FP16.
if training_hparams.apex_fp16:
if NO_APEX:
raise ImportError('Must install nvidia apex to use this model.')
model, step_optimizer = apex.amp.initialize(model,
optimizer,
loss_scale='dynamic',
verbosity=0)
# Handle parallelism if applicable.
if get_platform().is_distributed:
model = DistributedDataParallel(model,
device_ids=[get_platform().rank])
elif get_platform().is_parallel:
model = DataParallel(model)
# Get the random seed for the data order.
data_order_seed = training_hparams.data_order_seed
# Restore the model from a saved checkpoint if the checkpoint exists.
cp_step, cp_logger = restore_checkpoint(output_location, model, optimizer,
train_loader.iterations_per_epoch)
start_step = cp_step or start_step or Step.zero(
train_loader.iterations_per_epoch)
logger = cp_logger or MetricLogger()
with warnings.catch_warnings(): # Filter unnecessary warning.
warnings.filterwarnings("ignore", category=UserWarning)
for _ in range(start_step.iteration):
lr_schedule.step()
# Determine when to end training.
end_step = end_step or Step.from_str(training_hparams.training_steps,
train_loader.iterations_per_epoch)
if end_step <= start_step: return
# The training loop.
for ep in range(start_step.ep, end_step.ep + 1):
# Ensure the data order is different for each epoch.
train_loader.shuffle(None if data_order_seed is None else (
data_order_seed + ep))
for it, (examples, labels) in enumerate(train_loader):
# Advance the data loader until the start epoch and iteration.
if ep == start_step.ep and it < start_step.it: continue
# Run the callbacks.
step = Step.from_epoch(ep, it, train_loader.iterations_per_epoch)
for callback in callbacks:
callback(output_location, step, model, optimizer, logger)
# Exit at the end step.
if ep == end_step.ep and it == end_step.it: return
# Otherwise, train.
examples = examples.to(device=get_platform().torch_device)
labels = labels.to(device=get_platform().torch_device)
step_optimizer.zero_grad()
model.train()
loss = model.loss_criterion(model(examples), labels)
if training_hparams.apex_fp16:
with apex.amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
# Step forward. Ignore extraneous warnings that the lr_schedule generates.
step_optimizer.step()
with warnings.catch_warnings(): # Filter unnecessary warning.
warnings.filterwarnings("ignore", category=UserWarning)
lr_schedule.step()
get_platform().barrier()
def distill(
training_hparams: hparams.TrainingHparams,
distill_hparams: hparams.DistillHparams,
student: Model,
teacher: Model,
train_loader: DataLoader,
output_location: str,
callbacks: typing.List[typing.Callable] = [],
start_step: Step = None,
end_step: Step = None
):
"""The main training loop for this framework.
Args:
* training_hparams: The training hyperparameters whose schema is specified in hparams.py.
* distll_hparams: The knowledge distillation hyperparameters whose schema is specified in hparams.py.
* student: The student model to train. Must be a models.base.Model
* teacher: The teacher model to distill the knowledge. Must be a models.base.Model
* train_loader: The training data. Must be a datasets.base.DataLoader
* output_location: The string path where all outputs should be stored.
* callbacks: A list of functions that are called before each training step and once more
after the last training step. Each function takes five arguments: the current step,
the output location, the model, the optimizer, and the logger.
Callbacks are used for running the test set, saving the logger, saving the state of the
model, etc. The provide hooks into the training loop for customization so that the
training loop itself can remain simple.
* start_step: The step at which the training data and learning rate schedule should begin.
Defaults to step 0.
* end_step: The step at which training should cease. Otherwise, training will go for the
full `training_hparams.training_steps` steps.
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
# Create the output location if it doesn't already exist.
if not get_platform().exists(output_location) and get_platform().is_primary_process:
get_platform().makedirs(output_location)
# Get the optimizer and learning rate schedule.
student.to(get_platform().torch_device)
teacher.to(get_platform().torch_device)
optimizer = optimizers.get_optimizer(training_hparams, student)
step_optimizer = optimizer
lr_schedule = optimizers.get_lr_schedule(training_hparams, optimizer, train_loader.iterations_per_epoch)
ce_loss_fct = nn.KLDivLoss(reduction="batchmean")
if distill_hparams.alpha_mse > 0.0:
mse_loss_fct = nn.MSELoss(reduction='sum')
if distill_hparams.alpha_cos > 0.0:
cos_loss_fct = nn.CosineEmbeddingLoss(reduction='mean')
# Adapt for FP16.
if training_hparams.apex_fp16:
if NO_APEX: raise ImportError('Must install nvidia apex to use this model.')
(student, teacher), step_optimizer = apex.amp.initialize(
[student, teacher], optimizer, loss_scale='dynamic', verbosity=0
)
# Handle parallelism if applicable.
if get_platform().is_distributed:
student = DistributedDataParallel(student, device_ids=[get_platform().rank])
teacher = DistributedDataParallel(teacher, device_ids=[get_platform().rank])
elif get_platform().is_parallel:
student = DataParallel(student)
teacher = DataParallel(teacher)
# Get the random seed for the data order.
data_order_seed = training_hparams.data_order_seed
# Restore the model from a saved checkpoint if the checkpoint exists.
cp_step, cp_logger = restore_checkpoint(output_location, student, optimizer, train_loader.iterations_per_epoch)
start_step = cp_step or start_step or Step.zero(train_loader.iterations_per_epoch)
logger = cp_logger or MetricLogger()
with warnings.catch_warnings(): # Filter unnecessary warning.
warnings.filterwarnings("ignore", category=UserWarning)
for _ in range(start_step.iteration): lr_schedule.step()
# Determine when to end training.
end_step = end_step or Step.from_str(training_hparams.training_steps, train_loader.iterations_per_epoch)
if end_step <= start_step: return
# The training loop.
for ep in range(start_step.ep, end_step.ep + 1):
# Ensure the data order is different for each epoch.
train_loader.shuffle(None if data_order_seed is None else (data_order_seed + ep))
for it, (examples, labels) in enumerate(train_loader):
# Advance the data loader until the start epoch and iteration.
if ep == start_step.ep and it < start_step.it: continue
# Run the callbacks.
step = Step.from_epoch(ep, it, train_loader.iterations_per_epoch)
for callback in callbacks: callback(output_location, step, student, optimizer, logger)
# Exit at the end step.
if ep == end_step.ep and it == end_step.it: return
# Otherwise, train.
examples = examples.to(device=get_platform().torch_device)
labels = labels.to(device=get_platform().torch_device)
loss = 0.0
step_optimizer.zero_grad()
student.train()
teacher.eval()
student_outputs = student(examples)
with torch.no_grad():
teacher_outputs = teacher(examples)
s_logits = student_outputs
t_logits = teacher_outputs
# KL Divergence loss for the knowledge distillation
loss_ce = ce_loss_fct(
F.log_softmax(s_logits / distill_hparams.temperature, dim=-1),
F.softmax(t_logits / distill_hparams.temperature, dim=-1),
) * distill_hparams.temperature**2
loss += distill_hparams.alpha_ce * loss_ce
if distill_hparams.alpha_cls > 0.0:
loss_cls = student.loss_criterion(student_outputs, labels)
loss += distill_hparams.alpha_cls * loss_cls
if distill_hparams.alpha_mse > 0.0:
loss_mse = mse_loss_fct(s_logits, t_logits) / s_logits.size(0)
loss += distill_hparams.alpha_mse * loss_mse
if training_hparams.apex_fp16:
with apex.amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
# Step forward. Ignore extraneous warnings that the lr_schedule generates.
step_optimizer.step()
with warnings.catch_warnings(): # Filter unnecessary warning.
warnings.filterwarnings("ignore", category=UserWarning)
lr_schedule.step()
get_platform().barrier()
def test_train_in_parts(self):
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
end_step=Step.from_epoch(0, 7, len(self.train_loader)))
self.assertEqual(self.step_counter, 8)
self.assertEqual(self.ep, 0)
self.assertEqual(self.it, 7)
self.assertEqual(self.lr, 0.07)
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(0, 7, len(self.train_loader)),
end_step=Step.from_epoch(0, 8, len(self.train_loader)))
self.assertEqual(self.step_counter, 10)
self.assertEqual(self.ep, 0)
self.assertEqual(self.it, 8)
self.assertEqual(self.lr, 0.08)
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(0, 8, len(self.train_loader)),
end_step=Step.from_epoch(1, 2, len(self.train_loader)))
self.assertEqual(self.step_counter, 17)
self.assertEqual(self.ep, 1)
self.assertEqual(self.it, 2)
self.assertEqual(self.lr, 0.1)
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(1, 2, len(self.train_loader)),
end_step=Step.from_epoch(2, 1, len(self.train_loader)))
self.assertEqual(self.step_counter, 29)
self.assertEqual(self.ep, 2)
self.assertEqual(self.it, 1)
self.assertEqual(self.lr, 0.01)
train.train(self.hparams.training_hparams,
self.model,
self.train_loader,
self.root,
callbacks=[self.callback],
start_step=Step.from_epoch(2, 1, len(self.train_loader)),
end_step=Step.from_epoch(3, 0, len(self.train_loader)))
self.assertEqual(self.step_counter, 41)
self.assertEqual(self.ep, 3)
self.assertEqual(self.it, 0)
self.assertEqual(self.lr, 0.01)
