def get_lr_schedule(training_hparams: TrainingHparams,
optimizer: torch.optim.Optimizer,
iterations_per_epoch: int):
lambdas = [lambda it: 1.0]
# Drop the learning rate according to gamma at the specified milestones.
if bool(training_hparams.gamma) != bool(training_hparams.milestone_steps):
raise ValueError(
'milestones and gamma hyperparameters must both be set or not at all.'
)
if training_hparams.milestone_steps:
milestones = [
Step.from_str(x, iterations_per_epoch).iteration
for x in training_hparams.milestone_steps.split(',')
]
lambdas.append(
lambda it: training_hparams.gamma**bisect.bisect(milestones, it))
# Add linear learning rate warmup if specified.
if training_hparams.warmup_steps:
warmup_iters = Step.from_str(training_hparams.warmup_steps,
iterations_per_epoch).iteration
lambdas.append(lambda it: min(1.0, it / warmup_iters))
# Combine the lambdas.
return torch.optim.lr_scheduler.LambdaLR(
optimizer, lambda it: np.product([l(it) for l in lambdas]))
def standard_train(model: Model,
output_location: str,
dataset_hparams: hparams.DatasetHparams,
training_hparams: hparams.TrainingHparams,
start_step: Step = None,
verbose: bool = True,
evaluate_every_epoch: bool = True):
"""Train using the standard callbacks according to the provided hparams."""
# If the model file for the end of training already exists in this location, do not train.
iterations_per_epoch = datasets.registry.iterations_per_epoch(
dataset_hparams)
train_end_step = Step.from_str(training_hparams.training_steps,
iterations_per_epoch)
if (models.registry.exists(output_location, train_end_step)
and get_platform().exists(paths.logger(output_location))):
return
train_loader = datasets.registry.get(dataset_hparams, train=True)
test_loader = datasets.registry.get(dataset_hparams, train=False)
callbacks = standard_callbacks.standard_callbacks(
training_hparams,
train_loader,
test_loader,
start_step=start_step,
verbose=verbose,
evaluate_every_epoch=evaluate_every_epoch)
train(training_hparams,
model,
train_loader,
output_location,
callbacks,
start_step=start_step)
def lossonly_callbacks(training_hparams: hparams.TrainingHparams,
train_set_loader: DataLoader,
test_set_loader: DataLoader,
eval_on_train: bool = False,
verbose: bool = True,
start_step: Step = None,
evaluate_every_epoch: bool = True):
start = start_step or Step.zero(train_set_loader.iterations_per_epoch)
end = Step.from_str(training_hparams.training_steps,
train_set_loader.iterations_per_epoch)
loss_eval_callback = create_loss_eval_callback(verbose=verbose)
# Basic checkpointing and state saving at the beginning and end.
result = [
run_at_step(start, save_model),
run_at_step(end, save_model),
run_at_step(end, save_logger),
run_every_epoch(checkpointing.save_checkpoint_callback),
]
# Test every epoch if requested.
if evaluate_every_epoch:
result = [run_every_epoch(loss_eval_callback)] + result
elif verbose:
result.append(run_every_epoch(create_timekeeper_callback()))
# Ensure that testing occurs at least at the beginning and end of training.
if start.it != 0 or not evaluate_every_epoch:
result = [run_at_step(start, loss_eval_callback)] + result
if end.it != 0 or not evaluate_every_epoch:
result = [run_at_step(end, loss_eval_callback)] + result
return result
def standard_callbacks(training_hparams: hparams.TrainingHparams,
train_set_loader: DataLoader,
test_set_loader: DataLoader,
eval_on_train: bool = False,
verbose: bool = True,
start_step: Step = None,
evaluate_every_epoch: bool = True,
weight_save_steps: List[Step] = []):
start = start_step or Step.zero(train_set_loader.iterations_per_epoch)
end = Step.from_str(training_hparams.training_steps,
train_set_loader.iterations_per_epoch)
test_eval_callback = create_eval_callback('test',
test_set_loader,
verbose=verbose)
train_eval_callback = create_eval_callback('train',
train_set_loader,
verbose=verbose)
# Basic checkpointing and state saving at the beginning and end.
result = [
run_at_step(start, save_model),
run_at_step(end, save_model),
run_at_step(end, save_logger),
run_every_epoch(checkpointing.save_checkpoint_callback),
]
for s in weight_save_steps:
result.append(run_at_step(s, save_model))
# Test every epoch if requested.
if evaluate_every_epoch:
result = [run_every_epoch(test_eval_callback)] + result
elif verbose:
result.append(run_every_epoch(create_timekeeper_callback()))
# Ensure that testing occurs at least at the beginning and end of training.
if start.it != 0 or not evaluate_every_epoch:
result = [run_at_step(start, test_eval_callback)] + result
if end.it != 0 or not evaluate_every_epoch:
result = [run_at_step(end, test_eval_callback)] + result
# Do the same for the train set if requested.
if eval_on_train:
if evaluate_every_epoch:
result = [run_every_epoch(train_eval_callback)] + result
if start.it != 0 or not evaluate_every_epoch:
result = [run_at_step(start, train_eval_callback)] + result
if end.it != 0 or not evaluate_every_epoch:
result = [run_at_step(end, train_eval_callback)] + result
return result
def run(self):
if self.verbose and get_platform().is_primary_process:
print('='*82 + f'\nTraining a Model (Replicate {self.replicate})\n' + '-'*82)
print(self.desc.display)
print(f'Output Location: {self.desc.run_path(self.replicate)}' + '\n' + '='*82 + '\n')
self.desc.save(self.desc.run_path(self.replicate))
#TODO: make mask and model init paths configurable
init_path = os.path.join(get_platform().root, 'resnet18_lth')
model = models.registry.load(init_path, Step.from_str('2ep218it', 1000),
self.desc.model_hparams, self.desc.train_outputs)
pruned_model = PrunedModel(model, Mask.load(init_path))
train.standard_train(
#models.registry.get(self.desc.model_hparams), self.desc.run_path(self.replicate),
pruned_model, self.desc.run_path(self.replicate),
self.desc.dataset_hparams, self.desc.training_hparams, evaluate_every_epoch=self.evaluate_every_epoch)
def train(training_hparams: hparams.TrainingHparams,
model: Model,
train_loader: DataLoader,
output_location: str,
callbacks: typing.List[typing.Callable] = [],
start_step: Step = None,
end_step: Step = None):
"""The main training loop for this framework.
Args:
* training_hparams: The training hyperparameters whose schema is specified in hparams.py.
* model: The model to train. Must be a models.base.Model
* train_loader: The training data. Must be a datasets.base.DataLoader
* output_location: The string path where all outputs should be stored.
* callbacks: A list of functions that are called before each training step and once more
after the last training step. Each function takes five arguments: the current step,
the output location, the model, the optimizer, and the logger.
Callbacks are used for running the test set, saving the logger, saving the state of the
model, etc. The provide hooks into the training loop for customization so that the
training loop itself can remain simple.
* start_step: The step at which the training data and learning rate schedule should begin.
Defaults to step 0.
* end_step: The step at which training should cease. Otherwise, training will go for the
full `training_hparams.training_steps` steps.
"""
# Create the output location if it doesn't already exist.
if not get_platform().exists(output_location) and get_platform(
).is_primary_process:
get_platform().makedirs(output_location)
# Get the optimizer and learning rate schedule.
model.to(get_platform().torch_device)
optimizer = optimizers.get_optimizer(training_hparams, model)
step_optimizer = optimizer
lr_schedule = optimizers.get_lr_schedule(training_hparams, optimizer,
train_loader.iterations_per_epoch)
# Adapt for FP16.
if training_hparams.apex_fp16:
if NO_APEX:
raise ImportError('Must install nvidia apex to use this model.')
model, step_optimizer = apex.amp.initialize(model,
optimizer,
loss_scale='dynamic',
verbosity=0)
# Handle parallelism if applicable.
if get_platform().is_distributed:
model = DistributedDataParallel(model,
device_ids=[get_platform().rank])
elif get_platform().is_parallel:
model = DataParallel(model)
# Get the random seed for the data order.
data_order_seed = training_hparams.data_order_seed
# Restore the model from a saved checkpoint if the checkpoint exists.
cp_step, cp_logger = restore_checkpoint(output_location, model, optimizer,
train_loader.iterations_per_epoch)
start_step = cp_step or start_step or Step.zero(
train_loader.iterations_per_epoch)
logger = cp_logger or MetricLogger()
with warnings.catch_warnings(): # Filter unnecessary warning.
warnings.filterwarnings("ignore", category=UserWarning)
for _ in range(start_step.iteration):
lr_schedule.step()
# Determine when to end training.
end_step = end_step or Step.from_str(training_hparams.training_steps,
train_loader.iterations_per_epoch)
if end_step <= start_step: return
# The training loop.
for ep in range(start_step.ep, end_step.ep + 1):
# Ensure the data order is different for each epoch.
train_loader.shuffle(None if data_order_seed is None else (
data_order_seed + ep))
for it, (examples, labels) in enumerate(train_loader):
# Advance the data loader until the start epoch and iteration.
if ep == start_step.ep and it < start_step.it: continue
# Run the callbacks.
step = Step.from_epoch(ep, it, train_loader.iterations_per_epoch)
for callback in callbacks:
callback(output_location, step, model, optimizer, logger)
# Exit at the end step.
if ep == end_step.ep and it == end_step.it: return
# Otherwise, train.
examples = examples.to(device=get_platform().torch_device)
labels = labels.to(device=get_platform().torch_device)
step_optimizer.zero_grad()
model.train()
loss = model.loss_criterion(model(examples), labels)
if training_hparams.apex_fp16:
with apex.amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
# Step forward. Ignore extraneous warnings that the lr_schedule generates.
step_optimizer.step()
with warnings.catch_warnings(): # Filter unnecessary warning.
warnings.filterwarnings("ignore", category=UserWarning)
lr_schedule.step()
get_platform().barrier()
def str_to_step(self, s: str, pretrain: bool = False) -> Step:
dataset_hparams = self.pretrain_dataset_hparams if pretrain else self.dataset_hparams
iterations_per_epoch = datasets_registry.iterations_per_epoch(
dataset_hparams)
return Step.from_str(s, iterations_per_epoch)
def to_step(self, s):
return Step.from_str(
s,
datasets.registry.iterations_per_epoch(self.desc.dataset_hparams))
def distill(
training_hparams: hparams.TrainingHparams,
distill_hparams: hparams.DistillHparams,
student: Model,
teacher: Model,
train_loader: DataLoader,
output_location: str,
callbacks: typing.List[typing.Callable] = [],
start_step: Step = None,
end_step: Step = None
):
"""The main training loop for this framework.
Args:
* training_hparams: The training hyperparameters whose schema is specified in hparams.py.
* distll_hparams: The knowledge distillation hyperparameters whose schema is specified in hparams.py.
* student: The student model to train. Must be a models.base.Model
* teacher: The teacher model to distill the knowledge. Must be a models.base.Model
* train_loader: The training data. Must be a datasets.base.DataLoader
* output_location: The string path where all outputs should be stored.
* callbacks: A list of functions that are called before each training step and once more
after the last training step. Each function takes five arguments: the current step,
the output location, the model, the optimizer, and the logger.
Callbacks are used for running the test set, saving the logger, saving the state of the
model, etc. The provide hooks into the training loop for customization so that the
training loop itself can remain simple.
* start_step: The step at which the training data and learning rate schedule should begin.
Defaults to step 0.
* end_step: The step at which training should cease. Otherwise, training will go for the
full `training_hparams.training_steps` steps.
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
# Create the output location if it doesn't already exist.
if not get_platform().exists(output_location) and get_platform().is_primary_process:
get_platform().makedirs(output_location)
# Get the optimizer and learning rate schedule.
student.to(get_platform().torch_device)
teacher.to(get_platform().torch_device)
optimizer = optimizers.get_optimizer(training_hparams, student)
step_optimizer = optimizer
lr_schedule = optimizers.get_lr_schedule(training_hparams, optimizer, train_loader.iterations_per_epoch)
ce_loss_fct = nn.KLDivLoss(reduction="batchmean")
if distill_hparams.alpha_mse > 0.0:
mse_loss_fct = nn.MSELoss(reduction='sum')
if distill_hparams.alpha_cos > 0.0:
cos_loss_fct = nn.CosineEmbeddingLoss(reduction='mean')
# Adapt for FP16.
if training_hparams.apex_fp16:
if NO_APEX: raise ImportError('Must install nvidia apex to use this model.')
(student, teacher), step_optimizer = apex.amp.initialize(
[student, teacher], optimizer, loss_scale='dynamic', verbosity=0
)
# Handle parallelism if applicable.
if get_platform().is_distributed:
student = DistributedDataParallel(student, device_ids=[get_platform().rank])
teacher = DistributedDataParallel(teacher, device_ids=[get_platform().rank])
elif get_platform().is_parallel:
student = DataParallel(student)
teacher = DataParallel(teacher)
# Get the random seed for the data order.
data_order_seed = training_hparams.data_order_seed
# Restore the model from a saved checkpoint if the checkpoint exists.
cp_step, cp_logger = restore_checkpoint(output_location, student, optimizer, train_loader.iterations_per_epoch)
start_step = cp_step or start_step or Step.zero(train_loader.iterations_per_epoch)
logger = cp_logger or MetricLogger()
with warnings.catch_warnings(): # Filter unnecessary warning.
warnings.filterwarnings("ignore", category=UserWarning)
for _ in range(start_step.iteration): lr_schedule.step()
# Determine when to end training.
end_step = end_step or Step.from_str(training_hparams.training_steps, train_loader.iterations_per_epoch)
if end_step <= start_step: return
# The training loop.
for ep in range(start_step.ep, end_step.ep + 1):
# Ensure the data order is different for each epoch.
train_loader.shuffle(None if data_order_seed is None else (data_order_seed + ep))
for it, (examples, labels) in enumerate(train_loader):
# Advance the data loader until the start epoch and iteration.
if ep == start_step.ep and it < start_step.it: continue
# Run the callbacks.
step = Step.from_epoch(ep, it, train_loader.iterations_per_epoch)
for callback in callbacks: callback(output_location, step, student, optimizer, logger)
# Exit at the end step.
if ep == end_step.ep and it == end_step.it: return
# Otherwise, train.
examples = examples.to(device=get_platform().torch_device)
labels = labels.to(device=get_platform().torch_device)
loss = 0.0
step_optimizer.zero_grad()
student.train()
teacher.eval()
student_outputs = student(examples)
with torch.no_grad():
teacher_outputs = teacher(examples)
s_logits = student_outputs
t_logits = teacher_outputs
# KL Divergence loss for the knowledge distillation
loss_ce = ce_loss_fct(
F.log_softmax(s_logits / distill_hparams.temperature, dim=-1),
F.softmax(t_logits / distill_hparams.temperature, dim=-1),
) * distill_hparams.temperature**2
loss += distill_hparams.alpha_ce * loss_ce
if distill_hparams.alpha_cls > 0.0:
loss_cls = student.loss_criterion(student_outputs, labels)
loss += distill_hparams.alpha_cls * loss_cls
if distill_hparams.alpha_mse > 0.0:
loss_mse = mse_loss_fct(s_logits, t_logits) / s_logits.size(0)
loss += distill_hparams.alpha_mse * loss_mse
if training_hparams.apex_fp16:
with apex.amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
# Step forward. Ignore extraneous warnings that the lr_schedule generates.
step_optimizer.step()
with warnings.catch_warnings(): # Filter unnecessary warning.
warnings.filterwarnings("ignore", category=UserWarning)
lr_schedule.step()
get_platform().barrier()
def end_step(self):
iterations_per_epoch = datasets_registry.iterations_per_epoch(self.dataset_hparams)
return Step.from_str(self.training_hparams.training_steps, iterations_per_epoch)
def str_to_step(self, s: str) -> Step:
return Step.from_str(s, datasets_registry.iterations_per_epoch(self.dataset_hparams))
def test_comparisons(self):
self.assertLessEqual(Step.from_str('100it', 100),
Step.from_str('100it', 100))
self.assertLess(Step.from_str('100it', 100),
Step.from_str('101it', 100))
self.assertLessEqual(Step.from_str('100it', 100),
Step.from_str('101it', 100))
self.assertGreaterEqual(Step.from_str('100it', 100),
Step.from_str('100it', 100))
self.assertGreater(Step.from_str('102it', 100),
Step.from_str('101it', 100))
self.assertGreaterEqual(Step.from_str('102it', 100),
Step.from_str('101it', 100))
def test_equal(self):
self.assertEqual(Step.from_str('100it', 100),
Step.from_str('100it', 100))
self.assertNotEqual(Step.from_str('101it', 100),
Step.from_str('100it', 100))
self.assertEqual(Step.from_str('1ep', 100),
Step.from_str('100it', 100))
self.assertEqual(Step.from_str('5ep6it', 100),
Step.from_str('506it', 100))
with self.assertRaises(ValueError):
Step.from_str('100it', 101) == Step.from_str('100it', 100)
def test_from_str(self):
self.assertStepEquals(Step.from_str('0it', 100), 0, 0, 0)
self.assertStepEquals(Step.from_str('0ep', 100), 0, 0, 0)
self.assertStepEquals(Step.from_str('0ep0it', 100), 0, 0, 0)
self.assertStepEquals(Step.from_str('50it', 100), 50, 0, 50)
self.assertStepEquals(Step.from_str('100it', 100), 100, 1, 0)
self.assertStepEquals(Step.from_str('2021it', 100), 2021, 20, 21)
self.assertStepEquals(Step.from_str('5ep', 100), 500, 5, 0)
self.assertStepEquals(Step.from_str('20ep', 100), 2000, 20, 0)
self.assertStepEquals(Step.from_str('5ep3it', 100), 503, 5, 3)
with self.assertRaises(ValueError):
Step.from_str('', 100)
with self.assertRaises(ValueError):
Step.from_str('0', 100)
with self.assertRaises(ValueError):
Step.from_str('it', 100)
with self.assertRaises(ValueError):
Step.from_str('ep', 100)
with self.assertRaises(ValueError):
Step.from_str('ep0', 100)
with self.assertRaises(ValueError):
Step.from_str('20it50ep', 100)
def branch_function(
self,
strategy: str,
prune_fraction: float,
prune_experiment: str = 'main',
prune_step: str = '0ep0it', # The step for states used to prune.
prune_highest: bool = False,
prune_iterations: int = 1,
randomize_layerwise: bool = False,
state_experiment: str = 'main',
state_step:
str = '0ep0it', # The step of the state to use alongside the pruning mask.
start_step:
str = '0ep0it', # The step at which to start the learning rate schedule.
seed: int = None,
reinitialize: bool = False):
# Get the steps for each part of the process.
iterations_per_epoch = datasets.registry.iterations_per_epoch(
self.desc.dataset_hparams)
prune_step = Step.from_str(prune_step, iterations_per_epoch)
state_step = Step.from_str(state_step, iterations_per_epoch)
start_step = Step.from_str(start_step, iterations_per_epoch)
seed = self.replicate if seed is None else seed
# Try to load the mask.
try:
mask = Mask.load(self.branch_root)
except:
mask = None
result_folder = "Data_Distribution/"
if reinitialize:
result_folder = "Data_Distribution_Reinit/"
elif randomize_layerwise:
result_folder = "Data_Distribution__Randomize_Layerwise/"
if not mask and get_platform().is_primary_process:
# Gather the weights that will be used for pruning.
prune_path = self.desc.run_path(self.replicate, prune_experiment)
prune_model = models.registry.load(prune_path, prune_step,
self.desc.model_hparams)
# Ensure that a valid strategy is available.
strategy_class = [s for s in strategies if s.valid_name(strategy)]
if not strategy_class:
raise ValueError(f'No such pruning strategy {strategy}')
if len(strategy_class) > 1:
raise ValueError('Multiple matching strategies')
strategy_instance = strategy_class[0](strategy, self.desc, seed)
# Run the strategy for each iteration.
mask = Mask.ones_like(prune_model)
iteration_fraction = 1 - (1 - prune_fraction)**(
1 / float(prune_iterations))
if iteration_fraction > 0:
for it in range(0, prune_iterations):
# Make a defensive copy of the model and mask out the pruned weights.
prune_model2 = copy.deepcopy(prune_model)
with torch.no_grad():
for k, v in prune_model2.named_parameters():
v.mul_(mask.get(k, 1))
# Compute the scores.
scores = strategy_instance.score(prune_model2, mask)
# Prune.
mask = unvectorize(
prune(vectorize(scores),
iteration_fraction,
not prune_highest,
mask=vectorize(mask)), mask)
# Shuffle randomly per layer.
if randomize_layerwise: mask = shuffle_state_dict(mask, seed=seed)
mask = Mask({k: v.clone().detach() for k, v in mask.items()})
mask.save(self.branch_root)
# Plot graphs (Move below mask save?)
# plot_distribution_scores(strategy_instance.score(prune_model, mask), strategy, mask, prune_iterations, reinitialize, randomize_layerwise, result_folder)
# plot_distribution_scatter(strategy_instance.score(prune_model, mask), prune_model, strategy, mask, prune_iterations, reinitialize, randomize_layerwise, result_folder)
# pdb.set_trace()
# Load the mask.
get_platform().barrier()
mask = Mask.load(self.branch_root)
# Determine the start step.
state_path = self.desc.run_path(self.replicate, state_experiment)
if reinitialize: model = models.registry.get(self.desc.model_hparams)
else:
model = models.registry.load(state_path, state_step,
self.desc.model_hparams)
# plot_distribution_weights(model, strategy, mask, prune_iterations, reinitialize, randomize_layerwise, result_folder)
original_model = copy.deepcopy(model)
model = PrunedModel(model, mask)
# pdb.set_trace()
train.standard_train(model,
self.branch_root,
self.desc.dataset_hparams,
self.desc.training_hparams,
start_step=start_step,
verbose=self.verbose,
evaluate_every_epoch=self.evaluate_every_epoch)
weights_analysis(original_model, strategy, reinitialize,
randomize_layerwise, "original")
weights_analysis(model, strategy, reinitialize, randomize_layerwise,
"pruned")
