def train_iterations(self):
"""Train model for the 'self.epochs' number of batches."""
self.model.train()
pbar = ProgressBar(target=self.epochs, width=8)
iterator = InfiniteDataLoader(self.train_loader)
correct = 0
processed = 0
for iteration in range(self.epochs):
# Train a batch
data, target = iterator.get_batch()
loss = self.train_batch(data, target)
# Update Progress Bar
accuracy = 100 * self.train_correct / self.train_processed
pbar.update(iteration,
values=[('loss', round(loss, 2)),
('accuracy', round(accuracy, 2))])
# Update training history
self.update_training_history(loss, accuracy)
pbar.add(1,
values=[('loss', round(loss, 2)),
('accuracy', round(accuracy, 2))])
def train_iterations(self):
"""Train model for the 'self.epochs' number of batches."""
self.model.train()
pbar = ProgressBar(target=self.epochs, width=8)
iterator = InfiniteDataLoader(self.train_loader)
for iteration in range(self.epochs):
# Train a batch
loss = self.train_batch(iterator.get_batch())
# Update Progress Bar
pbar_values = self._get_pbar_values(loss)
pbar.update(iteration, values=pbar_values)
# Update training history
self.update_training_history(loss)
pbar.add(1, values=pbar_values)
def range_test(
self,
train_loader,
iterations,
mode='iteration',
learner=None,
val_loader=None,
start_lr=None,
end_lr=10,
step_mode='exp',
smooth_f=0.0,
diverge_th=5,
):
"""Performs the learning rate range test.
Args:
train_loader (torch.utils.data.DataLoader): The training set data loader.
iterations (int): The number of iterations/epochs over which the test occurs.
If 'mode' is set to 'iteration' then it will correspond to the
number of iterations else if mode is set to 'epoch' then it will correspond
to the number of epochs.
mode (str, optional): After which mode to update the learning rate. Can be
either 'iteration' or 'epoch'. (default: 'iteration')
learner (Learner, optional): Learner object for the model. (default: None)
val_loader (torch.utils.data.DataLoader, optional): If None, the range test
will only use the training metric. When given a data loader, the model is
evaluated after each iteration on that dataset and the evaluation metric
is used. Note that in this mode the test takes significantly longer but
generally produces more precise results. (default: None)
start_lr (float, optional): The starting learning rate for the range test.
If None, uses the learning rate from the optimizer. (default: None)
end_lr (float, optional): The maximum learning rate to test. (default: 10)
step_mode (str, optional): One of the available learning rate policies,
linear or exponential ('linear', 'exp'). (default: 'exp')
smooth_f (float, optional): The metric smoothing factor within the [0, 1]
interval. Disabled if set to 0, otherwise the metric is smoothed using
exponential smoothing. (default: 0.0)
diverge_th (int, optional): The test is stopped when the metric surpasses the
threshold: diverge_th * best_metric. To disable, set it to 0. (default: 5)
"""
# Check if correct 'mode' mode has been given
if not mode in ['iteration', 'epoch']:
raise ValueError(
f'For "mode" expected one of (iteration, epoch), got {mode}')
# Reset test results
self.history = {'lr': [], 'metric': []}
self.best_metric = None
self.best_lr = None
# Check if the optimizer is already attached to a scheduler
self._check_for_scheduler()
# Set the starting learning rate
if start_lr:
self._set_learning_rate(start_lr)
# Initialize the proper learning rate policy
if step_mode.lower() == 'exp':
lr_schedule = ExponentialLR(self.optimizer, end_lr, iterations)
elif step_mode.lower() == 'linear':
lr_schedule = LinearLR(self.optimizer, end_lr, iterations)
else:
raise ValueError(f'Expected one of (exp, linear), got {step_mode}')
if smooth_f < 0 or smooth_f >= 1:
raise ValueError('smooth_f is outside the range [0, 1]')
# Set accuracy metric if needed
metrics = None
if self.metric == 'accuracy':
metrics = ['accuracy']
# Get the learner object
if not learner is None:
self.learner = learner(train_loader,
self.optimizer,
self.criterion,
device=self.device,
val_loader=val_loader,
metrics=metrics)
else:
self.learner = Learner(train_loader,
self.optimizer,
self.criterion,
device=self.device,
val_loader=val_loader,
metrics=metrics)
self.learner.set_model(self.model)
train_iterator = InfiniteDataLoader(train_loader)
pbar = ProgressBar(target=iterations, width=8)
if mode == 'iteration':
print(mode.title() + 's')
for iteration in range(iterations):
# Train model
if mode == 'epoch':
print(f'{mode.title()} {iteration + 1}:')
self._train_model(mode, train_iterator)
if val_loader:
self.learner.validate(verbose=False)
# Get metric value
metric_value = self._get_metric(val_loader)
# Update the learning rate
lr_schedule.step()
self.history['lr'].append(lr_schedule.get_lr()[0])
# Track the best metric and smooth it if smooth_f is specified
if iteration == 0:
self.best_metric = metric_value
self.best_lr = self.history['lr'][-1]
else:
if smooth_f > 0:
metric_value = smooth_f * metric_value + (
1 - smooth_f) * self.history['metric'][-1]
if ((self.metric == 'loss' and metric_value < self.best_metric)
or (self.metric == 'accuracy'
and metric_value > self.best_metric)):
self.best_metric = metric_value
self.best_lr = self.history['lr'][-1]
# Check if the metric has diverged; if it has, stop the test
self.history['metric'].append(metric_value)
metric_value = self._display_metric_value(metric_value)
if (diverge_th > 0
and ((self.metric == 'loss'
and metric_value > self.best_metric * diverge_th) or
(self.metric == 'accuracy'
and metric_value < self.best_metric / diverge_th))):
if mode == 'iteration':
pbar.update(iterations - 1,
values=[('lr', self.history['lr'][-1]),
(self.metric.title(), metric_value)])
print('\nStopping early, the loss has diverged.')
break
else:
if mode == 'epoch':
lr = self.history['lr'][-1]
print(
f'Learning Rate: {lr:.4f}, {self.metric.title()}: {metric_value:.2f}\n'
)
elif mode == 'iteration':
pbar.update(iteration,
values=[('lr', self.history['lr'][-1]),
(self.metric.title(), metric_value)])
metric = self._display_metric_value(self.best_metric)
if mode == 'epoch':
print(
f'Learning Rate: {self.best_lr:.4f}, {self.metric.title()}: {metric:.2f}\n'
)
elif mode == 'iteration':
pbar.add(1,
values=[('lr', self.best_lr),
(self.metric.title(), metric)])
print('Learning rate search finished.')