def start(self,
valid_set,
valid_loader,
mode='Validation',
epoch=None,
global_bar=None,
save_folder=None):
confm_list = np.zeros((self.cf.num_classes, self.cf.num_classes))
self.val_loss = AverageMeter()
# Initialize epoch progress bar
val_num_batches = math.ceil(valid_set.num_images /
float(self.cf.valid_batch_size))
prev_msg = '\n' + mode + ' estimated time...\n'
bar = ProgressBar(val_num_batches, lenBar=20)
bar.set_prev_msg(prev_msg)
bar.update(show=False)
# Validate model
if self.cf.problem_type == 'detection':
self.validation_loop(epoch, valid_loader, valid_set, bar,
global_bar, save_folder)
else:
self.validation_loop(epoch, valid_loader, valid_set, bar,
global_bar, confm_list)
# Compute stats
self.compute_stats(np.asarray(self.stats.val.conf_m),
self.val_loss)
# Save stats
self.save_stats(epoch)
if mode == 'Epoch Validation':
self.logger_stats.write_stat(
self.stats.train, epoch,
os.path.join(self.cf.train_json_path,
'valid_epoch_' + str(epoch) + '.json'))
elif mode == 'Validation':
self.logger_stats.write_stat(self.stats.val, epoch,
self.cf.val_json_file)
elif mode == 'Test':
self.logger_stats.write_stat(self.stats.val, epoch,
self.cf.test_json_file)
def start(self, criterion, valid_set, valid_loader, epoch=None, global_bar=None):
confm_list = np.zeros((self.cf.num_classes,self.cf.num_classes))
val_loss = AverageMeter()
# Initialize epoch progress bar
val_num_batches = math.ceil(valid_set.num_images / float(self.cf.valid_batch_size))
prev_msg = '\nValidation estimated time...\n'
bar = ProgressBar(val_num_batches, lenBar=20)
bar.set_prev_msg(prev_msg)
bar.update(show=False)
# Validate model
for vi, data in enumerate(valid_loader):
# Read data
inputs, gts = data
n_images,w,h,c = inputs.size()
inputs = Variable(inputs, volatile=True).cuda()
gts = Variable(gts, volatile=True).cuda()
# Predict model
outputs = self.model.net(inputs)
predictions = outputs.data.max(1)[1].cpu().numpy()
# Compute batch stats
val_loss.update(criterion(outputs, gts).data[0] / n_images, n_images)
confm = compute_confusion_matrix(predictions,gts.cpu().data.numpy(),self.cf.num_classes,self.cf.void_class)
confm_list = map(operator.add, confm_list, confm)
# Save epoch stats
self.stats.val.conf_m = confm_list
self.stats.val.loss = val_loss.avg / (w * h * c)
# Update messages
self.update_msg(bar, global_bar)
# Compute stats
self.compute_stats(np.asarray(self.stats.val.conf_m), val_loss)
# Save stats
self.save_stats(epoch)
class train(object):
def __init__(self, logger_stats, model, cf, validator, stats, msg):
# Initialize training variables
self.logger_stats = logger_stats
self.model = model
self.cf = cf
self.validator = validator
self.logger_stats.write('\n- Starting train <--- \n')
self.curr_epoch = 1 if self.model.best_stats.epoch == 0 else self.model.best_stats.epoch
self.stop = False
self.stats = stats
self.best_acc = 0
self.msg = msg
self.loss = None
self.outputs = None
self.labels = None
self.writer = SummaryWriter(
os.path.join(cf.tensorboard_path, 'train'))
def start(self,
train_loader,
train_set,
valid_set=None,
valid_loader=None):
self.train_num_batches = math.ceil(train_set.num_images /
float(self.cf.train_batch_size))
self.val_num_batches = 0 if valid_set is None else math.ceil(valid_set.num_images / \
float(self.cf.valid_batch_size))
# Define early stopping control
if self.cf.early_stopping:
early_Stopping = Early_Stopping(self.cf)
else:
early_Stopping = None
prev_msg = '\nTotal estimated training time...\n'
self.global_bar = ProgressBar(
(self.cf.epochs + 1 - self.curr_epoch) *
(self.train_num_batches + self.val_num_batches),
lenBar=20)
self.global_bar.set_prev_msg(prev_msg)
# Train process
for epoch in range(self.curr_epoch, self.cf.epochs + 1):
# Shuffle train data
train_set.update_indexes()
# Initialize logger
epoch_time = time.time()
self.logger_stats.write('\t ------ Epoch: ' + str(epoch) +
' ------ \n')
# Initialize epoch progress bar
self.msg.accum_str = '\n\nEpoch %d/%d estimated time...\n' % \
(epoch, self.cf.epochs)
epoch_bar = ProgressBar(self.train_num_batches, lenBar=20)
epoch_bar.update(show=False)
# Initialize stats
self.stats.epoch = epoch
self.train_loss = AverageMeter()
self.confm_list = np.zeros(
(self.cf.num_classes, self.cf.num_classes))
# Train epoch
self.training_loop(epoch, train_loader, epoch_bar)
# Save stats
self.stats.train.conf_m = self.confm_list
self.compute_stats(np.asarray(self.confm_list),
self.train_loss)
self.save_stats_epoch(epoch)
self.logger_stats.write_stat(
self.stats.train, epoch,
os.path.join(self.cf.train_json_path,
'train_epoch_' + str(epoch) + '.json'))
# Validate epoch
self.validate_epoch(valid_set, valid_loader, early_Stopping,
epoch, self.global_bar)
# Update scheduler
if self.model.scheduler is not None:
self.model.scheduler.step(self.stats.val.loss)
# Saving model if score improvement
new_best = self.model.save(self.stats)
if new_best:
self.logger_stats.write_best_stats(self.stats, epoch,
self.cf.best_json_file)
# Update display values
self.update_messages(epoch, epoch_time, new_best)
if self.stop:
return
# Save model without training
if self.cf.epochs == 0:
self.model.save_model()
def training_loop(self, epoch, train_loader, epoch_bar):
# Train epoch
for i, data in enumerate(train_loader):
# Read Data
inputs, labels = data
N, w, h, c = inputs.size()
inputs = Variable(inputs).cuda()
self.inputs = inputs
self.labels = Variable(labels).cuda()
# Predict model
self.model.optimizer.zero_grad()
self.outputs = self.model.net(inputs)
predictions = self.outputs.data.max(1)[1].cpu().numpy()
# Compute gradients
self.compute_gradients()
# Compute batch stats
self.train_loss.update(float(self.loss.cpu().item()), N)
confm = compute_confusion_matrix(
predictions,
self.labels.cpu().data.numpy(), self.cf.num_classes,
self.cf.void_class)
self.confm_list = map(operator.add, self.confm_list, confm)
if self.cf.normalize_loss:
self.stats.train.loss = self.train_loss.avg
else:
self.stats.train.loss = self.train_loss.avg
if not self.cf.debug:
# Save stats
self.save_stats_batch((epoch - 1) *
self.train_num_batches + i)
# Update epoch messages
self.update_epoch_messages(epoch_bar, self.global_bar,
self.train_num_batches, epoch,
i)
def save_stats_epoch(self, epoch):
# Save logger
if epoch is not None:
# Epoch loss tensorboard
self.writer.add_scalar('losses/epoch', self.stats.train.loss,
epoch)
self.writer.add_scalar('metrics/accuracy',
100. * self.stats.train.acc, epoch)
def save_stats_batch(self, batch):
# Save logger
if batch is not None:
self.writer.add_scalar('losses/batch', self.stats.train.loss,
batch)
def compute_gradients(self):
self.loss = self.model.loss(self.outputs, self.labels)
self.loss.backward()
self.model.optimizer.step()
def compute_stats(self, confm_list, train_loss):
TP_list, TN_list, FP_list, FN_list = extract_stats_from_confm(
confm_list)
mean_accuracy = compute_accuracy(TP_list, TN_list, FP_list,
FN_list)
self.stats.train.acc = np.nanmean(mean_accuracy)
self.stats.train.loss = float(train_loss.avg.cpu().data)
def validate_epoch(self, valid_set, valid_loader, early_Stopping,
epoch, global_bar):
if valid_set is not None and valid_loader is not None:
# Set model in validation mode
self.model.net.eval()
self.validator.start(valid_set,
valid_loader,
'Epoch Validation',
epoch,
global_bar=global_bar)
# Early stopping checking
if self.cf.early_stopping:
early_Stopping.check(self.stats.train.loss,
self.stats.val.loss,
self.stats.val.mIoU,
self.stats.val.acc)
if early_Stopping.stop == True:
self.stop = True
# Set model in training mode
self.model.net.train()
def update_messages(self, epoch, epoch_time):
# Update logger
epoch_time = time.time() - epoch_time
self.logger_stats.write('\t Epoch step finished: %ds \n' %
(epoch_time))
# Compute best stats
self.msg.msg_stats_last = '\nLast epoch: acc = %.2f, loss = %.5f\n' % (
100 * self.stats.val.acc, self.stats.val.loss)
if self.best_acc < self.stats.val.acc:
self.msg.msg_stats_best = 'Best case: epoch = %d, acc = %.2f, loss = %.5f\n' % (
epoch, 100 * self.stats.val.acc, self.stats.val.loss)
msg_confm = self.stats.val.get_confm_str()
self.logger_stats.write(msg_confm)
self.msg.msg_stats_best = self.msg.msg_stats_best + msg_confm
self.best_acc = self.stats.val.acc
def update_epoch_messages(self, epoch_bar, global_bar,
train_num_batches, epoch, batch):
# Update progress bar
epoch_bar.set_msg('loss = %.5f' % self.stats.train.loss)
self.msg.last_str = epoch_bar.get_message(step=True)
global_bar.set_msg(self.msg.accum_str + self.msg.last_str + self.msg.msg_stats_last + \
self.msg.msg_stats_best)
global_bar.update()
# writer.add_scalar('train_loss', train_loss.avg, curr_iter)
# Display progress
curr_iter = (epoch - 1) * train_num_batches + batch + 1
if (batch + 1) % math.ceil(train_num_batches / 20.) == 0:
self.logger_stats.write(
'[Global iteration %d], [iter %d / %d], [train loss %.5f] \n'
% (curr_iter, batch + 1, train_num_batches,
self.stats.train.loss))
def start(self, criterion, optimizer, train_loader, train_set, valid_set=None, valid_loader=None, scheduler=None):
train_num_batches = math.ceil(train_set.num_images / float(self.cf.train_batch_size))
val_num_batches = 0 if valid_set is None else math.ceil(valid_set.num_images / float(self.cf.valid_batch_size))
# Define early stopping control
if self.cf.early_stopping:
early_Stopping = Early_Stopping(self.cf)
else:
early_Stopping = None
prev_msg = '\nTotal estimated training time...\n'
global_bar = ProgressBar((self.cf.epochs+1-self.curr_epoch)*(train_num_batches+val_num_batches), lenBar=20)
global_bar.set_prev_msg(prev_msg)
# Train process
for epoch in range(self.curr_epoch, self.cf.epochs + 1):
# Shuffle train data
train_set.update_indexes()
# Initialize logger
epoch_time = time.time()
self.logger_stats.write('\t ------ Epoch: ' + str(epoch) + ' ------ \n')
# Initialize epoch progress bar
self.msg.accum_str = '\n\nEpoch %d/%d estimated time...\n' % (epoch, self.cf.epochs + 1 - self.curr_epoch)
epoch_bar = ProgressBar(train_num_batches, lenBar=20)
epoch_bar.update(show=False)
# Initialize stats
train_loss = AverageMeter()
confm_list = np.zeros((self.cf.num_classes, self.cf.num_classes))
# Train epoch
for i, data in enumerate(train_loader):
# Read Data
inputs, labels = data
N,w,h,c = inputs.size()
inputs = Variable(inputs).cuda()
labels = Variable(labels).cuda()
# Predict model
optimizer.zero_grad()
outputs = self.model.net(inputs)
predictions = outputs.data.max(1)[1].cpu().numpy()
# Compute gradients
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
# Compute batch stats
train_loss.update(loss.data[0], N)
confm = compute_confusion_matrix(predictions, labels.cpu().data.numpy(), self.cf.num_classes,
self.cf.void_class)
confm_list = map(operator.add, confm_list, confm)
self.stats.train.loss = train_loss.avg / (w*h*c)
# Save stats
self.save_stats_batch((epoch - 1) * train_num_batches + i)
# Update epoch messages
self.update_epoch_messages(epoch_bar, global_bar, train_num_batches,epoch, i)
# Save stats
self.stats.train.conf_m = confm_list
self.compute_stats(np.asarray(confm_list),train_loss)
self.save_stats_epoch(epoch)
# Validate epoch
self.validate_epoch(valid_set, valid_loader, criterion, early_Stopping, epoch, global_bar)
# Update scheduler
if scheduler is not None:
scheduler.step(self.stats.val.loss)
# Saving model if needed
self.model.net.save(self.stats)
# Update display values
self.update_messages(epoch, epoch_time)
if self.stop:
return
# Save model without training
if self.cf.epochs == 0:
self.model.save_model(self.model.net)