def _train_epoch(self, epoch):
# lr update
if self.lr_scheduler is not None:
self.lr_scheduler.step(epoch)
for param_group in self.optimizer.param_groups:
self.current_lr = param_group['lr']
batch_time = AverageMeter()
data_time = AverageMeter()
ave_total_loss = AverageMeter()
ave_acc = AverageMeter()
ave_iou = AverageMeter()
# set model mode
self.model.train()
tic = time.time()
for steps, (data, target) in enumerate(self.train_data_loader,
start=1):
data = data.to(self.device, non_blocking=True)
target = target.to(self.device, non_blocking=True)
# 加载数据所用的时间
data_time.update(time.time() - tic)
# forward calculate
logits = self.model(data)
loss = self.loss(logits, target)
acc = Accuracy(logits, target)
miou = MIoU(logits, target, self.config.nb_classes)
# compute gradient and do SGD step
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# update average metrics
batch_time.update(time.time() - tic)
ave_total_loss.update(loss.data.item())
ave_acc.update(acc.item())
ave_iou.update(miou.item())
# display on the screen per display_steps
if steps % self.dis_period == 0:
print(
'Epoch: [{}][{}/{}],\n'
'Learning_Rate: {:.6f},\n'
'Time: {:.4f}, Data: {:.4f},\n'
'MIoU: {:6.4f}, Accuracy: {:6.4f}, Loss: {:.6f}'.
format(epoch, steps, len(self.train_data_loader),
self.current_lr, batch_time.average(),
data_time.average(), ave_iou.average(),
ave_acc.average(), ave_total_loss.average()))
tic = time.time()
# train log and return
self.history['train']['epoch'].append(epoch)
self.history['train']['loss'].append(ave_total_loss.average())
self.history['train']['acc'].append(ave_acc.average())
self.history['train']['miou'].append(ave_iou.average())
return {
'epoch': epoch,
'loss': ave_total_loss.average(),
'acc': ave_acc.average(),
'miou': ave_iou.average(),
}
def _eval_epoch(self, epoch):
batch_time = AverageMeter()
data_time = AverageMeter()
ave_total_loss = AverageMeter()
ave_acc = AverageMeter()
ave_iou = AverageMeter()
# set model mode
self.model.eval()
with torch.no_grad():
tic = time.time()
for steps, (data, target) in enumerate(self.valid_data_loder,
start=1):
# processing no blocking
# non_blocking tries to convert asynchronously with respect to the host if possible
# converting CPU tensor with pinned memory to CUDA tensor
# overlap transfer if pinned memory
data = data.to(self.device, non_blocking=True)
target = target.to(self.device, non_blocking=True)
data_time.update(time.time() - tic)
logits = self.model(data)
loss = self.loss(logits, target)
# calculate metrics
acc = Accuracy(logits, target)
miou = MIoU(logits, target, self.config.nb_classes)
#print("===========acc, miou==========", acc, miou)
# update ave metrics
batch_time.update(time.time() - tic)
ave_total_loss.update(loss.data.item())
ave_acc.update(acc.item())
ave_iou.update(miou.item())
tic = time.time()
# display validation at the end
print('Epoch {} validation done !'.format(epoch))
print('Time: {:.4f}, Data: {:.4f},\n'
'MIoU: {:6.4f}, Accuracy: {:6.4f}, Loss: {:.6f}'.
format(batch_time.average(), data_time.average(),
ave_iou.average(), ave_acc.average(),
ave_total_loss.average()))
self.history['valid']['epoch'].append(epoch)
self.history['valid']['loss'].append(ave_total_loss.average())
self.history['valid']['acc'].append(ave_acc.average())
self.history['valid']['miou'].append(ave_iou.average())
# validation log and return
return {
'epoch': epoch,
'val_Loss': ave_total_loss.average(),
'val_Accuracy': ave_acc.average(),
'val_MIoU': ave_iou.average(),
}
def eval_and_predict(self):
self._resume_ckpt()
self.model.eval()
#predictions = []
#filenames = []
predict_time = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
ave_total_loss = AverageMeter()
ave_acc = AverageMeter()
ave_iou = AverageMeter()
with torch.no_grad():
tic = time.time()
for steps, (data, target, filename) in enumerate(self.test_data_loader,start=1):
# data
data = data.to(self.device, non_blocking=True)
target = target.to(self.device, non_blocking=True)
data_time.update(time.time()-tic)
# output, loss, and metrics
pre_tic = time.time()
logits = self.model(data)
self._save_pred(logits, filename)
predict_time.update(time.time()-pre_tic)
loss = self.loss(logits, target)
acc = Accuracy(logits, target)
miou = MIoU(logits, target, self.config.nb_classes)
# update ave loss and metrics
batch_time.update(time.time()-tic)
tic = time.time()
ave_total_loss.update(loss.data.item())
ave_acc.update(acc)
ave_iou.update(miou)
# display evaluation result at the end
print('Evaluation phase !\n'
'Time: {:.2f}, Data: {:.2f},\n'
'MIoU: {:6.4f}, Accuracy: {:6.4f}, Loss: {:.6f}'
.format(batch_time.average(), data_time.average(),
ave_iou.average(), ave_acc.average(), ave_total_loss.average()))
#print('Saving Predict Map ... ...')
#self._save_pred(predictions, filenames)
print('Prediction Phase !\n'
'Total Time cost: {}s\n'
'Average Time cost per batch: {}s!'
.format(predict_time._get_sum(), predict_time.average()))
self.history['eval']['loss'].append(ave_total_loss.average())
self.history['eval']['acc'].append(ave_acc.average())
self.history['eval']['miou'].append(ave_iou.average())
self.history['eval']['time'].append(predict_time.average())
#TODO
print(" + Saved history of evaluation phase !")
hist_path = os.path.join(self.test_log_path, "history1.txt")
with open(hist_path, 'w') as f:
f.write(str(self.history))
