def cv(batch_idx):
model.eval()
test_num=len(val_dataset)
eval_acc = 0
eval_acc_cls = 0
eval_mean_iu = 0
eval_fwavacc = 0
for step, (imgs, sem_labels, ins_labels) in (enumerate(val_dataloader)):
# for step in tqdm(range(500)):
# (imgs, sem_labels, ins_labels)=next(iter(train_dataloader))
test_data, test_label, _ = imgs[0], sem_labels[0], ins_labels
pred = predict(test_data.cuda())
label_true = test_label.data.cpu().numpy()
acc, acc_cls, mean_iu, fwavacc = label_accuracy_score(label_true, pred, 9)
# break
# for lbt, lbp in zip(label_true, pred):
# acc, acc_cls, mean_iu, fwavacc = label_accuracy_score(lbt, lbp, 9)
eval_acc += acc
eval_acc_cls += acc_cls
eval_mean_iu += mean_iu
eval_fwavacc += fwavacc
if step % np.floor(test_num*0.2)==0 : print("进度:",step/test_num);
if step==test_num:
# Tensorboard
info = {'acc': eval_acc/test_num, 'acc_cls': eval_acc_cls/test_num, 'mean_iu': eval_mean_iu/test_num, 'fwavacc': eval_fwavacc/test_num}
for tag, value in info.items():
logger.scalar_summary(tag, value, batch_idx + 1)
print(eval_acc/test_num,eval_acc_cls/test_num,eval_mean_iu/test_num,eval_fwavacc/test_num)
break
model.train()
# end of epoch
model.model.eval()
if dataset_val != None:
label_trues, label_preds = [], []
for i, data in enumerate(dataset_val):
with torch.no_grad():
seggt, segpred = model.forward(data, False)
seggt = seggt.data.cpu().numpy()
segpred = segpred.data.cpu().numpy()
label_trues.append(seggt)
label_preds.append(segpred)
metrics = util.label_accuracy_score(label_trues,
label_preds,
n_class=opt.label_nc)
metrics = np.array(metrics)
metrics *= 100
print('''\
Validation:
Accuracy: {0}
Accuracy Class: {1}
Mean IU: {2}
FWAV Accuracy: {3}'''.format(*metrics))
model.update_tensorboard(metrics, total_steps)
iter_end_time = time.time()
print('End of epoch %d / %d \t Time Taken: %d sec' %
(epoch + 1, opt.nepochs, time.time() - epoch_start_time))
if metrics[2] > best_iou:
if i >= opt.how_many and opt.how_many != 0:
break
seggt, segpred = model.forward(data, False)
print(time.time() - tic)
tic = time.time()
seggt = seggt.data.cpu().numpy()
segpred = segpred.data.cpu().numpy()
label_trues.append(seggt)
label_preds.append(segpred)
visuals = model.get_visuals(i)
img_path = data['imgpath']
print('process image... %s' % img_path)
visualizer.save_images(webpage, visuals, img_path)
metrics0 = util.label_accuracy_score(label_trues,
label_preds,
n_class=opt.label_nc,
returniu=True)
metrics = np.array(metrics0[:4])
metrics *= 100
print('''\
Accuracy: {0}
Accuracy Class: {1}
Mean IU: {2}
FWAV Accuracy: {3}'''.format(*metrics))
webpage.save()
def train_epoch(self):
self.model.train()
n_class = max(get_label_classes())
train_loss = 0
for batch_idx, (data, target,
_) in tqdm.tqdm(enumerate(self.train_loader),
total=len(self.train_loader),
desc='Train epoch=%d' % self.epoch,
ncols=80,
leave=False):
iteration = batch_idx + self.epoch * len(self.train_loader)
if self.iteration != 0 and (iteration - 1) != self.iteration:
continue # for resuming
self.iteration = iteration
# if self.iteration % self.interval_validate == 0:
# self.validate()
assert self.model.training
self.adjust_learning_rate(self.optim, iteration, args=self.args)
if self.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
self.optim.zero_grad()
score = self.model(data, self.input_size)
loss = CrossEntropyLoss2d(score, target)
loss /= len(data)
if np.isnan(float(loss.data[0])):
raise ValueError('loss is nan while training')
loss.backward()
self.optim.step()
train_loss += loss.data[0]
metrics = []
lbl_pred = score.data.max(1)[1].cpu().numpy()[:, :, :]
lbl_true = target.data.cpu().numpy()
for lt, lp in zip(lbl_true, lbl_pred):
acc, acc_cls, mean_iu, fwavacc = \
label_accuracy_score([lt], [lp], n_class=n_class)
metrics.append((acc, acc_cls, mean_iu, fwavacc))
metrics = np.mean(metrics, axis=0)
#visualize
if self.iteration % 100 == 0:
metrics = {
'loss': train_loss / 100,
'acc': metrics[0],
'acc_cls': metrics[1],
'mean_iu': metrics[2],
'fwavacc': metrics[3]
}
self.train_loss_list.append(metrics)
# print('train loss: %.4f (epoch: %d, step: %d)' % \
# (metrics[-1]['loss'],self.epoch, self.iteration%len(self.train_loader))
# image = data[0].data.cpu()
# image[0] = image[0] + 122.67891434
# image[1] = image[1] + 116.66876762
# image[2] = image[2] + 104.00698793
# step = self.iteration % len(self.train_loader)
# title = 'input: (epoch: %d, step: %d)' % (self.epoch,step)
# vis.image(image, win=win1, env='fcn', opts=dict(title=title))
# title = 'output (epoch: %d, step: %d)' % (self.epoch,step)
# vis.image(index2rgb(lbl_pred[0]),
# win=win2, env='fcn', opts=dict(title=title))
# title = 'target (epoch: %d, step: %d)' % (self.epoch,step)
# vis.image(index2rgb(lbl_true[0]),
# win=win3, env='fcn', opts=dict(title=title))
# epoch_loss = train_loss / 100
# x = np.arange(1, len(epoch_loss) + 1, 1)
# title = 'loss (epoch: %d, step: %d)' % (self.epoch,step)
# vis.line(np.array(epoch_loss), x, env='fcn', win=win0,
# opts=dict(title=title))
train_loss = 0
if self.iteration >= self.max_iter:
break
def main_task():
# define params
opt = BaseOptions().parse()
iter_path = os.path.join(opt.checkpoints_dir, 'iter.txt')
ioupath_path = os.path.join(opt.checkpoints_dir, 'MIoU.txt')
# load training data
if opt.continue_train:
try:
start_epoch, epoch_iter = np.loadtxt(iter_path,
delimiter=',',
dtype=int)
except:
start_epoch, epoch_iter = 1, 0
try:
best_iou = np.loadtxt(ioupath_path, dtype=float)
except:
best_iou = 0.
else:
start_epoch, epoch_iter = 1, 0
best_iou = 0.
os.environ["CUDA_VISIBLE_DEVICES"] = str(opt.gpu_ids[0])
# define data mode
data_loader = CustomDatasetDataLoader()
data_loader.initialize(opt)
dataset, dataset_val = data_loader.load_data()
dataset_size = len(dataset)
# define model
model = Deeplab_Solver(opt)
total_steps = (start_epoch - 1) * dataset_size + epoch_iter
print("starting training model......")
for epoch in range(start_epoch, opt.nepochs):
if epoch != start_epoch:
epoch_iter = epoch_iter % dataset_size
# for train
opt.isTrain = True
model.model.train()
for i, data in enumerate(dataset, start=epoch_iter):
total_steps += opt.batchSize
epoch_iter += opt.batchSize
# keep time to watch how times each one epoch
epoch_start_time = time.time()
# forward and backward pass
model.forward(data, isTrain=True)
model.backward(total_steps,
opt.nepochs * dataset_size * opt.batchSize + 1)
# save latest model
if total_steps % opt.save_latest_freq == 0:
print('saving the latest model (epoch %d, total_steps %d)' %
(epoch, total_steps))
model.save('latest')
np.savetxt(iter_path, (epoch, epoch_iter),
delimiter=',',
fmt='%d')
if model.trainingavgloss < 0.010:
break
# for eval
opt.isTrain = False
model.model.eval()
if dataset_val != None:
label_trues, labels_preds = [], []
for i, data in enumerate(dataset_val):
seggt, segpred = model.forward(data, isTrain=False)
seggt = seggt.data.cpu().numpy()
segpred = segpred.data.cpu().numpy()
label_trues.append(seggt)
labels_preds.append(segpred)
metrics = util.label_accuracy_score(label_trues,
labels_preds,
n_class=opt.label_nc)
metrics *= 100
print('''\
Validation:
Accuracy: {0}
AccuracyClass: {1}
MeanIOU: {2}
FWAVAccuracy: {3}
'''.format(*metrics))
# save model for best
if metrics[2] > best_iou:
best_iou = metrics[2]
model.save('best')
print('end of epoch %d / %d \t Time Taken: %d sec' %
(epoch + 1, opt.nepochs, time.time() - epoch_start_time))
def validate(self):
training = self.model.training
self.model.eval()
n_class = max(get_label_classes())
val_loss = 0
visualizations = []
label_trues, label_preds = [], []
for batch_idx, (data, target) in tqdm.tqdm(
enumerate(self.val_loader), total=len(self.val_loader),
desc='Valid iteration=%d' % self.iteration, ncols=80,
leave=False):
if self.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
score = self.model(data)
loss = CrossEntropyLoss2d(score, target)
if np.isnan(float(loss.data[0])):
raise ValueError('loss is nan while validating')
val_loss += float(loss.data[0]) / len(data)
imgs = data.data.cpu()
lbl_pred = score.data.max(1)[1].cpu().numpy()[:, :, :]
lbl_true = target.data.cpu()
for img, lt, lp in zip(imgs, lbl_true, lbl_pred):
img, lt = self.val_loader.dataset.untransform(img, lt)
label_trues.append(lt)
label_preds.append(lp)
metrics = label_accuracy_score(label_trues, label_preds, n_class)
val_loss /= len(self.val_loader)
mean_iu = metrics[2]
is_best = mean_iu > self.best_mean_iu
if is_best:
self.best_mean_iu = mean_iu
filename = ('%s/epoch-%d.pth' \
% (self.checkpoint_dir, self.epoch))
torch.save({
'epoch': self.epoch,
'iteration': self.iteration,
'arch': self.model.__class__.__name__,
'optim_state_dict': self.optim.state_dict(),
'model_state_dict': self.model.state_dict(),
'best_mean_iu': self.best_mean_iu,
}, filename)
if is_best:
shutil.copy(osp.join(self.out, filename),
osp.join(self.out, 'model_best.pth.tar'))
metrics ={
'loss':loss.data[0],
'acc' : metrics[0],
'acc_cls':metrics[1],
'mean_iu':metrics[2],
'fwavacc':metrics[3]
}
self.val_loss_list.append(metrics)
if training:
self.model.train()