def cv(batch_idx): model.eval() test_num=len(val_dataset) eval_acc = 0 eval_acc_cls = 0 eval_mean_iu = 0 eval_fwavacc = 0 for step, (imgs, sem_labels, ins_labels) in (enumerate(val_dataloader)): # for step in tqdm(range(500)): # (imgs, sem_labels, ins_labels)=next(iter(train_dataloader)) test_data, test_label, _ = imgs[0], sem_labels[0], ins_labels pred = predict(test_data.cuda()) label_true = test_label.data.cpu().numpy() acc, acc_cls, mean_iu, fwavacc = label_accuracy_score(label_true, pred, 9) # break # for lbt, lbp in zip(label_true, pred): # acc, acc_cls, mean_iu, fwavacc = label_accuracy_score(lbt, lbp, 9) eval_acc += acc eval_acc_cls += acc_cls eval_mean_iu += mean_iu eval_fwavacc += fwavacc if step % np.floor(test_num*0.2)==0 : print("进度:",step/test_num); if step==test_num: # Tensorboard info = {'acc': eval_acc/test_num, 'acc_cls': eval_acc_cls/test_num, 'mean_iu': eval_mean_iu/test_num, 'fwavacc': eval_fwavacc/test_num} for tag, value in info.items(): logger.scalar_summary(tag, value, batch_idx + 1) print(eval_acc/test_num,eval_acc_cls/test_num,eval_mean_iu/test_num,eval_fwavacc/test_num) break model.train()
# end of epoch model.model.eval() if dataset_val != None: label_trues, label_preds = [], [] for i, data in enumerate(dataset_val): with torch.no_grad(): seggt, segpred = model.forward(data, False) seggt = seggt.data.cpu().numpy() segpred = segpred.data.cpu().numpy() label_trues.append(seggt) label_preds.append(segpred) metrics = util.label_accuracy_score(label_trues, label_preds, n_class=opt.label_nc) metrics = np.array(metrics) metrics *= 100 print('''\ Validation: Accuracy: {0} Accuracy Class: {1} Mean IU: {2} FWAV Accuracy: {3}'''.format(*metrics)) model.update_tensorboard(metrics, total_steps) iter_end_time = time.time() print('End of epoch %d / %d \t Time Taken: %d sec' % (epoch + 1, opt.nepochs, time.time() - epoch_start_time)) if metrics[2] > best_iou:
if i >= opt.how_many and opt.how_many != 0: break seggt, segpred = model.forward(data, False) print(time.time() - tic) tic = time.time() seggt = seggt.data.cpu().numpy() segpred = segpred.data.cpu().numpy() label_trues.append(seggt) label_preds.append(segpred) visuals = model.get_visuals(i) img_path = data['imgpath'] print('process image... %s' % img_path) visualizer.save_images(webpage, visuals, img_path) metrics0 = util.label_accuracy_score(label_trues, label_preds, n_class=opt.label_nc, returniu=True) metrics = np.array(metrics0[:4]) metrics *= 100 print('''\ Accuracy: {0} Accuracy Class: {1} Mean IU: {2} FWAV Accuracy: {3}'''.format(*metrics)) webpage.save()
def train_epoch(self): self.model.train() n_class = max(get_label_classes()) train_loss = 0 for batch_idx, (data, target, _) in tqdm.tqdm(enumerate(self.train_loader), total=len(self.train_loader), desc='Train epoch=%d' % self.epoch, ncols=80, leave=False): iteration = batch_idx + self.epoch * len(self.train_loader) if self.iteration != 0 and (iteration - 1) != self.iteration: continue # for resuming self.iteration = iteration # if self.iteration % self.interval_validate == 0: # self.validate() assert self.model.training self.adjust_learning_rate(self.optim, iteration, args=self.args) if self.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data), Variable(target) self.optim.zero_grad() score = self.model(data, self.input_size) loss = CrossEntropyLoss2d(score, target) loss /= len(data) if np.isnan(float(loss.data[0])): raise ValueError('loss is nan while training') loss.backward() self.optim.step() train_loss += loss.data[0] metrics = [] lbl_pred = score.data.max(1)[1].cpu().numpy()[:, :, :] lbl_true = target.data.cpu().numpy() for lt, lp in zip(lbl_true, lbl_pred): acc, acc_cls, mean_iu, fwavacc = \ label_accuracy_score([lt], [lp], n_class=n_class) metrics.append((acc, acc_cls, mean_iu, fwavacc)) metrics = np.mean(metrics, axis=0) #visualize if self.iteration % 100 == 0: metrics = { 'loss': train_loss / 100, 'acc': metrics[0], 'acc_cls': metrics[1], 'mean_iu': metrics[2], 'fwavacc': metrics[3] } self.train_loss_list.append(metrics) # print('train loss: %.4f (epoch: %d, step: %d)' % \ # (metrics[-1]['loss'],self.epoch, self.iteration%len(self.train_loader)) # image = data[0].data.cpu() # image[0] = image[0] + 122.67891434 # image[1] = image[1] + 116.66876762 # image[2] = image[2] + 104.00698793 # step = self.iteration % len(self.train_loader) # title = 'input: (epoch: %d, step: %d)' % (self.epoch,step) # vis.image(image, win=win1, env='fcn', opts=dict(title=title)) # title = 'output (epoch: %d, step: %d)' % (self.epoch,step) # vis.image(index2rgb(lbl_pred[0]), # win=win2, env='fcn', opts=dict(title=title)) # title = 'target (epoch: %d, step: %d)' % (self.epoch,step) # vis.image(index2rgb(lbl_true[0]), # win=win3, env='fcn', opts=dict(title=title)) # epoch_loss = train_loss / 100 # x = np.arange(1, len(epoch_loss) + 1, 1) # title = 'loss (epoch: %d, step: %d)' % (self.epoch,step) # vis.line(np.array(epoch_loss), x, env='fcn', win=win0, # opts=dict(title=title)) train_loss = 0 if self.iteration >= self.max_iter: break
def main_task(): # define params opt = BaseOptions().parse() iter_path = os.path.join(opt.checkpoints_dir, 'iter.txt') ioupath_path = os.path.join(opt.checkpoints_dir, 'MIoU.txt') # load training data if opt.continue_train: try: start_epoch, epoch_iter = np.loadtxt(iter_path, delimiter=',', dtype=int) except: start_epoch, epoch_iter = 1, 0 try: best_iou = np.loadtxt(ioupath_path, dtype=float) except: best_iou = 0. else: start_epoch, epoch_iter = 1, 0 best_iou = 0. os.environ["CUDA_VISIBLE_DEVICES"] = str(opt.gpu_ids[0]) # define data mode data_loader = CustomDatasetDataLoader() data_loader.initialize(opt) dataset, dataset_val = data_loader.load_data() dataset_size = len(dataset) # define model model = Deeplab_Solver(opt) total_steps = (start_epoch - 1) * dataset_size + epoch_iter print("starting training model......") for epoch in range(start_epoch, opt.nepochs): if epoch != start_epoch: epoch_iter = epoch_iter % dataset_size # for train opt.isTrain = True model.model.train() for i, data in enumerate(dataset, start=epoch_iter): total_steps += opt.batchSize epoch_iter += opt.batchSize # keep time to watch how times each one epoch epoch_start_time = time.time() # forward and backward pass model.forward(data, isTrain=True) model.backward(total_steps, opt.nepochs * dataset_size * opt.batchSize + 1) # save latest model if total_steps % opt.save_latest_freq == 0: print('saving the latest model (epoch %d, total_steps %d)' % (epoch, total_steps)) model.save('latest') np.savetxt(iter_path, (epoch, epoch_iter), delimiter=',', fmt='%d') if model.trainingavgloss < 0.010: break # for eval opt.isTrain = False model.model.eval() if dataset_val != None: label_trues, labels_preds = [], [] for i, data in enumerate(dataset_val): seggt, segpred = model.forward(data, isTrain=False) seggt = seggt.data.cpu().numpy() segpred = segpred.data.cpu().numpy() label_trues.append(seggt) labels_preds.append(segpred) metrics = util.label_accuracy_score(label_trues, labels_preds, n_class=opt.label_nc) metrics *= 100 print('''\ Validation: Accuracy: {0} AccuracyClass: {1} MeanIOU: {2} FWAVAccuracy: {3} '''.format(*metrics)) # save model for best if metrics[2] > best_iou: best_iou = metrics[2] model.save('best') print('end of epoch %d / %d \t Time Taken: %d sec' % (epoch + 1, opt.nepochs, time.time() - epoch_start_time))
def validate(self): training = self.model.training self.model.eval() n_class = max(get_label_classes()) val_loss = 0 visualizations = [] label_trues, label_preds = [], [] for batch_idx, (data, target) in tqdm.tqdm( enumerate(self.val_loader), total=len(self.val_loader), desc='Valid iteration=%d' % self.iteration, ncols=80, leave=False): if self.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data, volatile=True), Variable(target) score = self.model(data) loss = CrossEntropyLoss2d(score, target) if np.isnan(float(loss.data[0])): raise ValueError('loss is nan while validating') val_loss += float(loss.data[0]) / len(data) imgs = data.data.cpu() lbl_pred = score.data.max(1)[1].cpu().numpy()[:, :, :] lbl_true = target.data.cpu() for img, lt, lp in zip(imgs, lbl_true, lbl_pred): img, lt = self.val_loader.dataset.untransform(img, lt) label_trues.append(lt) label_preds.append(lp) metrics = label_accuracy_score(label_trues, label_preds, n_class) val_loss /= len(self.val_loader) mean_iu = metrics[2] is_best = mean_iu > self.best_mean_iu if is_best: self.best_mean_iu = mean_iu filename = ('%s/epoch-%d.pth' \ % (self.checkpoint_dir, self.epoch)) torch.save({ 'epoch': self.epoch, 'iteration': self.iteration, 'arch': self.model.__class__.__name__, 'optim_state_dict': self.optim.state_dict(), 'model_state_dict': self.model.state_dict(), 'best_mean_iu': self.best_mean_iu, }, filename) if is_best: shutil.copy(osp.join(self.out, filename), osp.join(self.out, 'model_best.pth.tar')) metrics ={ 'loss':loss.data[0], 'acc' : metrics[0], 'acc_cls':metrics[1], 'mean_iu':metrics[2], 'fwavacc':metrics[3] } self.val_loss_list.append(metrics) if training: self.model.train()