def val(args, model, dataloader): with torch.no_grad(): precision_record = [] hist = torch.zeros((cfgs.num_classes, cfgs.num_classes)) if torch.cuda.is_available(): hist = hist.cuda() for i, (data, label) in enumerate(dataloader): if torch.cuda.is_available() and args.use_gpu: data = data.cuda() label = label.cuda() # get RGB predict image # label = label.squeeze(1) t1 = time.time() predict = model(data) t2 = time.time() print('inference time:', t2 - t1) predict = reverse_one_hot(predict) if args.losstype == 'dice': label = reverse_one_hot(label) precision = compute_global_accuracy(predict, label) hist += fast_hist(label[0], predict[0], cfgs.num_classes) precision_record.append(precision) precision_record = torch.Tensor(precision_record) precision = torch.mean(precision_record) miou_list = per_class_iu(hist) miou_list = miou_list.cpu().numpy() miou_dict, miou = cal_miou(miou_list, cfgs.LabelFile) # logger.info('mIoU for validation: %.3f' % miou) return miou_dict, miou, precision.cpu().numpy()
def main(): # Make output dir if not exists if not os.path.isdir(args.output_dir): os.makedirs(args.output_dir) # Make image output dirs if args.save_output_images > 0: output_image_dir = args.output_dir + 'image_outputs/' if not os.path.isdir(output_image_dir): os.makedirs(output_image_dir) # Create cityscapes manager class CS = cityscapes(args.cityscapes_dir) n_cl = len(CS.classes) # Get labeled frames for evaluation label_frames = CS.list_label_frames(args.split) # GPU runs out of memory, so keep using CPU # caffe.set_device(args.gpu_id) # caffe.set_mode_gpu() # Load FCN-8s network net = caffe.Net(args.caffemodel_dir + '/deploy.prototxt', args.caffemodel_dir + 'fcn-8s-cityscapes.caffemodel', caffe.TEST) hist_perframe = np.zeros((n_cl, n_cl)) # For all labeled frames for i, idx in enumerate(label_frames): if i % 10 == 0: print('Evaluating: %d/%d' % (i, len(label_frames))) city = idx.split('_')[0] # idx is city_shot_frame label = CS.load_label(args.split, city, idx) im_file = args.result_dir + '/' + idx + '_leftImg8bit.png' im = np.array( Image.open(im_file).resize((label.shape[1], label.shape[2]))) out = segrun(net, CS.preprocess(im)) hist_perframe += fast_hist(label.flatten(), out.flatten(), n_cl) if args.save_output_images > 0: label_im = CS.palette(label) pred_im = CS.palette(out) scipy.misc.imsave(output_image_dir + '/' + str(i) + '_pred.jpg', pred_im) scipy.misc.imsave(output_image_dir + '/' + str(i) + '_gt.jpg', label_im) scipy.misc.imsave(output_image_dir + '/' + str(i) + '_input.jpg', im) mean_pixel_acc, mean_class_acc, mean_class_iou, per_class_acc, per_class_iou = get_scores( hist_perframe) with open(args.output_dir + '/evaluation_results.txt', 'w') as f: f.write('Mean pixel accuracy: %f\n' % mean_pixel_acc) f.write('Mean class accuracy: %f\n' % mean_class_acc) f.write('Mean class IoU: %f\n' % mean_class_iou) f.write('************ Per class numbers below ************\n') for i, cl in enumerate(CS.classes): while len(cl) < 15: cl = cl + ' ' f.write('%s: acc = %f, iou = %f\n' % (cl, per_class_acc[i], per_class_iou[i]))
def main(): if not os.path.isdir(args.output_dir): os.makedirs(args.output_dir) if args.save_output_images > 0: output_image_dir = args.output_dir + 'image_outputs/' if not os.path.isdir(output_image_dir): os.makedirs(output_image_dir) CS = cityscapes(args.cityscapes_dir) n_cl = len(CS.classes) label_frames = CS.list_label_frames(args.split) print(label_frames) caffe.set_mode_cpu() net = caffe.Net(args.caffemodel_dir + '/deploy.prototxt', args.caffemodel_dir + 'fcn-8s-cityscapes.caffemodel', caffe.TEST) hist_perframe = np.zeros((n_cl, n_cl)) for i, idx in enumerate(label_frames): if i % 10 == 0: print('Evaluating: %d/%d' % (i, len(label_frames))) city = idx.split('_')[0] # idx is city_shot_frame label = CS.load_label(args.split, city, idx) im_file = args.result_dir + '/' + idx + '_leftImg8bit.png' ##print(im_file) im = np.array(Image.open(im_file)) #im = scipy.misc.imresize(im, (label.shape[1], label.shape[2])) im = np.array(Image.fromarray(im).resize((label.shape[2], label.shape[1]))) out = segrun(net, CS.preprocess(im)) hist_perframe += fast_hist(label.flatten(), out.flatten(), n_cl) if args.save_output_images > 0: label_im = CS.palette(label) pred_im = CS.palette(out) #scipy.misc.imsave(output_image_dir + '/' + str(i) + '_pred.jpg', pred_im) #scipy.misc.imsave(output_image_dir + '/' + str(i) + '_gt.jpg', label_im) #scipy.misc.imsave(output_image_dir + '/' + str(i) + '_input.jpg', im) imageio.imwrite(output_image_dir + '/' + str(i) + '_pred.jpg', pred_im) imageio.imwrite(output_image_dir + '/' + str(i) + '_gt.jpg', label_im) imageio.imwrite(output_image_dir + '/' + str(i) + '_input.jpg', im) #if i > 3: # break mean_pixel_acc, mean_class_acc, mean_class_iou, per_class_acc, per_class_iou = get_scores(hist_perframe) with open(args.output_dir + '/evaluation_results.txt', 'w') as f: f.write('Mean pixel accuracy: %f\n' % mean_pixel_acc) f.write('Mean class accuracy: %f\n' % mean_class_acc) f.write('Mean class IoU: %f\n' % mean_class_iou) f.write('************ Per class numbers below ************\n') for i, cl in enumerate(CS.classes): while len(cl) < 15: cl = cl + ' ' f.write('%s: acc = %f, iou = %f\n' % (cl, per_class_acc[i], per_class_iou[i]))
def main(): if not os.path.isdir(args.output_dir): os.makedirs(args.output_dir) if args.save_output_images > 0: output_image_dir = args.output_dir + 'image_outputs/' if not os.path.isdir(output_image_dir): os.makedirs(output_image_dir) CS = cityscapes(args.cityscapes_dir) n_cl = len(CS.classes) label_frames = CS.list_label_frames(args.split) # caffe.set_device(args.gpu_id) # caffe.set_mode_gpu() # net = caffe.Net(args.caffemodel_dir + '/deploy.prototxt', # args.caffemodel_dir + 'fcn-8s-cityscapes.caffemodel', # caffe.TEST) hist_perframe = np.zeros((n_cl, n_cl)) for i, idx in enumerate(label_frames): if i % 10 == 0: print('Evaluating: %d/%d' % (i, len(label_frames))) city = idx.split('_')[0] # idx is city_shot_frame label = CS.load_label(args.split, city, idx) im_file = args.result_dir + '/' + idx + '_leftImg8bit.png' # im = np.array(Image.open(im_file)) # out_path = args.cityscapes_dir + 'gtFine/val/' + city + '/' + idx + '_gtFine_color.png' # im = scipy.misc.imresize(im, (256, 256)) # im = scipy.misc.imresize(im, (label.shape[1], label.shape[2])) # out = segrun(net, CS.preprocess(im)) # out = scipy.misc.imresize(out, (label.shape[1], label.shape[2])) out = ReturnGrayImage(im_file) hist_perframe += fast_hist(label.flatten(), out.flatten(), n_cl) mean_pixel_acc, mean_class_acc, mean_class_iou, per_class_acc, per_class_iou = get_scores(hist_perframe) with open(args.output_dir + '/evaluation_results.txt', 'w') as f: f.write('Mean pixel accuracy: %f\n' % mean_pixel_acc) f.write('Mean class accuracy: %f\n' % mean_class_acc) f.write('Mean class IoU: %f\n' % mean_class_iou) f.write('************ Per class numbers below ************\n') for i, cl in enumerate(CS.classes): while len(cl) < 15: cl = cl + ' ' f.write('%s: acc = %f, iou = %f\n' % (cl, per_class_acc[i], per_class_iou[i]))
def evaluate_semantic(benchmark_path, result_dir, num_of_classes=11, need_merge_result=False, im_downsample=False, gt_downsample=False): gt_paths = open(benchmark_path, 'r').read().splitlines() d_paths = [p.split('\t')[2] for p in gt_paths] # 1 denote wall, 2 denote door, 3 denote room r_paths = [p.split('\t')[3] for p in gt_paths] # 1 denote wall, 2 denote door, 3 denote room cw_paths = [p.split('\t')[-1] for p in gt_paths] # 1 denote wall, 2 denote door, 3 denote room, last one denote close wall im_paths = [os.path.join(result_dir, p.split('/')[-1]) for p in r_paths] if need_merge_result: im_paths = [os.path.join(result_dir+'/room', p.split('/')[-1]) for p in r_paths] im_d_paths = [os.path.join(result_dir+'/door', p.split('/')[-1]) for p in d_paths] im_cw_paths = [os.path.join(result_dir+'/close_wall', p.split('/')[-1]) for p in cw_paths] n = len(im_paths) # n = 1 hist = np.zeros((num_of_classes, num_of_classes)) for i in range(n): im = imread(im_paths[i], mode='RGB') if need_merge_result: im_d = imread(im_d_paths[i], mode='L') im_cw = imread(im_cw_paths[i], mode='L') # create fuse semantic label cw = imread(cw_paths[i], mode='L') dd = imread(d_paths[i], mode='L') rr = imread(r_paths[i], mode='RGB') if im_downsample: im = imresize(im, (512, 512, 3)) if need_merge_result: im_d = imresize(im_d, (512, 512)) im_cw = imresize(im_cw, (512, 512)) im_d = im_d / 255 im_cw = im_cw / 255 if gt_downsample: cw = imresize(cw, (512, 512)) dd = imresize(dd, (512, 512)) rr = imresize(rr, (512, 512, 3)) # normalize cw = cw / 255 dd = dd / 255 im_ind = rgb2ind(im, color_map=floorplan_fuse_map) if im_ind.sum()==0: im_ind = rgb2ind(im+1) rr_ind = rgb2ind(rr, color_map=floorplan_fuse_map) if rr_ind.sum()==0: rr_ind = rgb2ind(rr+1) if need_merge_result: im_d = (im_d>0.5).astype(np.uint8) im_cw = (im_cw>0.5).astype(np.uint8) im_ind[im_cw==1] = 10 im_ind[im_d ==1] = 9 # merge the label and produce cw = (cw>0.5).astype(np.uint8) dd = (dd>0.5).astype(np.uint8) rr_ind[cw==1] = 10 rr_ind[dd==1] = 9 name = im_paths[i].split('/')[-1] r_name = r_paths[i].split('/')[-1] print('Evaluating {}(im) <=> {}(gt)...'.format(name, r_name)) hist += fast_hist(im_ind.flatten(), rr_ind.flatten(), num_of_classes) print('*'*60) # overall accuracy acc = np.diag(hist).sum() / hist.sum() print('overall accuracy {:.4}'.format(acc)) # per-class accuracy, avoid div zero acc = np.diag(hist) / (hist.sum(1) + 1e-6) print('room-type: mean accuracy {:.4}, room-type+bd: mean accuracy {:.4}'.format(np.nanmean(acc[:7]), (np.nansum(acc[:7])+np.nansum(acc[-2:]))/9.)) for t in range(0, acc.shape[0]): if t not in [7, 8]: print('room type {}th, accuracy = {:.4}'.format(t, acc[t])) print('*'*60) # per-class IU, avoid div zero iu = np.diag(hist) / (hist.sum(1) + 1e-6 + hist.sum(0) - np.diag(hist)) print('room-type: mean IoU {:.4}, room-type+bd: mean IoU {:.4}'.format(np.nanmean(iu[:7]), (np.nansum(iu[:7])+np.nansum(iu[-2:]))/9.)) for t in range(iu.shape[0]): if t not in [7,8]: # ignore class 7 & 8 print('room type {}th, IoU = {:.4}'.format(t, iu[t]))