def val(args, model, dataloader):
with torch.no_grad():
precision_record = []
hist = torch.zeros((cfgs.num_classes, cfgs.num_classes))
if torch.cuda.is_available():
hist = hist.cuda()
for i, (data, label) in enumerate(dataloader):
if torch.cuda.is_available() and args.use_gpu:
data = data.cuda()
label = label.cuda()
# get RGB predict image
# label = label.squeeze(1)
t1 = time.time()
predict = model(data)
t2 = time.time()
print('inference time:', t2 - t1)
predict = reverse_one_hot(predict)
if args.losstype == 'dice':
label = reverse_one_hot(label)
precision = compute_global_accuracy(predict, label)
hist += fast_hist(label[0], predict[0], cfgs.num_classes)
precision_record.append(precision)
precision_record = torch.Tensor(precision_record)
precision = torch.mean(precision_record)
miou_list = per_class_iu(hist)
miou_list = miou_list.cpu().numpy()
miou_dict, miou = cal_miou(miou_list, cfgs.LabelFile)
# logger.info('mIoU for validation: %.3f' % miou)
return miou_dict, miou, precision.cpu().numpy()
def main():
# Make output dir if not exists
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
# Make image output dirs
if args.save_output_images > 0:
output_image_dir = args.output_dir + 'image_outputs/'
if not os.path.isdir(output_image_dir):
os.makedirs(output_image_dir)
# Create cityscapes manager class
CS = cityscapes(args.cityscapes_dir)
n_cl = len(CS.classes)
# Get labeled frames for evaluation
label_frames = CS.list_label_frames(args.split)
# GPU runs out of memory, so keep using CPU
# caffe.set_device(args.gpu_id)
# caffe.set_mode_gpu()
# Load FCN-8s network
net = caffe.Net(args.caffemodel_dir + '/deploy.prototxt',
args.caffemodel_dir + 'fcn-8s-cityscapes.caffemodel',
caffe.TEST)
hist_perframe = np.zeros((n_cl, n_cl))
# For all labeled frames
for i, idx in enumerate(label_frames):
if i % 10 == 0:
print('Evaluating: %d/%d' % (i, len(label_frames)))
city = idx.split('_')[0]
# idx is city_shot_frame
label = CS.load_label(args.split, city, idx)
im_file = args.result_dir + '/' + idx + '_leftImg8bit.png'
im = np.array(
Image.open(im_file).resize((label.shape[1], label.shape[2])))
out = segrun(net, CS.preprocess(im))
hist_perframe += fast_hist(label.flatten(), out.flatten(), n_cl)
if args.save_output_images > 0:
label_im = CS.palette(label)
pred_im = CS.palette(out)
scipy.misc.imsave(output_image_dir + '/' + str(i) + '_pred.jpg',
pred_im)
scipy.misc.imsave(output_image_dir + '/' + str(i) + '_gt.jpg',
label_im)
scipy.misc.imsave(output_image_dir + '/' + str(i) + '_input.jpg',
im)
mean_pixel_acc, mean_class_acc, mean_class_iou, per_class_acc, per_class_iou = get_scores(
hist_perframe)
with open(args.output_dir + '/evaluation_results.txt', 'w') as f:
f.write('Mean pixel accuracy: %f\n' % mean_pixel_acc)
f.write('Mean class accuracy: %f\n' % mean_class_acc)
f.write('Mean class IoU: %f\n' % mean_class_iou)
f.write('************ Per class numbers below ************\n')
for i, cl in enumerate(CS.classes):
while len(cl) < 15:
cl = cl + ' '
f.write('%s: acc = %f, iou = %f\n' %
(cl, per_class_acc[i], per_class_iou[i]))
def main():
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
if args.save_output_images > 0:
output_image_dir = args.output_dir + 'image_outputs/'
if not os.path.isdir(output_image_dir):
os.makedirs(output_image_dir)
CS = cityscapes(args.cityscapes_dir)
n_cl = len(CS.classes)
label_frames = CS.list_label_frames(args.split)
print(label_frames)
caffe.set_mode_cpu()
net = caffe.Net(args.caffemodel_dir + '/deploy.prototxt',
args.caffemodel_dir + 'fcn-8s-cityscapes.caffemodel',
caffe.TEST)
hist_perframe = np.zeros((n_cl, n_cl))
for i, idx in enumerate(label_frames):
if i % 10 == 0:
print('Evaluating: %d/%d' % (i, len(label_frames)))
city = idx.split('_')[0]
# idx is city_shot_frame
label = CS.load_label(args.split, city, idx)
im_file = args.result_dir + '/' + idx + '_leftImg8bit.png'
##print(im_file)
im = np.array(Image.open(im_file))
#im = scipy.misc.imresize(im, (label.shape[1], label.shape[2]))
im = np.array(Image.fromarray(im).resize((label.shape[2], label.shape[1])))
out = segrun(net, CS.preprocess(im))
hist_perframe += fast_hist(label.flatten(), out.flatten(), n_cl)
if args.save_output_images > 0:
label_im = CS.palette(label)
pred_im = CS.palette(out)
#scipy.misc.imsave(output_image_dir + '/' + str(i) + '_pred.jpg', pred_im)
#scipy.misc.imsave(output_image_dir + '/' + str(i) + '_gt.jpg', label_im)
#scipy.misc.imsave(output_image_dir + '/' + str(i) + '_input.jpg', im)
imageio.imwrite(output_image_dir + '/' + str(i) + '_pred.jpg', pred_im)
imageio.imwrite(output_image_dir + '/' + str(i) + '_gt.jpg', label_im)
imageio.imwrite(output_image_dir + '/' + str(i) + '_input.jpg', im)
#if i > 3:
# break
mean_pixel_acc, mean_class_acc, mean_class_iou, per_class_acc, per_class_iou = get_scores(hist_perframe)
with open(args.output_dir + '/evaluation_results.txt', 'w') as f:
f.write('Mean pixel accuracy: %f\n' % mean_pixel_acc)
f.write('Mean class accuracy: %f\n' % mean_class_acc)
f.write('Mean class IoU: %f\n' % mean_class_iou)
f.write('************ Per class numbers below ************\n')
for i, cl in enumerate(CS.classes):
while len(cl) < 15:
cl = cl + ' '
f.write('%s: acc = %f, iou = %f\n' % (cl, per_class_acc[i], per_class_iou[i]))
def main():
if not os.path.isdir(args.output_dir):
os.makedirs(args.output_dir)
if args.save_output_images > 0:
output_image_dir = args.output_dir + 'image_outputs/'
if not os.path.isdir(output_image_dir):
os.makedirs(output_image_dir)
CS = cityscapes(args.cityscapes_dir)
n_cl = len(CS.classes)
label_frames = CS.list_label_frames(args.split)
# caffe.set_device(args.gpu_id)
# caffe.set_mode_gpu()
# net = caffe.Net(args.caffemodel_dir + '/deploy.prototxt',
# args.caffemodel_dir + 'fcn-8s-cityscapes.caffemodel',
# caffe.TEST)
hist_perframe = np.zeros((n_cl, n_cl))
for i, idx in enumerate(label_frames):
if i % 10 == 0:
print('Evaluating: %d/%d' % (i, len(label_frames)))
city = idx.split('_')[0]
# idx is city_shot_frame
label = CS.load_label(args.split, city, idx)
im_file = args.result_dir + '/' + idx + '_leftImg8bit.png'
# im = np.array(Image.open(im_file))
# out_path = args.cityscapes_dir + 'gtFine/val/' + city + '/' + idx + '_gtFine_color.png'
# im = scipy.misc.imresize(im, (256, 256))
# im = scipy.misc.imresize(im, (label.shape[1], label.shape[2]))
# out = segrun(net, CS.preprocess(im))
# out = scipy.misc.imresize(out, (label.shape[1], label.shape[2]))
out = ReturnGrayImage(im_file)
hist_perframe += fast_hist(label.flatten(), out.flatten(), n_cl)
mean_pixel_acc, mean_class_acc, mean_class_iou, per_class_acc, per_class_iou = get_scores(hist_perframe)
with open(args.output_dir + '/evaluation_results.txt', 'w') as f:
f.write('Mean pixel accuracy: %f\n' % mean_pixel_acc)
f.write('Mean class accuracy: %f\n' % mean_class_acc)
f.write('Mean class IoU: %f\n' % mean_class_iou)
f.write('************ Per class numbers below ************\n')
for i, cl in enumerate(CS.classes):
while len(cl) < 15:
cl = cl + ' '
f.write('%s: acc = %f, iou = %f\n' % (cl, per_class_acc[i], per_class_iou[i]))
def evaluate_semantic(benchmark_path, result_dir, num_of_classes=11, need_merge_result=False, im_downsample=False, gt_downsample=False):
gt_paths = open(benchmark_path, 'r').read().splitlines()
d_paths = [p.split('\t')[2] for p in gt_paths] # 1 denote wall, 2 denote door, 3 denote room
r_paths = [p.split('\t')[3] for p in gt_paths] # 1 denote wall, 2 denote door, 3 denote room
cw_paths = [p.split('\t')[-1] for p in gt_paths] # 1 denote wall, 2 denote door, 3 denote room, last one denote close wall
im_paths = [os.path.join(result_dir, p.split('/')[-1]) for p in r_paths]
if need_merge_result:
im_paths = [os.path.join(result_dir+'/room', p.split('/')[-1]) for p in r_paths]
im_d_paths = [os.path.join(result_dir+'/door', p.split('/')[-1]) for p in d_paths]
im_cw_paths = [os.path.join(result_dir+'/close_wall', p.split('/')[-1]) for p in cw_paths]
n = len(im_paths)
# n = 1
hist = np.zeros((num_of_classes, num_of_classes))
for i in range(n):
im = imread(im_paths[i], mode='RGB')
if need_merge_result:
im_d = imread(im_d_paths[i], mode='L')
im_cw = imread(im_cw_paths[i], mode='L')
# create fuse semantic label
cw = imread(cw_paths[i], mode='L')
dd = imread(d_paths[i], mode='L')
rr = imread(r_paths[i], mode='RGB')
if im_downsample:
im = imresize(im, (512, 512, 3))
if need_merge_result:
im_d = imresize(im_d, (512, 512))
im_cw = imresize(im_cw, (512, 512))
im_d = im_d / 255
im_cw = im_cw / 255
if gt_downsample:
cw = imresize(cw, (512, 512))
dd = imresize(dd, (512, 512))
rr = imresize(rr, (512, 512, 3))
# normalize
cw = cw / 255
dd = dd / 255
im_ind = rgb2ind(im, color_map=floorplan_fuse_map)
if im_ind.sum()==0:
im_ind = rgb2ind(im+1)
rr_ind = rgb2ind(rr, color_map=floorplan_fuse_map)
if rr_ind.sum()==0:
rr_ind = rgb2ind(rr+1)
if need_merge_result:
im_d = (im_d>0.5).astype(np.uint8)
im_cw = (im_cw>0.5).astype(np.uint8)
im_ind[im_cw==1] = 10
im_ind[im_d ==1] = 9
# merge the label and produce
cw = (cw>0.5).astype(np.uint8)
dd = (dd>0.5).astype(np.uint8)
rr_ind[cw==1] = 10
rr_ind[dd==1] = 9
name = im_paths[i].split('/')[-1]
r_name = r_paths[i].split('/')[-1]
print('Evaluating {}(im) <=> {}(gt)...'.format(name, r_name))
hist += fast_hist(im_ind.flatten(), rr_ind.flatten(), num_of_classes)
print('*'*60)
# overall accuracy
acc = np.diag(hist).sum() / hist.sum()
print('overall accuracy {:.4}'.format(acc))
# per-class accuracy, avoid div zero
acc = np.diag(hist) / (hist.sum(1) + 1e-6)
print('room-type: mean accuracy {:.4}, room-type+bd: mean accuracy {:.4}'.format(np.nanmean(acc[:7]), (np.nansum(acc[:7])+np.nansum(acc[-2:]))/9.))
for t in range(0, acc.shape[0]):
if t not in [7, 8]:
print('room type {}th, accuracy = {:.4}'.format(t, acc[t]))
print('*'*60)
# per-class IU, avoid div zero
iu = np.diag(hist) / (hist.sum(1) + 1e-6 + hist.sum(0) - np.diag(hist))
print('room-type: mean IoU {:.4}, room-type+bd: mean IoU {:.4}'.format(np.nanmean(iu[:7]), (np.nansum(iu[:7])+np.nansum(iu[-2:]))/9.))
for t in range(iu.shape[0]):
if t not in [7,8]: # ignore class 7 & 8
print('room type {}th, IoU = {:.4}'.format(t, iu[t]))