def make_subplot(patch, pred, truth, title_str): plt.imshow(patch) pred = pred[x:x + window_size, y:y + window_size] plt.title('{} IoU={:.2f}'.format(title_str, util_functions.iou_metric(truth, pred) * 100)) plt.box(True) plt.xticks([]) plt.yticks([])
def make_subplot(patch, pred, truth): plt.imshow(patch) pred = pred[x:x + window_size, y:y + window_size] iou = util_functions.iou_metric(truth, pred) * 100 plt.title('IoU={:.2f}'.format(iou)) plt.box(True) plt.xticks([]) plt.yticks([]) return iou
def make_subplot(patch, pred, truth, title_str): plt.imshow(patch) pred = pred[x:x + window_size, y:y + window_size] fn = np.sum(truth - pred == 1) / (window_size**2) * 100 fp = np.sum(pred - truth == 1) / (window_size**2) * 100 plt.title('{} IoU={:.2f}\nFN={:.2f},FP={:.2f}'.format( title_str, util_functions.iou_metric(truth, pred) * 100, fn, fp)) plt.box(True) plt.xticks([]) plt.yticks([])
def evaluate(self, rgb_list, gt_list, rgb_dir, gt_dir, input_size, tile_size, batch_size, img_mean, model_dir, gpu=None, save_result=True, save_result_parent_dir=None, show_figure=False, verb=True, ds_name='default', load_epoch_num=None, best_model=True): if show_figure: import matplotlib.pyplot as plt if save_result: self.model_name = model_dir.split('/')[-1] if save_result_parent_dir is None: score_save_dir = os.path.join(uabRepoPaths.evalPath, self.model_name, ds_name) else: score_save_dir = os.path.join(uabRepoPaths.evalPath, save_result_parent_dir, self.model_name, ds_name) if not os.path.exists(score_save_dir): os.makedirs(score_save_dir) with open(os.path.join(score_save_dir, 'result.txt'), 'w'): pass iou_record = [] iou_return = {} for file_name, file_name_truth in zip(rgb_list, gt_list): tile_size = ersa_utils.load_file( os.path.join(rgb_dir[0], file_name[0])).shape[:2] tile_name = file_name_truth.split('_')[0] if verb: print('Evaluating {} ... '.format(tile_name)) start_time = time.time() # prepare the reader reader = uabDataReader.ImageLabelReader( gtInds=[0], dataInds=[0], nChannels=3, parentDir=rgb_dir, chipFiles=[file_name], chip_size=input_size, tile_size=tile_size, batchSize=batch_size, block_mean=img_mean, overlap=self.get_overlap(), padding=np.array( (self.get_overlap() / 2, self.get_overlap() / 2)), isTrain=False) rManager = reader.readManager # run the model pred = self.run(pretrained_model_dir=model_dir, test_reader=rManager, tile_size=tile_size, patch_size=input_size, gpu=gpu, load_epoch_num=load_epoch_num, best_model=best_model, tile_name=tile_name) truth_label_img = imageio.imread( os.path.join(gt_dir, file_name_truth)) iou = util_functions.iou_metric(truth_label_img, pred, divide_flag=True) iou_record.append(iou) iou_return[tile_name] = iou duration = time.time() - start_time if verb: print('{} mean IoU={:.3f}, duration: {:.3f}'.format( tile_name, iou[0] / iou[1], duration)) # save results if save_result: pred_save_dir = os.path.join(score_save_dir, 'pred') if not os.path.exists(pred_save_dir): os.makedirs(pred_save_dir) imageio.imsave(os.path.join(pred_save_dir, tile_name + '.png'), pred.astype(np.uint8)) with open(os.path.join(score_save_dir, 'result.txt'), 'a+') as file: file.write('{} {}\n'.format(tile_name, iou)) if show_figure: plt.figure(figsize=(12, 4)) ax1 = plt.subplot(121) ax1.imshow(truth_label_img) plt.title('Truth') ax2 = plt.subplot(122, sharex=ax1, sharey=ax1) ax2.imshow(pred) plt.title('pred') plt.suptitle('{} Results on {} IoU={:3f}'.format( self.model_name, file_name_truth.split('_')[0], iou[0] / iou[1])) plt.show() iou_record = np.array(iou_record) mean_iou = np.sum(iou_record[:, 0]) / np.sum(iou_record[:, 1]) print('Overall mean IoU={:.3f}'.format(mean_iou)) if save_result: if save_result_parent_dir is None: score_save_dir = os.path.join(uabRepoPaths.evalPath, self.model_name, ds_name) else: score_save_dir = os.path.join(uabRepoPaths.evalPath, save_result_parent_dir, self.model_name, ds_name) with open(os.path.join(score_save_dir, 'result.txt'), 'a+') as file: file.write('{}'.format(mean_iou)) return iou_return
pad = model.get_overlap() image_pred = uabUtilreader.un_patchify_shrink( result, [tile_size[0] + pad, 4576 + pad], [5000, 4576], input_size, [input_size[0] - pad, input_size[1] - pad], overlap=pad) pred_overall = util_functions.get_pred_labels(image_pred) * 1 pred_overall = np.roll(pred_overall, shift=slide_step, axis=1) pred_overall = pred_overall[:, 1000:-1000] #pred_overall = pred_overall[:, shift_max-slide_step:-slide_step-1] truth_label_img = imageio.imread( os.path.join(parent_dir_truth, file_name_truth)) #truth_label_img = np.roll(truth_label_img, -slide_step, axis=1) truth_label_img = truth_label_img[:, :4576] truth_label_img = truth_label_img[:, 1000:-1000] iou = util_functions.iou_metric(truth_label_img, pred_overall, divide_flag=True) duration = time.time() - start_time if iou[1] != 0: print('{} mean IoU={:.3f}, duration: {:.3f}'.format( tile_name, iou[0] / iou[1], duration)) iou_record.append(iou) else: print('{} mean IoU=NA, duration: {:.3f}'.format( tile_name, duration)) pred_save_dir = os.path.join(score_save_dir, 'pred') if not os.path.exists(pred_save_dir): os.makedirs(pred_save_dir) imageio.imsave(os.path.join(pred_save_dir, tile_name + '.png'), pred_overall.astype(np.uint8))
sess.run(init) model.load(model_dir, sess) result = model.test('X', sess, test_reader) image_pred = uabUtilreader.un_patchify_shrink(result, [tile_size[0] + model.get_overlap(), tile_size[1] + model.get_overlap()], tile_size, input_size, [input_size[0] - model.get_overlap(), input_size[1] - model.get_overlap()], overlap=model.get_overlap()) pred = util_functions.get_pred_labels(image_pred) * 255 gt = imageio.imread(os.path.join(gt_dir, '{}_GT.png'.format(tile_ids[tile_cnt].replace('_', '-')))) cfm_norm = image_pred[:, :, 1]/np.sum(image_pred, axis=2) cfm_norm = cfm_norm.astype(np.uint8) #imageio.imsave(os.path.join(data_path, '{}_pred.png'.format(tile_sizes[tile_cnt])), pred) iou = iou_metric(gt, pred/255) # view result lt = imageio.imread(os.path.join(data_path, large_tile[0])) plt.figure(figsize=(15, 6)) ax1 = plt.subplot(131) plt.axis('off') plt.imshow(lt) ax2 = plt.subplot(132, sharex=ax1, sharey=ax1) plt.imshow(pred) plt.title('Pred IoU={:.3f}'.format(iou)) plt.axis('off') ax3 = plt.subplot(133, sharex=ax1, sharey=ax1) plt.imshow(gt) plt.title('GT') plt.axis('off')
img = imageio.imread(os.path.join(data_dir, city_name + '_RGB.tif'))[:chip_size, :chip_size, :] gt = imageio.imread(os.path.join( data_dir, city_name + '_GT.tif'))[:chip_size, :chip_size] / 255 unet_pred = imageio.imread(os.path.join( unet_pred_dir, city_name + '.png'))[:chip_size, :chip_size] / 255 frrn_pred = imageio.imread(os.path.join( frrn_pred_dir, city_name + '.png'))[:chip_size, :chip_size] / 255 deeplab_pred = imageio.imread( os.path.join(deeplab_pred_dir, city_name + '.png'))[:chip_size, :chip_size] / 255 fpn_pred = imageio.imread(os.path.join( fpn_pred_dir, city_name + '.png'))[:chip_size, :chip_size] / 255 unet_iou = iou_metric(gt, unet_pred, truth_val=1) * 100 frrn_iou = iou_metric(gt, frrn_pred, truth_val=1) * 100 deeplab_iou = iou_metric(gt, deeplab_pred, truth_val=1) * 100 fpn_iou = iou_metric(gt, fpn_pred, truth_val=1) * 100 plt.figure(figsize=(16, 4)) ax1 = plt.subplot(151) plt.imshow(img) plt.axis('off') plt.title(city_name) plt.subplot(152, sharex=ax1, sharey=ax1) plt.imshow(unet_pred - gt, cmap='bwr') plt.xticks([], []) plt.yticks([], []) plt.title('U-Net {:.2f}'.format(unet_iou)) plt.subplot(153, sharex=ax1, sharey=ax1)
import os import imageio from glob import glob from tqdm import tqdm import util_functions pred_dir = r'/media/batcave/personal/huang.bohao/CA' gt_dir = r'/media/ei-edl01/data/uab_datasets/sp/CA/data/test' pred_imgs = sorted(glob(os.path.join(pred_dir, '*.png'))) A = 0 B = 0 for pred_img in tqdm(pred_imgs): img_name = os.path.basename(pred_img) tile_id = img_name.split('_')[0] gt_img_name = '{}_GT.png'.format(tile_id) pred = imageio.imread(pred_img) / 255 gt = imageio.imread(os.path.join(gt_dir, gt_img_name)) a, b = util_functions.iou_metric(gt, pred, truth_val=1, divide_flag=True) A += a B += b print(A / B)
ref_dir, fig_name)) pred_map = pred_map + pred pred_map = pred_map / len(patch_select) pred_map = (pred_map > 0.5).astype(np.int) truth_dir = r'/media/ei-edl01/data/uab_datasets/spca/data/Original_Tiles' gt = ersa_utils.load_file( os.path.join(truth_dir, '{}{}_GT.png'.format(city_name, city_id))) gt = gt[:, :4576] gt = gt[:, 1000:-1000] a, b = util_functions.iou_metric(gt, pred_map, divide_flag=True) iou_record.append([a, b]) cnt += 1 print('{}{}: {:.4f}'.format(city_name, city_id, a / b)) except OSError: continue ersa_utils.save_file(save_file_name, np.array(iou_record)) '''save_file_name = os.path.join(task_dir, 'iou_record_ref.npy') iou_record = np.zeros((25, 2)) ref_dir = r'/hdd/Results/inria_decay/UnetCrop_inria_decay_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60.0_DR0.1_SFN32/inria/pred' cnt = 0 for city_name in city_list: for city_id in range(1, 6):
# diff pred_diff = pred - pred_stitch gt_window = gt[92:chip_size_large - 92, 92:chip_size_large - 92] gt_plot = tile[92:chip_size_large-92,92:chip_size_large-92,:] + img_mean gt_cmp_large = 255 + np.zeros((gt_window.shape[0], gt_window.shape[1], 3), dtype=np.float32) gt_cmp_small = 255 + np.zeros((gt_window.shape[0], gt_window.shape[1], 3), dtype=np.float32) gt_plot = util_functions.add_mask(gt_plot, gt_window, [0, 0, 0], 255) gt_cmp_large = util_functions.add_mask(gt_cmp_large, gt_window, [0, 0, 0], 255) gt_cmp_large = util_functions.add_mask(gt_cmp_large, gt_window/255-pred, [255, 0, 0], 1) gt_cmp_large = util_functions.add_mask(gt_cmp_large, gt_window/255-pred, [0, 255, 0], -1) gt_cmp_small = util_functions.add_mask(gt_cmp_small, gt_window, [0, 0, 0], 255) gt_cmp_small = util_functions.add_mask(gt_cmp_small, gt_window/255-pred_stitch, [255, 0, 0], 1) gt_cmp_small = util_functions.add_mask(gt_cmp_small, gt_window/255-pred_stitch, [0, 255, 0], -1) # compute iou iou_pred = util_functions.iou_metric(gt_window[6:-6, 6:-6]/255, pred[6:-6, 6:-6]) * 100 iou_stitch = util_functions.iou_metric(gt_window[6:-6, 6:-6]/255, pred_stitch[6:-6, 6:-6]) * 100 if not (np.isnan(iou_pred) or np.isnan(iou_stitch)): large_ious.append(iou_pred) small_ious.append(iou_stitch) # show plt.figure(figsize=(13, 5)) plt.subplot(131) plt.xticks([], []) plt.yticks([], []) plt.imshow(gt_plot/255) plt.subplot(132) plt.xticks([], []) plt.yticks([], [])
[tile_size[0] + model.get_overlap(), tile_size[1] + model.get_overlap()], tile_size, input_size, [input_size[0] - model.get_overlap(), input_size[1] - model.get_overlap()], overlap=model.get_overlap()) image_pred_ct = uabUtilreader.un_patchify_shrink( result2, [tile_size[0] + model.get_overlap(), tile_size[1] + model.get_overlap()], tile_size, input_size, [input_size[0] - model.get_overlap(), input_size[1] - model.get_overlap()], overlap=model.get_overlap()) pred = util_functions.get_pred_labels(image_pred) * 255 pred_ct = util_functions.get_pred_labels(image_pred_ct) * 255 iou = iou_metric(pred_ct, pred, truth_val=255) # view result lt = imageio.imread(os.path.join(data_path, large_tile[0])) plt.figure(figsize=(15, 6)) ax1 = plt.subplot(131) plt.axis('off') plt.imshow(lt) ax2 = plt.subplot(132, sharex=ax1, sharey=ax1) plt.imshow(pred) plt.title('Pred IoU={:.3f}'.format(iou)) plt.axis('off') ax3 = plt.subplot(133, sharex=ax1, sharey=ax1) plt.imshow(pred_ct) plt.title('\'GT\'') plt.axis('off')
region_num = 5 for i in range(region_num): error, x, y = error_record_loo[-1 - i, :] x = int(x) y = int(y) emi_base = get_error_mask_img( rgb_img[x:x + error_size, y:y + error_size, :], gt_img[x:x + error_size, y:y + error_size], mask_base[x:x + error_size, y:y + error_size]) fig = plt.figure(figsize=(5, 5.5)) #plt.subplot(121) plt.imshow(emi_base) plt.axis('off') '''plt.subplot(122) plt.imshow(rgb_img_copy[x:x+error_size, y:y+error_size, :]) plt.axis('off')''' title_str = 'IoU={:.3f}'.format( iou_metric(gt_img[x:x + error_size, y:y + error_size], pred_img_base[x:x + error_size, y:y + error_size], truth_val=1) * 100) title_str += ' {}:({},{})'.format(city_name, x, y) plt.title(title_str) plt.tight_layout() plt.savefig( os.path.join(img_dir, '{}_{}_{}_boning.png'.format(city_name, x, y))) plt.close(fig) # plt.show()
gt_img[x:x + 500, y:y + 500], mask_xr[x:x + 500, y:y + 500]) emi_base = get_error_mask_img(rgb_img_copy2[x:x + 500, y:y + 500, :], gt_img[x:x + 500, y:y + 500], mask_base[x:x + 500, y:y + 500]) emi_auto = get_error_mask_img(rgb_img_copy3[x:x + 500, y:y + 500, :], gt_img[x:x + 500, y:y + 500], mask_auto[x:x + 500, y:y + 500]) plt.figure(figsize=(15, 5.5)) plt.subplot(131) plt.imshow(emi_xr) plt.axis('off') plt.title('XRegion IoU={:.3f}'.format( iou_metric(gt_img[x:x + 500, y:y + 500], pred_img_xr[x:x + 500, y:y + 500], truth_val=255) * 100)) plt.subplot(132) plt.imshow(emi_auto) plt.axis('off') plt.title('Auto IoU={:.3f}'.format( iou_metric(gt_img[x:x + 500, y:y + 500], pred_img_auto[x:x + 500, y:y + 500], truth_val=255) * 100)) plt.subplot(133) plt.imshow(emi_base) plt.axis('off') plt.title('Finetune IoU={:.3f}'.format( iou_metric(gt_img[x:x + 500, y:y + 500], pred_img_base[x:x + 500, y:y + 500], truth_val=255) * 100))
rgb_img_copy = np.copy(rgb_img) mask_loo = get_error_mask(pred_img_loo, gt_img) mask_xr = get_error_mask(pred_img_xr, gt_img) error_record_loo = get_high_error_region(mask_loo, 500, 400) region_num = 5 for i in range(region_num): error, x, y = error_record_loo[-1-i, :] x = int(x) y = int(y) emi_loo = get_error_mask_img(rgb_img[x:x+500, y:y+500, :], gt_img[x:x+500, y:y+500], mask_loo[x:x+500, y:y+500]) emi_xr = get_error_mask_img(rgb_img_copy[x:x+500, y:y+500, :], gt_img[x:x+500, y:y+500], mask_xr[x:x+500, y:y+500]) plt.figure(figsize=(16, 6)) plt.subplot(131) plt.imshow(emi_loo) plt.axis('off') plt.title('Leave-one out IoU={:.3f}'.format(iou_metric(gt_img[x:x+500, y:y+500], pred_img_loo[x:x+500, y:y+500], truth_val=255)*100)) plt.subplot(132) plt.imshow(emi_xr) plt.axis('off') plt.title('Cross region IoU={:.3f}'.format(iou_metric(gt_img[x:x+500, y:y+500], pred_img_xr[x:x+500, y:y+500], truth_val=255)*100)) plt.subplot(133) plt.imshow(gt_img[x:x+500, y:y+500]) plt.axis('off') plt.title('Ground truth {}({},{})'.format(city_name, x, y)) plt.tight_layout() plt.savefig(os.path.join(img_dir, '{}_{}_{}_deeplab.png'.format(city_name, x, y))) plt.show()