def make_subplot(patch, pred, truth, title_str):
plt.imshow(patch)
pred = pred[x:x + window_size, y:y + window_size]
plt.title('{} IoU={:.2f}'.format(title_str, util_functions.iou_metric(truth, pred) * 100))
plt.box(True)
plt.xticks([])
plt.yticks([])
def make_subplot(patch, pred, truth):
plt.imshow(patch)
pred = pred[x:x + window_size, y:y + window_size]
iou = util_functions.iou_metric(truth, pred) * 100
plt.title('IoU={:.2f}'.format(iou))
plt.box(True)
plt.xticks([])
plt.yticks([])
return iou
def make_subplot(patch, pred, truth, title_str):
plt.imshow(patch)
pred = pred[x:x + window_size, y:y + window_size]
fn = np.sum(truth - pred == 1) / (window_size**2) * 100
fp = np.sum(pred - truth == 1) / (window_size**2) * 100
plt.title('{} IoU={:.2f}\nFN={:.2f},FP={:.2f}'.format(
title_str,
util_functions.iou_metric(truth, pred) * 100, fn, fp))
plt.box(True)
plt.xticks([])
plt.yticks([])
def evaluate(self,
rgb_list,
gt_list,
rgb_dir,
gt_dir,
input_size,
tile_size,
batch_size,
img_mean,
model_dir,
gpu=None,
save_result=True,
save_result_parent_dir=None,
show_figure=False,
verb=True,
ds_name='default',
load_epoch_num=None,
best_model=True):
if show_figure:
import matplotlib.pyplot as plt
if save_result:
self.model_name = model_dir.split('/')[-1]
if save_result_parent_dir is None:
score_save_dir = os.path.join(uabRepoPaths.evalPath,
self.model_name, ds_name)
else:
score_save_dir = os.path.join(uabRepoPaths.evalPath,
save_result_parent_dir,
self.model_name, ds_name)
if not os.path.exists(score_save_dir):
os.makedirs(score_save_dir)
with open(os.path.join(score_save_dir, 'result.txt'), 'w'):
pass
iou_record = []
iou_return = {}
for file_name, file_name_truth in zip(rgb_list, gt_list):
tile_size = ersa_utils.load_file(
os.path.join(rgb_dir[0], file_name[0])).shape[:2]
tile_name = file_name_truth.split('_')[0]
if verb:
print('Evaluating {} ... '.format(tile_name))
start_time = time.time()
# prepare the reader
reader = uabDataReader.ImageLabelReader(
gtInds=[0],
dataInds=[0],
nChannels=3,
parentDir=rgb_dir,
chipFiles=[file_name],
chip_size=input_size,
tile_size=tile_size,
batchSize=batch_size,
block_mean=img_mean,
overlap=self.get_overlap(),
padding=np.array(
(self.get_overlap() / 2, self.get_overlap() / 2)),
isTrain=False)
rManager = reader.readManager
# run the model
pred = self.run(pretrained_model_dir=model_dir,
test_reader=rManager,
tile_size=tile_size,
patch_size=input_size,
gpu=gpu,
load_epoch_num=load_epoch_num,
best_model=best_model,
tile_name=tile_name)
truth_label_img = imageio.imread(
os.path.join(gt_dir, file_name_truth))
iou = util_functions.iou_metric(truth_label_img,
pred,
divide_flag=True)
iou_record.append(iou)
iou_return[tile_name] = iou
duration = time.time() - start_time
if verb:
print('{} mean IoU={:.3f}, duration: {:.3f}'.format(
tile_name, iou[0] / iou[1], duration))
# save results
if save_result:
pred_save_dir = os.path.join(score_save_dir, 'pred')
if not os.path.exists(pred_save_dir):
os.makedirs(pred_save_dir)
imageio.imsave(os.path.join(pred_save_dir, tile_name + '.png'),
pred.astype(np.uint8))
with open(os.path.join(score_save_dir, 'result.txt'),
'a+') as file:
file.write('{} {}\n'.format(tile_name, iou))
if show_figure:
plt.figure(figsize=(12, 4))
ax1 = plt.subplot(121)
ax1.imshow(truth_label_img)
plt.title('Truth')
ax2 = plt.subplot(122, sharex=ax1, sharey=ax1)
ax2.imshow(pred)
plt.title('pred')
plt.suptitle('{} Results on {} IoU={:3f}'.format(
self.model_name,
file_name_truth.split('_')[0], iou[0] / iou[1]))
plt.show()
iou_record = np.array(iou_record)
mean_iou = np.sum(iou_record[:, 0]) / np.sum(iou_record[:, 1])
print('Overall mean IoU={:.3f}'.format(mean_iou))
if save_result:
if save_result_parent_dir is None:
score_save_dir = os.path.join(uabRepoPaths.evalPath,
self.model_name, ds_name)
else:
score_save_dir = os.path.join(uabRepoPaths.evalPath,
save_result_parent_dir,
self.model_name, ds_name)
with open(os.path.join(score_save_dir, 'result.txt'),
'a+') as file:
file.write('{}'.format(mean_iou))
return iou_return
pad = model.get_overlap()
image_pred = uabUtilreader.un_patchify_shrink(
result, [tile_size[0] + pad, 4576 + pad], [5000, 4576],
input_size, [input_size[0] - pad, input_size[1] - pad],
overlap=pad)
pred_overall = util_functions.get_pred_labels(image_pred) * 1
pred_overall = np.roll(pred_overall, shift=slide_step, axis=1)
pred_overall = pred_overall[:, 1000:-1000]
#pred_overall = pred_overall[:, shift_max-slide_step:-slide_step-1]
truth_label_img = imageio.imread(
os.path.join(parent_dir_truth, file_name_truth))
#truth_label_img = np.roll(truth_label_img, -slide_step, axis=1)
truth_label_img = truth_label_img[:, :4576]
truth_label_img = truth_label_img[:, 1000:-1000]
iou = util_functions.iou_metric(truth_label_img,
pred_overall,
divide_flag=True)
duration = time.time() - start_time
if iou[1] != 0:
print('{} mean IoU={:.3f}, duration: {:.3f}'.format(
tile_name, iou[0] / iou[1], duration))
iou_record.append(iou)
else:
print('{} mean IoU=NA, duration: {:.3f}'.format(
tile_name, duration))
pred_save_dir = os.path.join(score_save_dir, 'pred')
if not os.path.exists(pred_save_dir):
os.makedirs(pred_save_dir)
imageio.imsave(os.path.join(pred_save_dir, tile_name + '.png'),
pred_overall.astype(np.uint8))
sess.run(init)
model.load(model_dir, sess)
result = model.test('X', sess, test_reader)
image_pred = uabUtilreader.un_patchify_shrink(result,
[tile_size[0] + model.get_overlap(), tile_size[1] + model.get_overlap()],
tile_size, input_size,
[input_size[0] - model.get_overlap(), input_size[1] - model.get_overlap()],
overlap=model.get_overlap())
pred = util_functions.get_pred_labels(image_pred) * 255
gt = imageio.imread(os.path.join(gt_dir, '{}_GT.png'.format(tile_ids[tile_cnt].replace('_', '-'))))
cfm_norm = image_pred[:, :, 1]/np.sum(image_pred, axis=2)
cfm_norm = cfm_norm.astype(np.uint8)
#imageio.imsave(os.path.join(data_path, '{}_pred.png'.format(tile_sizes[tile_cnt])), pred)
iou = iou_metric(gt, pred/255)
# view result
lt = imageio.imread(os.path.join(data_path, large_tile[0]))
plt.figure(figsize=(15, 6))
ax1 = plt.subplot(131)
plt.axis('off')
plt.imshow(lt)
ax2 = plt.subplot(132, sharex=ax1, sharey=ax1)
plt.imshow(pred)
plt.title('Pred IoU={:.3f}'.format(iou))
plt.axis('off')
ax3 = plt.subplot(133, sharex=ax1, sharey=ax1)
plt.imshow(gt)
plt.title('GT')
plt.axis('off')
img = imageio.imread(os.path.join(data_dir, city_name +
'_RGB.tif'))[:chip_size, :chip_size, :]
gt = imageio.imread(os.path.join(
data_dir, city_name + '_GT.tif'))[:chip_size, :chip_size] / 255
unet_pred = imageio.imread(os.path.join(
unet_pred_dir, city_name + '.png'))[:chip_size, :chip_size] / 255
frrn_pred = imageio.imread(os.path.join(
frrn_pred_dir, city_name + '.png'))[:chip_size, :chip_size] / 255
deeplab_pred = imageio.imread(
os.path.join(deeplab_pred_dir,
city_name + '.png'))[:chip_size, :chip_size] / 255
fpn_pred = imageio.imread(os.path.join(
fpn_pred_dir, city_name + '.png'))[:chip_size, :chip_size] / 255
unet_iou = iou_metric(gt, unet_pred, truth_val=1) * 100
frrn_iou = iou_metric(gt, frrn_pred, truth_val=1) * 100
deeplab_iou = iou_metric(gt, deeplab_pred, truth_val=1) * 100
fpn_iou = iou_metric(gt, fpn_pred, truth_val=1) * 100
plt.figure(figsize=(16, 4))
ax1 = plt.subplot(151)
plt.imshow(img)
plt.axis('off')
plt.title(city_name)
plt.subplot(152, sharex=ax1, sharey=ax1)
plt.imshow(unet_pred - gt, cmap='bwr')
plt.xticks([], [])
plt.yticks([], [])
plt.title('U-Net {:.2f}'.format(unet_iou))
plt.subplot(153, sharex=ax1, sharey=ax1)
import os
import imageio
from glob import glob
from tqdm import tqdm
import util_functions
pred_dir = r'/media/batcave/personal/huang.bohao/CA'
gt_dir = r'/media/ei-edl01/data/uab_datasets/sp/CA/data/test'
pred_imgs = sorted(glob(os.path.join(pred_dir, '*.png')))
A = 0
B = 0
for pred_img in tqdm(pred_imgs):
img_name = os.path.basename(pred_img)
tile_id = img_name.split('_')[0]
gt_img_name = '{}_GT.png'.format(tile_id)
pred = imageio.imread(pred_img) / 255
gt = imageio.imread(os.path.join(gt_dir, gt_img_name))
a, b = util_functions.iou_metric(gt, pred, truth_val=1, divide_flag=True)
A += a
B += b
print(A / B)
ref_dir, fig_name))
pred_map = pred_map + pred
pred_map = pred_map / len(patch_select)
pred_map = (pred_map > 0.5).astype(np.int)
truth_dir = r'/media/ei-edl01/data/uab_datasets/spca/data/Original_Tiles'
gt = ersa_utils.load_file(
os.path.join(truth_dir,
'{}{}_GT.png'.format(city_name, city_id)))
gt = gt[:, :4576]
gt = gt[:, 1000:-1000]
a, b = util_functions.iou_metric(gt,
pred_map,
divide_flag=True)
iou_record.append([a, b])
cnt += 1
print('{}{}: {:.4f}'.format(city_name, city_id, a / b))
except OSError:
continue
ersa_utils.save_file(save_file_name, np.array(iou_record))
'''save_file_name = os.path.join(task_dir, 'iou_record_ref.npy')
iou_record = np.zeros((25, 2))
ref_dir = r'/hdd/Results/inria_decay/UnetCrop_inria_decay_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60.0_DR0.1_SFN32/inria/pred'
cnt = 0
for city_name in city_list:
for city_id in range(1, 6):
# diff
pred_diff = pred - pred_stitch
gt_window = gt[92:chip_size_large - 92, 92:chip_size_large - 92]
gt_plot = tile[92:chip_size_large-92,92:chip_size_large-92,:] + img_mean
gt_cmp_large = 255 + np.zeros((gt_window.shape[0], gt_window.shape[1], 3), dtype=np.float32)
gt_cmp_small = 255 + np.zeros((gt_window.shape[0], gt_window.shape[1], 3), dtype=np.float32)
gt_plot = util_functions.add_mask(gt_plot, gt_window, [0, 0, 0], 255)
gt_cmp_large = util_functions.add_mask(gt_cmp_large, gt_window, [0, 0, 0], 255)
gt_cmp_large = util_functions.add_mask(gt_cmp_large, gt_window/255-pred, [255, 0, 0], 1)
gt_cmp_large = util_functions.add_mask(gt_cmp_large, gt_window/255-pred, [0, 255, 0], -1)
gt_cmp_small = util_functions.add_mask(gt_cmp_small, gt_window, [0, 0, 0], 255)
gt_cmp_small = util_functions.add_mask(gt_cmp_small, gt_window/255-pred_stitch, [255, 0, 0], 1)
gt_cmp_small = util_functions.add_mask(gt_cmp_small, gt_window/255-pred_stitch, [0, 255, 0], -1)
# compute iou
iou_pred = util_functions.iou_metric(gt_window[6:-6, 6:-6]/255, pred[6:-6, 6:-6]) * 100
iou_stitch = util_functions.iou_metric(gt_window[6:-6, 6:-6]/255, pred_stitch[6:-6, 6:-6]) * 100
if not (np.isnan(iou_pred) or np.isnan(iou_stitch)):
large_ious.append(iou_pred)
small_ious.append(iou_stitch)
# show
plt.figure(figsize=(13, 5))
plt.subplot(131)
plt.xticks([], [])
plt.yticks([], [])
plt.imshow(gt_plot/255)
plt.subplot(132)
plt.xticks([], [])
plt.yticks([], [])
[tile_size[0] + model.get_overlap(), tile_size[1] + model.get_overlap()],
tile_size,
input_size,
[input_size[0] - model.get_overlap(), input_size[1] - model.get_overlap()],
overlap=model.get_overlap())
image_pred_ct = uabUtilreader.un_patchify_shrink(
result2,
[tile_size[0] + model.get_overlap(), tile_size[1] + model.get_overlap()],
tile_size,
input_size,
[input_size[0] - model.get_overlap(), input_size[1] - model.get_overlap()],
overlap=model.get_overlap())
pred = util_functions.get_pred_labels(image_pred) * 255
pred_ct = util_functions.get_pred_labels(image_pred_ct) * 255
iou = iou_metric(pred_ct, pred, truth_val=255)
# view result
lt = imageio.imread(os.path.join(data_path, large_tile[0]))
plt.figure(figsize=(15, 6))
ax1 = plt.subplot(131)
plt.axis('off')
plt.imshow(lt)
ax2 = plt.subplot(132, sharex=ax1, sharey=ax1)
plt.imshow(pred)
plt.title('Pred IoU={:.3f}'.format(iou))
plt.axis('off')
ax3 = plt.subplot(133, sharex=ax1, sharey=ax1)
plt.imshow(pred_ct)
plt.title('\'GT\'')
plt.axis('off')
region_num = 5
for i in range(region_num):
error, x, y = error_record_loo[-1 - i, :]
x = int(x)
y = int(y)
emi_base = get_error_mask_img(
rgb_img[x:x + error_size, y:y + error_size, :],
gt_img[x:x + error_size, y:y + error_size],
mask_base[x:x + error_size, y:y + error_size])
fig = plt.figure(figsize=(5, 5.5))
#plt.subplot(121)
plt.imshow(emi_base)
plt.axis('off')
'''plt.subplot(122)
plt.imshow(rgb_img_copy[x:x+error_size, y:y+error_size, :])
plt.axis('off')'''
title_str = 'IoU={:.3f}'.format(
iou_metric(gt_img[x:x + error_size, y:y + error_size],
pred_img_base[x:x + error_size, y:y + error_size],
truth_val=1) * 100)
title_str += ' {}:({},{})'.format(city_name, x, y)
plt.title(title_str)
plt.tight_layout()
plt.savefig(
os.path.join(img_dir,
'{}_{}_{}_boning.png'.format(city_name, x, y)))
plt.close(fig)
# plt.show()
gt_img[x:x + 500, y:y + 500],
mask_xr[x:x + 500, y:y + 500])
emi_base = get_error_mask_img(rgb_img_copy2[x:x + 500, y:y + 500, :],
gt_img[x:x + 500, y:y + 500],
mask_base[x:x + 500, y:y + 500])
emi_auto = get_error_mask_img(rgb_img_copy3[x:x + 500, y:y + 500, :],
gt_img[x:x + 500, y:y + 500],
mask_auto[x:x + 500, y:y + 500])
plt.figure(figsize=(15, 5.5))
plt.subplot(131)
plt.imshow(emi_xr)
plt.axis('off')
plt.title('XRegion IoU={:.3f}'.format(
iou_metric(gt_img[x:x + 500, y:y + 500],
pred_img_xr[x:x + 500, y:y + 500],
truth_val=255) * 100))
plt.subplot(132)
plt.imshow(emi_auto)
plt.axis('off')
plt.title('Auto IoU={:.3f}'.format(
iou_metric(gt_img[x:x + 500, y:y + 500],
pred_img_auto[x:x + 500, y:y + 500],
truth_val=255) * 100))
plt.subplot(133)
plt.imshow(emi_base)
plt.axis('off')
plt.title('Finetune IoU={:.3f}'.format(
iou_metric(gt_img[x:x + 500, y:y + 500],
pred_img_base[x:x + 500, y:y + 500],
truth_val=255) * 100))
rgb_img_copy = np.copy(rgb_img)
mask_loo = get_error_mask(pred_img_loo, gt_img)
mask_xr = get_error_mask(pred_img_xr, gt_img)
error_record_loo = get_high_error_region(mask_loo, 500, 400)
region_num = 5
for i in range(region_num):
error, x, y = error_record_loo[-1-i, :]
x = int(x)
y = int(y)
emi_loo = get_error_mask_img(rgb_img[x:x+500, y:y+500, :], gt_img[x:x+500, y:y+500], mask_loo[x:x+500, y:y+500])
emi_xr = get_error_mask_img(rgb_img_copy[x:x+500, y:y+500, :], gt_img[x:x+500, y:y+500], mask_xr[x:x+500, y:y+500])
plt.figure(figsize=(16, 6))
plt.subplot(131)
plt.imshow(emi_loo)
plt.axis('off')
plt.title('Leave-one out IoU={:.3f}'.format(iou_metric(gt_img[x:x+500, y:y+500], pred_img_loo[x:x+500, y:y+500], truth_val=255)*100))
plt.subplot(132)
plt.imshow(emi_xr)
plt.axis('off')
plt.title('Cross region IoU={:.3f}'.format(iou_metric(gt_img[x:x+500, y:y+500], pred_img_xr[x:x+500, y:y+500], truth_val=255)*100))
plt.subplot(133)
plt.imshow(gt_img[x:x+500, y:y+500])
plt.axis('off')
plt.title('Ground truth {}({},{})'.format(city_name, x, y))
plt.tight_layout()
plt.savefig(os.path.join(img_dir, '{}_{}_{}_deeplab.png'.format(city_name, x, y)))
plt.show()
