def visualize_box(sample, boxes=None, idx=None, display=False):
if idx is None:
idx = 0
if boxes is None:
heatmaps = [hm[idx, :, :, 0] for hm in sample['heatmaps']]
offsets = [of[idx, :, :, :] for of in sample['offsets']]
boxes = heatmap2box(heatmaps, offsets)
else:
boxes = boxes[idx]
# Visualize reconstructed bounding boxes on image in original scale
orig_image = sample['orig_image'][idx, :, :, :]
box_on_image = concat_box_on_image(orig_image, boxes, color=(0, 255, 0))
if display:
visualize_image(box_on_image)
# Visualize reconstructed bounding boxes on depth in original scale
orig_depth = sample['orig_depth'][idx, :, :, 0]
orig_depth = flowlib.visualize_disp(orig_depth)
box_on_depth = concat_box_on_image(orig_depth,
boxes,
color=(255, 255, 255))
if display:
visualize_image(box_on_depth)
# Visualize reconstructed bounding boxes on flow in original scale
orig_flow = sample['orig_flow'][idx, :, :, :]
orig_flow = flowlib.visualize_flow(orig_flow)
box_on_flow = concat_box_on_image(orig_flow.astype(np.uint8),
boxes,
color=(0, 0, 0))
if display:
visualize_image(box_on_flow)
return box_on_image, box_on_depth, box_on_flow
def visualize_prediction(self,
images,
depths,
flows,
prediction,
prediction_offset,
loss,
figure_prefix=''):
# construct a full image containing all scale images and
# a full attention map containing all scale attention maps
max_im_size = np.max(np.array(self.data.im_widths))
sum_im_size = np.sum(np.array(self.data.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
max_ou_size = np.max(np.array(self.data.output_widths))
sum_ou_size = np.sum(np.array(self.data.output_heights))
pred_all = np.zeros((sum_ou_size, max_ou_size))
cnt_im, cnt_pred = 0, 0
for i in range(len(images)):
im = images[i][0, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt_im:cnt_im + height, 0:width, :] = im
dp = depths[i][0, 0, :, :]
dp = flowlib.visualize_disp(dp)
dp_all[cnt_im:cnt_im + height, 0:width, :] = dp
fl = flows[i][0, :, :, :].transpose(1, 2, 0)
fl = flowlib.visualize_flow(fl)
fl_all[cnt_im:cnt_im + height, 0:width, :] = fl
cnt_im = cnt_im + height
pred = prediction[i][0, 0, :, :]
height, width = pred.shape[0], pred.shape[1]
pred_all[cnt_pred:cnt_pred + height, 0:width] = pred
cnt_pred = cnt_pred + height
pred_on_image = self.visualize_prediction_on_image(im_all, pred_all)
if figure_prefix is '':
fig, ax = plt.subplots(1)
ax.imshow(im_all)
fig, ax = plt.subplots(1)
ax.imshow(dp_all)
fig, ax = plt.subplots(1)
ax.imshow(fl_all)
fig, ax = plt.subplots(1)
ax.imshow(pred_all)
fig, ax = plt.subplots(1)
ax.imshow(pred_on_image)
else:
# self.save_image(im_all, figure_prefix + '_image_all.jpg')
# self.save_image(dp_all, figure_prefix + '_depth_all.jpg')
# self.save_image(fl_all, figure_prefix + '_flow_all.jpg')
# self.save_image(pred_all, figure_prefix + '_pred_all.jpg')
self.save_image(pred_on_image,
figure_prefix + '_pred_loss_%.4f.jpg' % loss)
def visualize_prediction(self,
images,
orig_im,
depths,
flows,
pred_label,
acc,
figure_prefix=''):
# construct a full image containing all scale images and
# a full attention map containing all scale attention maps
max_im_size = np.max(np.array(self.data.im_widths))
sum_im_size = np.sum(np.array(self.data.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
cnt_im = 0
for i in range(len(images)):
im = images[i][0, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt_im:cnt_im + height, 0:width, :] = im
dp = depths[i][0, 0, :, :]
dp = flowlib.visualize_disp(dp)
dp_all[cnt_im:cnt_im + height, 0:width, :] = dp
fl = flows[i][0, :, :, :].transpose(1, 2, 0)
fl = flowlib.visualize_flow(fl)
fl_all[cnt_im:cnt_im + height, 0:width, :] = fl
cnt_im = cnt_im + height
pred = pred_label[0, :, :]
pred = vdrift.visualize_seg(pred, self.data.inverse_color_map)
orig_im = orig_im[0, :, :, :].transpose(1, 2, 0)
pred_on_image = self.data.visualize_seg_on_image(orig_im, pred)
if figure_prefix is '':
fig, ax = plt.subplots(1)
ax.imshow(im_all)
fig, ax = plt.subplots(1)
ax.imshow(dp_all)
fig, ax = plt.subplots(1)
ax.imshow(fl_all)
fig, ax = plt.subplots(1)
ax.imshow(pred)
fig, ax = plt.subplots(1)
ax.imshow(pred_on_image)
else:
# self.save_image(im_all, figure_prefix + '_image_all.jpg')
# self.save_image(dp_all, figure_prefix + '_depth_all.jpg')
# self.save_image(fl_all, figure_prefix + '_flow_all.jpg')
# self.save_image(pred, figure_prefix + '_pred.jpg')
self.save_image(pred_on_image,
figure_prefix + '_acc_%.4f.jpg' % acc)
def visualize_input(sample, idx=None, display=False):
if idx is None:
idx = 0
# Visualize input images in all scales
images = [im[idx, :, :, :] for im in sample['images']]
image_all = concat_images(images)
if display:
visualize_image(image_all)
# Visualize input depths in all scales
depths = [
flowlib.visualize_disp(dp[idx, :, :, 0]) for dp in sample['depths']
]
depth_all = concat_images(depths)
if display:
visualize_image(depth_all)
# Visualize input flows in all scales
flows = [
flowlib.visualize_flow(fl[idx, :, :, :]) for fl in sample['flows']
]
flow_all = concat_images(flows)
if display:
visualize_image(flow_all.astype(np.uint8))
def visualize(self,
images,
orig_image,
depths,
orig_depth,
flows,
orig_flow,
boxes,
label_maps,
offsets,
idx=None):
if idx is None:
idx = 0
# Plot original image and depth with bounding box
orig_im = orig_image[idx, :, :, :].transpose(1, 2, 0)
orig_dp = orig_depth[idx, 0, :, :]
orig_fl = orig_flow[idx, :, :, :].transpose(1, 2, 0)
if len(boxes[idx]) > 0:
b = boxes[idx].copy()
im_height, im_width = orig_im.shape[0], orig_im.shape[1]
b[:, 0], b[:, 2] = b[:, 0] * im_width, b[:, 2] * im_width
b[:, 1], b[:, 3] = b[:, 1] * im_height, b[:, 3] * im_height
else:
b = []
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
if len(b) > 0:
for i in range(b.shape[0]):
rect = patches.Rectangle((b[i][0], b[i][1]),
b[i][2] - 1 - b[i][0],
b[i][3] - 1 - b[i][1],
linewidth=2,
edgecolor='g',
facecolor='none')
ax.add_patch(rect)
plt.show()
fig, ax = plt.subplots(1)
orig_dp = flowlib.visualize_disp(orig_dp)
ax.imshow(orig_dp)
if len(b) > 0:
for i in range(b.shape[0]):
rect = patches.Rectangle((b[i][0], b[i][1]),
b[i][2] - 1 - b[i][0],
b[i][3] - 1 - b[i][1],
linewidth=2,
edgecolor='w',
facecolor='none')
ax.add_patch(rect)
plt.show()
fig, ax = plt.subplots(1)
orig_fl = flowlib.visualize_flow(orig_fl)
ax.imshow(orig_fl)
if len(b) > 0:
for i in range(b.shape[0]):
rect = patches.Rectangle((b[i][0], b[i][1]),
b[i][2] - 1 - b[i][0],
b[i][3] - 1 - b[i][1],
linewidth=2,
edgecolor='k',
facecolor='none')
ax.add_patch(rect)
plt.show()
# construct a full image containing all scale images
max_im_size = np.max(np.array(self.im_widths))
sum_im_size = np.sum(np.array(self.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
cnt = 0
for i in range(len(images)):
im = images[i][idx, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt:cnt + height, 0:width, :] = im
dp = depths[i][idx, 0, :, :]
dp = flowlib.visualize_disp(dp)
dp_all[cnt:cnt + height, 0:width, :] = dp
fl = flows[i][idx, :, :, :].transpose(1, 2, 0)
fl = flowlib.visualize_flow(fl)
fl_all[cnt:cnt + height, 0:width, :] = fl
cnt = cnt + height
fig, ax = plt.subplots(1)
ax.imshow(im_all)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(dp_all)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(fl_all.astype(np.uint8))
plt.show()
max_ou_size = np.max(np.array(self.output_widths))
sum_ou_size = np.sum(np.array(self.output_heights))
lb_all = np.zeros((sum_ou_size, max_ou_size))
cnt = 0
for i in range(len(label_maps)):
lb = label_maps[i][idx, 0, :, :]
height, width = lb.shape[0], lb.shape[1]
lb_all[cnt:cnt + height, 0:width] = lb
cnt = cnt + height
fig, ax = plt.subplots(1)
ax.imshow(lb_all)
plt.show()
label_on_image = self.visualize_prediction_on_image(im_all, lb_all)
fig, ax = plt.subplots(1)
ax.imshow(label_on_image)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
for i in range(len(label_maps)):
of = offsets[i][idx, :, :, :].transpose(1, 2, 0)
lb = label_maps[i][idx, 0, :, :]
ou_height, ou_width = lb.shape[0], lb.shape[1]
b_idx = np.nonzero(lb)
y_c, x_c = b_idx[0], b_idx[1]
b = np.zeros((len(b_idx[0]), 4))
for k in range(b.shape[0]):
offset = of[y_c[k], x_c[k], :]
b[k, 0] = x_c[k] * 1.0 / ou_width + offset[0]
b[k, 1] = y_c[k] * 1.0 / ou_height + offset[1]
b[k, 2] = x_c[k] * 1.0 / ou_width + offset[2]
b[k, 3] = y_c[k] * 1.0 / ou_height + offset[3]
if len(b) > 0:
b[:, 0], b[:, 2] = b[:, 0] * im_width, b[:, 2] * im_width
b[:, 1], b[:, 3] = b[:, 1] * im_height, b[:, 3] * im_height
for k in range(b.shape[0]):
rect = patches.Rectangle((b[k][0], b[k][1]),
b[k][2] - 1 - b[k][0],
b[k][3] - 1 - b[k][1],
linewidth=2,
edgecolor='g',
facecolor='none')
ax.add_patch(rect)
plt.show()
def visualize(self,
images,
orig_image,
depths,
orig_depth,
flows,
orig_flow,
label,
orig_label,
idx=None):
if idx is None:
idx = 0
orig_im = orig_image[idx, :, :, :].transpose(1, 2, 0)
orig_dp = orig_depth[idx, 0, :, :]
orig_fl = orig_flow[idx, :, :, :].transpose(1, 2, 0)
orig_lb = orig_label[idx, :, :]
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
plt.show()
fig, ax = plt.subplots(1)
orig_dp = flowlib.visualize_disp(orig_dp)
ax.imshow(orig_dp)
plt.show()
fig, ax = plt.subplots(1)
orig_fl = flowlib.visualize_flow(orig_fl)
ax.imshow(orig_fl)
plt.show()
fig, ax = plt.subplots(1)
orig_lb = vdrift.visualize_seg(orig_lb, self.inverse_color_map)
ax.imshow(orig_lb)
plt.show()
# construct a full image containing all scale images
max_im_size = np.max(np.array(self.im_widths))
sum_im_size = np.sum(np.array(self.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
cnt = 0
for i in range(len(images)):
im = images[i][idx, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt:cnt + height, 0:width, :] = im
dp = depths[i][idx, 0, :, :]
dp = flowlib.visualize_disp(dp)
dp_all[cnt:cnt + height, 0:width, :] = dp
fl = flows[i][idx, :, :, :].transpose(1, 2, 0)
fl = flowlib.visualize_flow(fl)
fl_all[cnt:cnt + height, 0:width, :] = fl
cnt = cnt + height
fig, ax = plt.subplots(1)
ax.imshow(im_all)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(dp_all)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(fl_all.astype(np.uint8))
plt.show()
seg = vdrift.visualize_seg(label[idx, :, :], self.inverse_color_map)
fig, ax = plt.subplots(1)
ax.imshow(seg.astype(np.uint8))
plt.show()
seg_on_image = self.visualize_seg_on_image(im_all, seg)
fig, ax = plt.subplots(1)
ax.imshow(seg_on_image)
plt.show()
def visualize_box(self,
orig_image,
orig_depth,
orig_flow,
boxes,
loss,
figure_prefix=''):
orig_im = orig_image[0, :, :, :].transpose(1, 2, 0)
im_height, im_width = orig_im.shape[0], orig_im.shape[1]
assert (im_height == self.data.orig_im_size[0])
assert (im_width == self.data.orig_im_size[1])
# Plot original large image with bounding box
boxes = boxes[0]
if len(boxes) > 0:
b = boxes.copy()
sc = b[:, 4]
b[:, 0], b[:, 2] = b[:, 0] * im_width, b[:, 2] * im_width
b[:, 1], b[:, 3] = b[:, 1] * im_width, b[:, 3] * im_width
b = b.astype(np.int)
else:
b = []
orig_im = orig_im * 255.0
orig_im = orig_im.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_im, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(0, 255, 0), 3)
cv2.putText(orig_im, '%.4f' % sc[i],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2,
cv2.LINE_AA)
# Plot original depth with bounding box
orig_dp = orig_depth[0, 0, :, :]
orig_dp = flowlib.visualize_disp(orig_dp)
orig_dp = orig_dp * 255.0
orig_dp = orig_dp.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_dp, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(255, 255, 255), 3)
cv2.putText(orig_dp, '%.4f' % sc[i],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2,
cv2.LINE_AA)
# Plot original flow with bounding box
orig_fl = orig_flow[0, :, :, :].transpose(1, 2, 0)
orig_fl = flowlib.visualize_flow(orig_fl)
orig_fl = orig_fl.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_fl, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(0, 0, 0), 3)
cv2.putText(orig_fl, '%.4f' % sc[i],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2,
cv2.LINE_AA)
if figure_prefix is '':
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
fig, ax = plt.subplots(1)
ax.imshow(orig_dp)
fig, ax = plt.subplots(1)
ax.imshow(orig_fl)
fig, ax = plt.subplots(1)
ax.imshow(label_on_image)
else:
self.save_image(
orig_im,
figure_prefix + '_pred_orig_image_loss_%.4f.jpg' % loss)
self.save_image(
orig_dp,
figure_prefix + '_pred_orig_depth_loss_%.4f.jpg' % loss)
self.save_image(
orig_fl,
figure_prefix + '_pred_orig_flow_loss_%.4f.jpg' % loss)
def visualize_groundtruth(self,
orig_image,
images,
orig_depth,
depths,
orig_flow,
flows,
boxes,
label_maps,
offsets,
figure_prefix=''):
orig_im = orig_image[0, :, :, :].transpose(1, 2, 0)
im_height, im_width = orig_im.shape[0], orig_im.shape[1]
# Plot original large image with bounding box
if len(boxes[0]) > 0:
b = boxes[0].copy()
b[:, 0], b[:, 2] = b[:, 0] * im_width, b[:, 2] * im_width
b[:, 1], b[:, 3] = b[:, 1] * im_height, b[:, 3] * im_height
b = b.astype(np.int)
else:
b = []
orig_im = orig_im * 255.0
orig_im = orig_im.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_im, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(0, 255, 0), 3)
cv2.putText(orig_im, self.data.inverse_class_map[1],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2,
cv2.LINE_AA)
# Plot original depth with bounding box
orig_dp = orig_depth[0, 0, :, :]
orig_dp = flowlib.visualize_disp(orig_dp)
orig_dp = orig_dp * 255.0
orig_dp = orig_dp.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_dp, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(255, 255, 255), 3)
cv2.putText(orig_dp, self.data.inverse_class_map[1],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2,
cv2.LINE_AA)
# Plot original flow with bounding box
orig_fl = orig_flow[0, :, :, :].transpose(1, 2, 0)
orig_fl = flowlib.visualize_flow(orig_fl)
orig_fl = orig_fl.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_fl, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(0, 0, 0), 3)
cv2.putText(orig_fl, self.data.inverse_class_map[1],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2,
cv2.LINE_AA)
max_im_size = np.max(np.array(self.data.im_widths))
sum_im_size = np.sum(np.array(self.data.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
max_ou_size = np.max(np.array(self.data.output_widths))
sum_ou_size = np.sum(np.array(self.data.output_heights))
pred_all = np.zeros((sum_ou_size, max_ou_size))
cnt_im = 0
for i in range(len(images)):
im = images[i][0, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt_im:cnt_im + height, 0:width, :] = im
cnt_im = cnt_im + height
max_ou_size = np.max(np.array(self.data.output_widths))
sum_ou_size = np.sum(np.array(self.data.output_heights))
lb_all = np.zeros((sum_ou_size, max_ou_size))
cnt_lb = 0
for i in range(len(label_maps)):
lb = label_maps[i][0, 0, :, :]
height, width = lb.shape[0], lb.shape[1]
lb_all[cnt_lb:cnt_lb + height, 0:width] = lb
cnt_lb = cnt_lb + height
label_on_image = self.visualize_prediction_on_image(im_all, lb_all)
orig_im2 = orig_image[0, :, :, :].transpose(1, 2, 0)
im_height, im_width = orig_im2.shape[0], orig_im2.shape[1]
orig_im2 = orig_im2 * 255.0
orig_im2 = orig_im2.astype(np.uint8).copy()
for i in range(len(label_maps)):
of = offsets[i][0, :, :, :].transpose(1, 2, 0)
lb = label_maps[i][0, 0, :, :]
ou_height, ou_width = lb.shape[0], lb.shape[1]
b_idx = np.nonzero(lb)
y_c, x_c = b_idx[0], b_idx[1]
b = np.zeros((len(b_idx[0]), 4))
for k in range(b.shape[0]):
offset = of[y_c[k], x_c[k], :]
b[k, 0] = x_c[k] * 1.0 / ou_width + offset[0]
b[k, 1] = y_c[k] * 1.0 / ou_height + offset[1]
b[k, 2] = x_c[k] * 1.0 / ou_width + offset[2]
b[k, 3] = y_c[k] * 1.0 / ou_height + offset[3]
b[k, 0], b[k, 2] = b[k, 0] * im_width, b[k, 2] * im_width
b[k, 1], b[k, 3] = b[k, 1] * im_height, b[k, 3] * im_height
b = b.astype(np.int)
for k in range(b.shape[0]):
cv2.rectangle(orig_im2, (b[k, 0], b[k, 1]), (b[k, 2], b[k, 3]),
(0, 255, 0), 3)
cv2.putText(orig_im2, self.data.inverse_class_map[1],
(b[k, 0] + 2, b[k, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2,
cv2.LINE_AA)
if figure_prefix is '':
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
fig, ax = plt.subplots(1)
ax.imshow(orig_dp)
fig, ax = plt.subplots(1)
ax.imshow(orig_fl)
fig, ax = plt.subplots(1)
ax.imshow(label_on_image)
fig, ax = plt.subplots(1)
ax.imshow(orig_im2)
else:
self.save_image(orig_im, figure_prefix + '_orig_image.jpg')
self.save_image(orig_dp, figure_prefix + '_orig_depth.jpg')
self.save_image(orig_fl, figure_prefix + '_orig_flow.jpg')
self.save_image(label_on_image, figure_prefix + '_label.jpg')
self.save_image(orig_im2, figure_prefix + '_image_with_box.jpg')