def visualize_box(sample, boxes=None, idx=None, display=False):
if idx is None:
idx = 0
if boxes is None:
heatmaps = [hm[idx, :, :, 0] for hm in sample['heatmaps']]
offsets = [of[idx, :, :, :] for of in sample['offsets']]
boxes = heatmap2box(heatmaps, offsets)
else:
boxes = boxes[idx]
# Visualize reconstructed bounding boxes on image in original scale
orig_image = sample['orig_image'][idx, :, :, :]
box_on_image = concat_box_on_image(orig_image, boxes, color=(0, 255, 0))
if display:
visualize_image(box_on_image)
# Visualize reconstructed bounding boxes on depth in original scale
orig_depth = sample['orig_depth'][idx, :, :, 0]
orig_depth = flowlib.visualize_disp(orig_depth)
box_on_depth = concat_box_on_image(orig_depth,
boxes,
color=(255, 255, 255))
if display:
visualize_image(box_on_depth)
# Visualize reconstructed bounding boxes on flow in original scale
orig_flow = sample['orig_flow'][idx, :, :, :]
orig_flow = flowlib.visualize_flow(orig_flow)
box_on_flow = concat_box_on_image(orig_flow.astype(np.uint8),
boxes,
color=(0, 0, 0))
if display:
visualize_image(box_on_flow)
return box_on_image, box_on_depth, box_on_flow
def main():
data_dir = '/media/yi/DATA/data-orig/vdrift/scripts/images'
im = np.array(Image.open(os.path.join(data_dir, 'cam.png')))
plt.subplots()
plt.imshow(im)
plt.show()
depth = decode_depth(os.path.join(data_dir, 'depth.png'))
max_depth = 100
depth_image = (depth * 255) / max_depth
depth_image = np.clip(depth_image, 0, 255)
depth_image = depth_image.astype(np.uint8)
plt.subplots()
plt.imshow(depth_image)
plt.show()
flow = decode_flow(os.path.join(data_dir, 'flow_x.png'),
os.path.join(data_dir, 'flow_y.png'))
flow_image = flowlib.visualize_flow(flow, 10)
plt.subplots()
plt.imshow(flow_image)
plt.show()
seg = np.array(Image.open(os.path.join(data_dir, 'seg.png')))
plt.subplots()
plt.imshow(seg)
plt.show()
def visualize_prediction(self,
images,
depths,
flows,
prediction,
prediction_offset,
loss,
figure_prefix=''):
# construct a full image containing all scale images and
# a full attention map containing all scale attention maps
max_im_size = np.max(np.array(self.data.im_widths))
sum_im_size = np.sum(np.array(self.data.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
max_ou_size = np.max(np.array(self.data.output_widths))
sum_ou_size = np.sum(np.array(self.data.output_heights))
pred_all = np.zeros((sum_ou_size, max_ou_size))
cnt_im, cnt_pred = 0, 0
for i in range(len(images)):
im = images[i][0, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt_im:cnt_im + height, 0:width, :] = im
dp = depths[i][0, 0, :, :]
dp = flowlib.visualize_disp(dp)
dp_all[cnt_im:cnt_im + height, 0:width, :] = dp
fl = flows[i][0, :, :, :].transpose(1, 2, 0)
fl = flowlib.visualize_flow(fl)
fl_all[cnt_im:cnt_im + height, 0:width, :] = fl
cnt_im = cnt_im + height
pred = prediction[i][0, 0, :, :]
height, width = pred.shape[0], pred.shape[1]
pred_all[cnt_pred:cnt_pred + height, 0:width] = pred
cnt_pred = cnt_pred + height
pred_on_image = self.visualize_prediction_on_image(im_all, pred_all)
if figure_prefix is '':
fig, ax = plt.subplots(1)
ax.imshow(im_all)
fig, ax = plt.subplots(1)
ax.imshow(dp_all)
fig, ax = plt.subplots(1)
ax.imshow(fl_all)
fig, ax = plt.subplots(1)
ax.imshow(pred_all)
fig, ax = plt.subplots(1)
ax.imshow(pred_on_image)
else:
# self.save_image(im_all, figure_prefix + '_image_all.jpg')
# self.save_image(dp_all, figure_prefix + '_depth_all.jpg')
# self.save_image(fl_all, figure_prefix + '_flow_all.jpg')
# self.save_image(pred_all, figure_prefix + '_pred_all.jpg')
self.save_image(pred_on_image,
figure_prefix + '_pred_loss_%.4f.jpg' % loss)
def visualize_prediction(self,
images,
orig_im,
depths,
flows,
pred_label,
acc,
figure_prefix=''):
# construct a full image containing all scale images and
# a full attention map containing all scale attention maps
max_im_size = np.max(np.array(self.data.im_widths))
sum_im_size = np.sum(np.array(self.data.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
cnt_im = 0
for i in range(len(images)):
im = images[i][0, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt_im:cnt_im + height, 0:width, :] = im
dp = depths[i][0, 0, :, :]
dp = flowlib.visualize_disp(dp)
dp_all[cnt_im:cnt_im + height, 0:width, :] = dp
fl = flows[i][0, :, :, :].transpose(1, 2, 0)
fl = flowlib.visualize_flow(fl)
fl_all[cnt_im:cnt_im + height, 0:width, :] = fl
cnt_im = cnt_im + height
pred = pred_label[0, :, :]
pred = vdrift.visualize_seg(pred, self.data.inverse_color_map)
orig_im = orig_im[0, :, :, :].transpose(1, 2, 0)
pred_on_image = self.data.visualize_seg_on_image(orig_im, pred)
if figure_prefix is '':
fig, ax = plt.subplots(1)
ax.imshow(im_all)
fig, ax = plt.subplots(1)
ax.imshow(dp_all)
fig, ax = plt.subplots(1)
ax.imshow(fl_all)
fig, ax = plt.subplots(1)
ax.imshow(pred)
fig, ax = plt.subplots(1)
ax.imshow(pred_on_image)
else:
# self.save_image(im_all, figure_prefix + '_image_all.jpg')
# self.save_image(dp_all, figure_prefix + '_depth_all.jpg')
# self.save_image(fl_all, figure_prefix + '_flow_all.jpg')
# self.save_image(pred, figure_prefix + '_pred.jpg')
self.save_image(pred_on_image,
figure_prefix + '_acc_%.4f.jpg' % acc)
def visualize_input(sample, idx=None, display=False):
if idx is None:
idx = 0
# Visualize input images in all scales
images = [im[idx, :, :, :] for im in sample['images']]
image_all = concat_images(images)
if display:
visualize_image(image_all)
# Visualize input depths in all scales
depths = [
flowlib.visualize_disp(dp[idx, :, :, 0]) for dp in sample['depths']
]
depth_all = concat_images(depths)
if display:
visualize_image(depth_all)
# Visualize input flows in all scales
flows = [
flowlib.visualize_flow(fl[idx, :, :, :]) for fl in sample['flows']
]
flow_all = concat_images(flows)
if display:
visualize_image(flow_all.astype(np.uint8))
def visualize_result(self,
im_input,
im_output,
im_pred,
pred_motion,
gt_motion,
disappear,
appear,
file_name='tmp.png'):
width, height = self.get_img_size(3, max(self.num_frame + 1, 4))
img = numpy.ones((height, width, 3))
prev_im = None
for i in range(self.num_frame - 1):
curr_im = im_input[0, i * 3:(i + 1) *
3, :, :].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, i + 1)
img[y1:y2, x1:x2, :] = curr_im
if i > 0:
im_diff = abs(curr_im - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(2, i + 1)
img[y1:y2, x1:x2, :] = im_diff
prev_im = curr_im
im_output = im_output[0].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame)
img[y1:y2, x1:x2, :] = im_output
im_diff = numpy.abs(im_output - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame)
img[y1:y2, x1:x2, :] = im_diff
pred = im_pred[0].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame + 1)
img[y1:y2, x1:x2, :] = pred
im_diff = numpy.abs(pred - im_output)
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame + 1)
img[y1:y2, x1:x2, :] = im_diff
pred_motion = pred_motion[0].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(pred_motion, self.m_range)
x1, y1, x2, y2 = self.get_img_coordinate(3, 1)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
gt_motion = gt_motion[0].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(gt_motion, self.m_range)
x1, y1, x2, y2 = self.get_img_coordinate(3, 2)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
disappear = disappear[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
disappear = cmap(disappear)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 3)
img[y1:y2, x1:x2, :] = disappear
appear = appear[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
appear = cmap(appear)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 4)
img[y1:y2, x1:x2, :] = appear
if self.save_display:
img = img * 255.0
img = img.astype(numpy.uint8)
img = Image.fromarray(img)
img.save(os.path.join(self.save_display_dir, file_name))
else:
plt.figure(1)
plt.imshow(img)
plt.axis('off')
plt.show()
def visualize_result(self,
im_input,
im_output,
im_pred,
pred_motion,
gt_motion,
disappear,
appear,
file_name='tmp.png',
idx=0):
width, height = self.get_img_size(3, max(self.num_frame + 1, 6))
im_channel = self.im_channel
img = numpy.ones((height, width, 3))
prev_im = None
for i in range(self.num_frame - 1):
curr_im = im_input[idx, i * im_channel:(i + 1) *
im_channel, :, :].cpu().data.numpy().transpose(
1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, i + 1)
img[y1:y2, x1:x2, :] = curr_im
if i > 0:
im_diff = numpy.max(numpy.abs(curr_im - prev_im), 2)
cmap = plt.get_cmap('jet')
im_diff = cmap(im_diff)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(2, i)
img[y1:y2, x1:x2, :] = im_diff
prev_im = curr_im
im_output = im_output[idx].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame)
img[y1:y2, x1:x2, :] = im_output
im_diff = numpy.max(numpy.abs(im_output - prev_im), 2)
cmap = plt.get_cmap('jet')
im_diff = cmap(im_diff)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame - 1)
img[y1:y2, x1:x2, :] = im_diff
im_pred = im_pred[idx].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame + 1)
img[y1:y2, x1:x2, :] = im_pred
im_diff = numpy.max(numpy.abs(im_pred - im_output), 2)
cmap = plt.get_cmap('jet')
im_diff = cmap(im_diff)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame)
img[y1:y2, x1:x2, :] = im_diff
pred_motion = pred_motion[idx].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(pred_motion)
x1, y1, x2, y2 = self.get_img_coordinate(3, 1)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
if gt_motion is None:
im = prev_im * 255.0
if im_channel == 1:
prvs_frame = im.astype(numpy.uint8)
elif im_channel == 3:
prvs_frame = cv2.cvtColor(im.astype(numpy.uint8),
cv2.COLOR_RGB2GRAY)
im = im_output * 255.0
if im_channel == 1:
next_frame = im.astype(numpy.uint8)
elif im_channel == 3:
next_frame = cv2.cvtColor(im.astype(numpy.uint8),
cv2.COLOR_RGB2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs_frame, next_frame, None,
0.5, 5, 5, 3, 5, 1.1, 0)
optical_flow = flowlib.visualize_flow(flow)
x1, y1, x2, y2 = self.get_img_coordinate(3, 2)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
prvs_frame = numpy.asarray(prev_im * 255.0, order='C')
next_frame = numpy.asarray(im_output * 255.0, order='C')
flow = pyflowlib.calculate_flow(prvs_frame, next_frame)
optical_flow = flowlib.visualize_flow(flow)
x1, y1, x2, y2 = self.get_img_coordinate(3, 3)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
prvs_frame = numpy.asarray(prev_im * 255.0,
order='C',
dtype=numpy.uint8)
next_frame = numpy.asarray(im_output * 255.0,
order='C',
dtype=numpy.uint8)
flow = flownetlib.calculate_flow(prvs_frame, next_frame)
optical_flow = flowlib.visualize_flow(flow)
x1, y1, x2, y2 = self.get_img_coordinate(3, 4)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
else:
gt_motion = gt_motion[idx].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(gt_motion)
x1, y1, x2, y2 = self.get_img_coordinate(3, 2)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
disappear = disappear[idx].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
disappear_map = cmap(disappear)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 5)
img[y1:y2, x1:x2, :] = disappear_map
appear = appear[idx].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
appear_map = cmap(appear)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 6)
img[y1:y2, x1:x2, :] = appear_map
if self.save_display:
img = img * 255.0
img = img.astype(numpy.uint8)
img = Image.fromarray(img)
img.save(os.path.join(self.save_display_dir, file_name))
else:
plt.figure(1)
plt.imshow(img)
plt.axis('off')
plt.show()
def visualize(self,
images,
orig_image,
depths,
orig_depth,
flows,
orig_flow,
boxes,
label_maps,
offsets,
idx=None):
if idx is None:
idx = 0
# Plot original image and depth with bounding box
orig_im = orig_image[idx, :, :, :].transpose(1, 2, 0)
orig_dp = orig_depth[idx, 0, :, :]
orig_fl = orig_flow[idx, :, :, :].transpose(1, 2, 0)
if len(boxes[idx]) > 0:
b = boxes[idx].copy()
im_height, im_width = orig_im.shape[0], orig_im.shape[1]
b[:, 0], b[:, 2] = b[:, 0] * im_width, b[:, 2] * im_width
b[:, 1], b[:, 3] = b[:, 1] * im_height, b[:, 3] * im_height
else:
b = []
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
if len(b) > 0:
for i in range(b.shape[0]):
rect = patches.Rectangle((b[i][0], b[i][1]),
b[i][2] - 1 - b[i][0],
b[i][3] - 1 - b[i][1],
linewidth=2,
edgecolor='g',
facecolor='none')
ax.add_patch(rect)
plt.show()
fig, ax = plt.subplots(1)
orig_dp = flowlib.visualize_disp(orig_dp)
ax.imshow(orig_dp)
if len(b) > 0:
for i in range(b.shape[0]):
rect = patches.Rectangle((b[i][0], b[i][1]),
b[i][2] - 1 - b[i][0],
b[i][3] - 1 - b[i][1],
linewidth=2,
edgecolor='w',
facecolor='none')
ax.add_patch(rect)
plt.show()
fig, ax = plt.subplots(1)
orig_fl = flowlib.visualize_flow(orig_fl)
ax.imshow(orig_fl)
if len(b) > 0:
for i in range(b.shape[0]):
rect = patches.Rectangle((b[i][0], b[i][1]),
b[i][2] - 1 - b[i][0],
b[i][3] - 1 - b[i][1],
linewidth=2,
edgecolor='k',
facecolor='none')
ax.add_patch(rect)
plt.show()
# construct a full image containing all scale images
max_im_size = np.max(np.array(self.im_widths))
sum_im_size = np.sum(np.array(self.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
cnt = 0
for i in range(len(images)):
im = images[i][idx, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt:cnt + height, 0:width, :] = im
dp = depths[i][idx, 0, :, :]
dp = flowlib.visualize_disp(dp)
dp_all[cnt:cnt + height, 0:width, :] = dp
fl = flows[i][idx, :, :, :].transpose(1, 2, 0)
fl = flowlib.visualize_flow(fl)
fl_all[cnt:cnt + height, 0:width, :] = fl
cnt = cnt + height
fig, ax = plt.subplots(1)
ax.imshow(im_all)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(dp_all)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(fl_all.astype(np.uint8))
plt.show()
max_ou_size = np.max(np.array(self.output_widths))
sum_ou_size = np.sum(np.array(self.output_heights))
lb_all = np.zeros((sum_ou_size, max_ou_size))
cnt = 0
for i in range(len(label_maps)):
lb = label_maps[i][idx, 0, :, :]
height, width = lb.shape[0], lb.shape[1]
lb_all[cnt:cnt + height, 0:width] = lb
cnt = cnt + height
fig, ax = plt.subplots(1)
ax.imshow(lb_all)
plt.show()
label_on_image = self.visualize_prediction_on_image(im_all, lb_all)
fig, ax = plt.subplots(1)
ax.imshow(label_on_image)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
for i in range(len(label_maps)):
of = offsets[i][idx, :, :, :].transpose(1, 2, 0)
lb = label_maps[i][idx, 0, :, :]
ou_height, ou_width = lb.shape[0], lb.shape[1]
b_idx = np.nonzero(lb)
y_c, x_c = b_idx[0], b_idx[1]
b = np.zeros((len(b_idx[0]), 4))
for k in range(b.shape[0]):
offset = of[y_c[k], x_c[k], :]
b[k, 0] = x_c[k] * 1.0 / ou_width + offset[0]
b[k, 1] = y_c[k] * 1.0 / ou_height + offset[1]
b[k, 2] = x_c[k] * 1.0 / ou_width + offset[2]
b[k, 3] = y_c[k] * 1.0 / ou_height + offset[3]
if len(b) > 0:
b[:, 0], b[:, 2] = b[:, 0] * im_width, b[:, 2] * im_width
b[:, 1], b[:, 3] = b[:, 1] * im_height, b[:, 3] * im_height
for k in range(b.shape[0]):
rect = patches.Rectangle((b[k][0], b[k][1]),
b[k][2] - 1 - b[k][0],
b[k][3] - 1 - b[k][1],
linewidth=2,
edgecolor='g',
facecolor='none')
ax.add_patch(rect)
plt.show()
dest_size=base_size)
images = [img1, img2]
images = np.array(images).transpose(3, 0, 1, 2)
im = torch.from_numpy(images.astype(
np.float32)).unsqueeze(0).to(device)
# process the image pair to obtian the flow
result = net(im).squeeze()
data = result.data.cpu().numpy().transpose(1, 2, 0)
print("input shape:", im.shape)
print("data shape:", data.shape)
if args.visualize != 0:
visualize_flow(data)
else:
if os.path.exists(args.out_file) and args.recalc == 0:
print("File existed -> skip!!!")
else:
data = load_video_frames(args.in_path, dest_size=None)
n_frame = len(data) - 1 # -1 because we consider pairs of frames
outflows = []
for i in range(0, n_frame, args.batch):
# determine frame indices in batch
idx0, idx1 = i, min(i + args.batch, n_frame)
# forming data batch
frames_first = data[idx0:idx1]
frames_second = data[idx0 + 1:idx1 + 1]
if scale > 1:
frames_first = [
def visualize(self,
images,
orig_image,
depths,
orig_depth,
flows,
orig_flow,
label,
orig_label,
idx=None):
if idx is None:
idx = 0
orig_im = orig_image[idx, :, :, :].transpose(1, 2, 0)
orig_dp = orig_depth[idx, 0, :, :]
orig_fl = orig_flow[idx, :, :, :].transpose(1, 2, 0)
orig_lb = orig_label[idx, :, :]
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
plt.show()
fig, ax = plt.subplots(1)
orig_dp = flowlib.visualize_disp(orig_dp)
ax.imshow(orig_dp)
plt.show()
fig, ax = plt.subplots(1)
orig_fl = flowlib.visualize_flow(orig_fl)
ax.imshow(orig_fl)
plt.show()
fig, ax = plt.subplots(1)
orig_lb = vdrift.visualize_seg(orig_lb, self.inverse_color_map)
ax.imshow(orig_lb)
plt.show()
# construct a full image containing all scale images
max_im_size = np.max(np.array(self.im_widths))
sum_im_size = np.sum(np.array(self.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
cnt = 0
for i in range(len(images)):
im = images[i][idx, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt:cnt + height, 0:width, :] = im
dp = depths[i][idx, 0, :, :]
dp = flowlib.visualize_disp(dp)
dp_all[cnt:cnt + height, 0:width, :] = dp
fl = flows[i][idx, :, :, :].transpose(1, 2, 0)
fl = flowlib.visualize_flow(fl)
fl_all[cnt:cnt + height, 0:width, :] = fl
cnt = cnt + height
fig, ax = plt.subplots(1)
ax.imshow(im_all)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(dp_all)
plt.show()
fig, ax = plt.subplots(1)
ax.imshow(fl_all.astype(np.uint8))
plt.show()
seg = vdrift.visualize_seg(label[idx, :, :], self.inverse_color_map)
fig, ax = plt.subplots(1)
ax.imshow(seg.astype(np.uint8))
plt.show()
seg_on_image = self.visualize_seg_on_image(im_all, seg)
fig, ax = plt.subplots(1)
ax.imshow(seg_on_image)
plt.show()
def visualize_box(self,
orig_image,
orig_depth,
orig_flow,
boxes,
loss,
figure_prefix=''):
orig_im = orig_image[0, :, :, :].transpose(1, 2, 0)
im_height, im_width = orig_im.shape[0], orig_im.shape[1]
assert (im_height == self.data.orig_im_size[0])
assert (im_width == self.data.orig_im_size[1])
# Plot original large image with bounding box
boxes = boxes[0]
if len(boxes) > 0:
b = boxes.copy()
sc = b[:, 4]
b[:, 0], b[:, 2] = b[:, 0] * im_width, b[:, 2] * im_width
b[:, 1], b[:, 3] = b[:, 1] * im_width, b[:, 3] * im_width
b = b.astype(np.int)
else:
b = []
orig_im = orig_im * 255.0
orig_im = orig_im.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_im, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(0, 255, 0), 3)
cv2.putText(orig_im, '%.4f' % sc[i],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2,
cv2.LINE_AA)
# Plot original depth with bounding box
orig_dp = orig_depth[0, 0, :, :]
orig_dp = flowlib.visualize_disp(orig_dp)
orig_dp = orig_dp * 255.0
orig_dp = orig_dp.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_dp, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(255, 255, 255), 3)
cv2.putText(orig_dp, '%.4f' % sc[i],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2,
cv2.LINE_AA)
# Plot original flow with bounding box
orig_fl = orig_flow[0, :, :, :].transpose(1, 2, 0)
orig_fl = flowlib.visualize_flow(orig_fl)
orig_fl = orig_fl.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_fl, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(0, 0, 0), 3)
cv2.putText(orig_fl, '%.4f' % sc[i],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2,
cv2.LINE_AA)
if figure_prefix is '':
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
fig, ax = plt.subplots(1)
ax.imshow(orig_dp)
fig, ax = plt.subplots(1)
ax.imshow(orig_fl)
fig, ax = plt.subplots(1)
ax.imshow(label_on_image)
else:
self.save_image(
orig_im,
figure_prefix + '_pred_orig_image_loss_%.4f.jpg' % loss)
self.save_image(
orig_dp,
figure_prefix + '_pred_orig_depth_loss_%.4f.jpg' % loss)
self.save_image(
orig_fl,
figure_prefix + '_pred_orig_flow_loss_%.4f.jpg' % loss)
def visualize_groundtruth(self,
orig_image,
images,
orig_depth,
depths,
orig_flow,
flows,
boxes,
label_maps,
offsets,
figure_prefix=''):
orig_im = orig_image[0, :, :, :].transpose(1, 2, 0)
im_height, im_width = orig_im.shape[0], orig_im.shape[1]
# Plot original large image with bounding box
if len(boxes[0]) > 0:
b = boxes[0].copy()
b[:, 0], b[:, 2] = b[:, 0] * im_width, b[:, 2] * im_width
b[:, 1], b[:, 3] = b[:, 1] * im_height, b[:, 3] * im_height
b = b.astype(np.int)
else:
b = []
orig_im = orig_im * 255.0
orig_im = orig_im.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_im, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(0, 255, 0), 3)
cv2.putText(orig_im, self.data.inverse_class_map[1],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2,
cv2.LINE_AA)
# Plot original depth with bounding box
orig_dp = orig_depth[0, 0, :, :]
orig_dp = flowlib.visualize_disp(orig_dp)
orig_dp = orig_dp * 255.0
orig_dp = orig_dp.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_dp, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(255, 255, 255), 3)
cv2.putText(orig_dp, self.data.inverse_class_map[1],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2,
cv2.LINE_AA)
# Plot original flow with bounding box
orig_fl = orig_flow[0, :, :, :].transpose(1, 2, 0)
orig_fl = flowlib.visualize_flow(orig_fl)
orig_fl = orig_fl.astype(np.uint8).copy()
if len(b) > 0:
for i in range(b.shape[0]):
cv2.rectangle(orig_fl, (b[i, 0], b[i, 1]), (b[i, 2], b[i, 3]),
(0, 0, 0), 3)
cv2.putText(orig_fl, self.data.inverse_class_map[1],
(b[i, 0] + 2, b[i, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2,
cv2.LINE_AA)
max_im_size = np.max(np.array(self.data.im_widths))
sum_im_size = np.sum(np.array(self.data.im_heights))
im_all = np.zeros((sum_im_size, max_im_size, 3))
dp_all = np.zeros((sum_im_size, max_im_size, 3))
fl_all = np.zeros((sum_im_size, max_im_size, 3))
max_ou_size = np.max(np.array(self.data.output_widths))
sum_ou_size = np.sum(np.array(self.data.output_heights))
pred_all = np.zeros((sum_ou_size, max_ou_size))
cnt_im = 0
for i in range(len(images)):
im = images[i][0, :, :, :].transpose(1, 2, 0)
height, width = im.shape[0], im.shape[1]
im_all[cnt_im:cnt_im + height, 0:width, :] = im
cnt_im = cnt_im + height
max_ou_size = np.max(np.array(self.data.output_widths))
sum_ou_size = np.sum(np.array(self.data.output_heights))
lb_all = np.zeros((sum_ou_size, max_ou_size))
cnt_lb = 0
for i in range(len(label_maps)):
lb = label_maps[i][0, 0, :, :]
height, width = lb.shape[0], lb.shape[1]
lb_all[cnt_lb:cnt_lb + height, 0:width] = lb
cnt_lb = cnt_lb + height
label_on_image = self.visualize_prediction_on_image(im_all, lb_all)
orig_im2 = orig_image[0, :, :, :].transpose(1, 2, 0)
im_height, im_width = orig_im2.shape[0], orig_im2.shape[1]
orig_im2 = orig_im2 * 255.0
orig_im2 = orig_im2.astype(np.uint8).copy()
for i in range(len(label_maps)):
of = offsets[i][0, :, :, :].transpose(1, 2, 0)
lb = label_maps[i][0, 0, :, :]
ou_height, ou_width = lb.shape[0], lb.shape[1]
b_idx = np.nonzero(lb)
y_c, x_c = b_idx[0], b_idx[1]
b = np.zeros((len(b_idx[0]), 4))
for k in range(b.shape[0]):
offset = of[y_c[k], x_c[k], :]
b[k, 0] = x_c[k] * 1.0 / ou_width + offset[0]
b[k, 1] = y_c[k] * 1.0 / ou_height + offset[1]
b[k, 2] = x_c[k] * 1.0 / ou_width + offset[2]
b[k, 3] = y_c[k] * 1.0 / ou_height + offset[3]
b[k, 0], b[k, 2] = b[k, 0] * im_width, b[k, 2] * im_width
b[k, 1], b[k, 3] = b[k, 1] * im_height, b[k, 3] * im_height
b = b.astype(np.int)
for k in range(b.shape[0]):
cv2.rectangle(orig_im2, (b[k, 0], b[k, 1]), (b[k, 2], b[k, 3]),
(0, 255, 0), 3)
cv2.putText(orig_im2, self.data.inverse_class_map[1],
(b[k, 0] + 2, b[k, 1] + 28),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2,
cv2.LINE_AA)
if figure_prefix is '':
fig, ax = plt.subplots(1)
ax.imshow(orig_im)
fig, ax = plt.subplots(1)
ax.imshow(orig_dp)
fig, ax = plt.subplots(1)
ax.imshow(orig_fl)
fig, ax = plt.subplots(1)
ax.imshow(label_on_image)
fig, ax = plt.subplots(1)
ax.imshow(orig_im2)
else:
self.save_image(orig_im, figure_prefix + '_orig_image.jpg')
self.save_image(orig_dp, figure_prefix + '_orig_depth.jpg')
self.save_image(orig_fl, figure_prefix + '_orig_flow.jpg')
self.save_image(label_on_image, figure_prefix + '_label.jpg')
self.save_image(orig_im2, figure_prefix + '_image_with_box.jpg')
