def visualize_result(self, im_input, im_output, im_pred, pred_motion, gt_motion, file_name='tmp.png'):
width, height = self.get_img_size(3, max(self.num_frame + 1, 4))
img = numpy.ones((height, width, 3))
prev_im = None
for i in range(self.num_frame - 1):
curr_im = im_input[0, i*3:(i+1)*3, :, :].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, i+1)
img[y1:y2, x1:x2, :] = curr_im
if i > 0:
im_diff = abs(curr_im - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(2, i+1)
img[y1:y2, x1:x2, :] = im_diff
prev_im = curr_im
im_output = im_output[0].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame)
img[y1:y2, x1:x2, :] = im_output
im_diff = numpy.abs(im_output - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame)
img[y1:y2, x1:x2, :] = im_diff
pred = im_pred[0].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame + 1)
img[y1:y2, x1:x2, :] = pred
im_diff = numpy.abs(pred - im_output)
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame + 1)
img[y1:y2, x1:x2, :] = im_diff
pred_motion = pred_motion[0].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(pred_motion, self.m_range)
x1, y1, x2, y2 = self.get_img_coordinate(3, 1)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
gt_motion = gt_motion[0].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(gt_motion, self.m_range)
x1, y1, x2, y2 = self.get_img_coordinate(3, 2)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
if self.save_display:
img = img * 255.0
img = img.astype(numpy.uint8)
img = Image.fromarray(img)
img.save(os.path.join(self.save_display_dir, file_name))
else:
plt.figure(1)
plt.imshow(img)
plt.axis('off')
plt.show()
def display(self, im, motion, motion_r, seg_layer):
num_frame = self.num_frame
width, height = self.visualizer.get_img_size(5, num_frame)
img = numpy.ones((height, width, 3))
prev_im = None
for i in range(num_frame):
curr_im = im[0, i, :, :, :].transpose(1, 2, 0)
x1, y1, x2, y2 = self.visualizer.get_img_coordinate(1, i + 1)
img[y1:y2, x1:x2, :] = curr_im
if i > 0:
im_diff = abs(curr_im - prev_im)
x1, y1, x2, y2 = self.visualizer.get_img_coordinate(2, i + 1)
img[y1:y2, x1:x2, :] = im_diff
prev_im = curr_im
flow = motion[0, i, :, :, :].transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(flow, self.m_range)
x1, y1, x2, y2 = self.visualizer.get_img_coordinate(3, i + 1)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
flow = motion_r[0, i, :, :, :].transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(flow, self.m_range)
x1, y1, x2, y2 = self.visualizer.get_img_coordinate(4, i + 1)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
seg_max = seg_layer[0, i, :, :, :].max()
seg = seg_layer[0, i, :, :, :].squeeze() * 1.0 / seg_max
cmap = plt.get_cmap('jet')
seg_map = cmap(seg)[:, :, 0:3]
x1, y1, x2, y2 = self.visualizer.get_img_coordinate(5, i + 1)
img[y1:y2, x1:x2, :] = seg_map
if self.save_display:
img = img * 255.0
img = img.astype(numpy.uint8)
img = Image.fromarray(img)
img.save(os.path.join(self.save_display_dir, 'data.png'))
else:
plt.figure(1)
plt.imshow(img)
plt.axis('off')
plt.show()
def visualize_result(self,
im_input,
im_output,
im_pred,
pred_motion,
gt_motion,
depth,
file_name='tmp.png',
idx=0):
width, height = self.get_img_size(3, max(self.num_frame + 1, 4))
im_channel = self.im_channel
img = numpy.ones((height, width, 3))
prev_im = None
for i in range(self.num_frame - 1):
curr_im = im_input[idx, i * im_channel:(i + 1) *
im_channel, :, :].cpu().data.numpy().transpose(
1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, i + 1)
img[y1:y2, x1:x2, :] = curr_im
if i > 0:
im_diff = abs(curr_im - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(2, i)
img[y1:y2, x1:x2, :] = im_diff
prev_im = curr_im
im_output = im_output[idx].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame)
img[y1:y2, x1:x2, :] = im_output
im_diff = numpy.abs(im_output - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame - 1)
img[y1:y2, x1:x2, :] = im_diff
im_pred = im_pred[idx].cpu().data.numpy().transpose(1, 2, 0)
im_pred[im_pred > 1] = 1
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame + 1)
img[y1:y2, x1:x2, :] = im_pred
im_diff = numpy.abs(im_pred - im_output)
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame)
img[y1:y2, x1:x2, :] = im_diff
pred_motion = pred_motion[idx].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(pred_motion)
x1, y1, x2, y2 = self.get_img_coordinate(3, 1)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
if gt_motion is None:
im = prev_im * 255.0
if im_channel == 1:
prvs_frame = im.astype(numpy.uint8)
elif im_channel == 3:
prvs_frame = cv2.cvtColor(im.astype(numpy.uint8),
cv2.COLOR_RGB2GRAY)
im = im_output * 255.0
if im_channel == 1:
next_frame = im.astype(numpy.uint8)
elif im_channel == 3:
next_frame = cv2.cvtColor(im.astype(numpy.uint8),
cv2.COLOR_RGB2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs_frame, next_frame, None,
0.5, 5, 5, 3, 5, 1.1, 0)
optical_flow = flowlib.visualize_flow(flow)
else:
gt_motion = gt_motion[idx].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(gt_motion)
x1, y1, x2, y2 = self.get_img_coordinate(3, 2)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
depth = depth[idx].cpu().data.numpy().squeeze()
depth = depth * 1.0 / depth.max()
cmap = plt.get_cmap('jet')
depth_map = cmap(depth)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 4)
img[y1:y2, x1:x2, :] = depth_map
if self.save_display:
img = img * 255.0
img = img.astype(numpy.uint8)
img = Image.fromarray(img)
img.save(os.path.join(self.save_display_dir, file_name))
else:
plt.figure(1)
plt.imshow(img)
plt.axis('off')
plt.show()
def visualize_result_bidirect(self,
im_input_f,
im_input_b,
im_output,
im_pred,
pred_motion_f,
gt_motion_f,
depth_f,
gt_depth_f,
appear_f,
conflict_f,
attn_f,
pred_motion_b,
gt_motion_b,
depth_b,
gt_depth_b,
appear_b,
conflict_b,
attn_b,
file_name='tmp.png'):
width, height = self.get_img_size(4, max(self.num_frame + 1, 7))
img = numpy.ones((height, width, 3))
prev_im = None
for i in range(self.num_inputs):
curr_im = im_input_f[0, i * 3:(i + 1) *
3, :, :].cpu().data.numpy().transpose(
1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, i + 1)
img[y1:y2, x1:x2, :] = curr_im
if i > 0:
im_diff = abs(curr_im - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(2, i)
img[y1:y2, x1:x2, :] = im_diff
prev_im = curr_im
im_output = im_output[0].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_inputs + 1)
img[y1:y2, x1:x2, :] = im_output
im_diff = numpy.abs(im_output - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_inputs)
img[y1:y2, x1:x2, :] = im_diff
for i in range(self.num_inputs):
curr_im = im_input_b[0, i * 3:(i + 1) *
3, :, :].cpu().data.numpy().transpose(
1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame - i)
img[y1:y2, x1:x2, :] = curr_im
if i > 0:
im_diff = abs(curr_im - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(
2, self.num_frame - i + 1)
img[y1:y2, x1:x2, :] = im_diff
prev_im = curr_im
im_diff = numpy.abs(im_output - prev_im)
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_inputs + 2)
img[y1:y2, x1:x2, :] = im_diff
pred = im_pred[0].cpu().data.numpy().transpose(1, 2, 0)
x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame + 1)
img[y1:y2, x1:x2, :] = pred
im_diff = numpy.abs(pred - im_output)
x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame + 1)
img[y1:y2, x1:x2, :] = im_diff
pred_motion = pred_motion_f[0].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(pred_motion, self.m_range)
x1, y1, x2, y2 = self.get_img_coordinate(3, 1)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
gt_motion = gt_motion_f[0].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(gt_motion, self.m_range)
x1, y1, x2, y2 = self.get_img_coordinate(3, 2)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
appear = appear_f[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
appear_map = cmap(appear)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 3)
img[y1:y2, x1:x2, :] = appear_map
conflict = conflict_f[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
conflict_map = cmap(conflict)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 4)
img[y1:y2, x1:x2, :] = conflict_map
depth = depth_f[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
depth_map = cmap(depth)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 5)
img[y1:y2, x1:x2, :] = depth_map
depth = gt_depth_f[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
depth_map = cmap(depth)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 6)
img[y1:y2, x1:x2, :] = depth_map
attn = attn_f[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
attn_map = cmap(attn)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(3, 7)
img[y1:y2, x1:x2, :] = attn_map
pred_motion = pred_motion_b[0].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(pred_motion, self.m_range)
x1, y1, x2, y2 = self.get_img_coordinate(4, 1)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
gt_motion = gt_motion_b[0].cpu().data.numpy().transpose(1, 2, 0)
optical_flow = flowlib.visualize_flow(gt_motion, self.m_range)
x1, y1, x2, y2 = self.get_img_coordinate(4, 2)
img[y1:y2, x1:x2, :] = optical_flow / 255.0
appear = appear_b[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
appear_map = cmap(appear)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(4, 3)
img[y1:y2, x1:x2, :] = appear_map
conflict = conflict_b[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
conflict_map = cmap(conflict)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(4, 4)
img[y1:y2, x1:x2, :] = conflict_map
depth = depth_b[0].cpu().data.numpy().squeeze()
depth = depth * 1.0 / depth.max()
cmap = plt.get_cmap('jet')
depth_map = cmap(depth)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(4, 5)
img[y1:y2, x1:x2, :] = depth_map
depth = gt_depth_b[0].cpu().data.numpy().squeeze()
depth = depth * 1.0 / depth.max()
cmap = plt.get_cmap('jet')
depth_map = cmap(depth)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(4, 6)
img[y1:y2, x1:x2, :] = depth_map
attn = attn_b[0].cpu().data.numpy().squeeze()
cmap = plt.get_cmap('jet')
attn_map = cmap(attn)[:, :, 0:3]
x1, y1, x2, y2 = self.get_img_coordinate(4, 7)
img[y1:y2, x1:x2, :] = attn_map
if self.save_display:
img = img * 255.0
img = img.astype(numpy.uint8)
img = Image.fromarray(img)
img.save(os.path.join(self.save_display_dir, file_name))
else:
plt.figure(1)
plt.imshow(img)
plt.axis('off')
plt.show()
