def visualize_result(self, im_input, im_output, im_pred, pred_motion, gt_motion, file_name='tmp.png'): width, height = self.get_img_size(3, max(self.num_frame + 1, 4)) img = numpy.ones((height, width, 3)) prev_im = None for i in range(self.num_frame - 1): curr_im = im_input[0, i*3:(i+1)*3, :, :].cpu().data.numpy().transpose(1, 2, 0) x1, y1, x2, y2 = self.get_img_coordinate(1, i+1) img[y1:y2, x1:x2, :] = curr_im if i > 0: im_diff = abs(curr_im - prev_im) x1, y1, x2, y2 = self.get_img_coordinate(2, i+1) img[y1:y2, x1:x2, :] = im_diff prev_im = curr_im im_output = im_output[0].cpu().data.numpy().transpose(1, 2, 0) x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame) img[y1:y2, x1:x2, :] = im_output im_diff = numpy.abs(im_output - prev_im) x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame) img[y1:y2, x1:x2, :] = im_diff pred = im_pred[0].cpu().data.numpy().transpose(1, 2, 0) x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame + 1) img[y1:y2, x1:x2, :] = pred im_diff = numpy.abs(pred - im_output) x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame + 1) img[y1:y2, x1:x2, :] = im_diff pred_motion = pred_motion[0].cpu().data.numpy().transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(pred_motion, self.m_range) x1, y1, x2, y2 = self.get_img_coordinate(3, 1) img[y1:y2, x1:x2, :] = optical_flow / 255.0 gt_motion = gt_motion[0].cpu().data.numpy().transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(gt_motion, self.m_range) x1, y1, x2, y2 = self.get_img_coordinate(3, 2) img[y1:y2, x1:x2, :] = optical_flow / 255.0 if self.save_display: img = img * 255.0 img = img.astype(numpy.uint8) img = Image.fromarray(img) img.save(os.path.join(self.save_display_dir, file_name)) else: plt.figure(1) plt.imshow(img) plt.axis('off') plt.show()
def display(self, im, motion, motion_r, seg_layer): num_frame = self.num_frame width, height = self.visualizer.get_img_size(5, num_frame) img = numpy.ones((height, width, 3)) prev_im = None for i in range(num_frame): curr_im = im[0, i, :, :, :].transpose(1, 2, 0) x1, y1, x2, y2 = self.visualizer.get_img_coordinate(1, i + 1) img[y1:y2, x1:x2, :] = curr_im if i > 0: im_diff = abs(curr_im - prev_im) x1, y1, x2, y2 = self.visualizer.get_img_coordinate(2, i + 1) img[y1:y2, x1:x2, :] = im_diff prev_im = curr_im flow = motion[0, i, :, :, :].transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(flow, self.m_range) x1, y1, x2, y2 = self.visualizer.get_img_coordinate(3, i + 1) img[y1:y2, x1:x2, :] = optical_flow / 255.0 flow = motion_r[0, i, :, :, :].transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(flow, self.m_range) x1, y1, x2, y2 = self.visualizer.get_img_coordinate(4, i + 1) img[y1:y2, x1:x2, :] = optical_flow / 255.0 seg_max = seg_layer[0, i, :, :, :].max() seg = seg_layer[0, i, :, :, :].squeeze() * 1.0 / seg_max cmap = plt.get_cmap('jet') seg_map = cmap(seg)[:, :, 0:3] x1, y1, x2, y2 = self.visualizer.get_img_coordinate(5, i + 1) img[y1:y2, x1:x2, :] = seg_map if self.save_display: img = img * 255.0 img = img.astype(numpy.uint8) img = Image.fromarray(img) img.save(os.path.join(self.save_display_dir, 'data.png')) else: plt.figure(1) plt.imshow(img) plt.axis('off') plt.show()
def visualize_result(self, im_input, im_output, im_pred, pred_motion, gt_motion, depth, file_name='tmp.png', idx=0): width, height = self.get_img_size(3, max(self.num_frame + 1, 4)) im_channel = self.im_channel img = numpy.ones((height, width, 3)) prev_im = None for i in range(self.num_frame - 1): curr_im = im_input[idx, i * im_channel:(i + 1) * im_channel, :, :].cpu().data.numpy().transpose( 1, 2, 0) x1, y1, x2, y2 = self.get_img_coordinate(1, i + 1) img[y1:y2, x1:x2, :] = curr_im if i > 0: im_diff = abs(curr_im - prev_im) x1, y1, x2, y2 = self.get_img_coordinate(2, i) img[y1:y2, x1:x2, :] = im_diff prev_im = curr_im im_output = im_output[idx].cpu().data.numpy().transpose(1, 2, 0) x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame) img[y1:y2, x1:x2, :] = im_output im_diff = numpy.abs(im_output - prev_im) x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame - 1) img[y1:y2, x1:x2, :] = im_diff im_pred = im_pred[idx].cpu().data.numpy().transpose(1, 2, 0) im_pred[im_pred > 1] = 1 x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame + 1) img[y1:y2, x1:x2, :] = im_pred im_diff = numpy.abs(im_pred - im_output) x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame) img[y1:y2, x1:x2, :] = im_diff pred_motion = pred_motion[idx].cpu().data.numpy().transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(pred_motion) x1, y1, x2, y2 = self.get_img_coordinate(3, 1) img[y1:y2, x1:x2, :] = optical_flow / 255.0 if gt_motion is None: im = prev_im * 255.0 if im_channel == 1: prvs_frame = im.astype(numpy.uint8) elif im_channel == 3: prvs_frame = cv2.cvtColor(im.astype(numpy.uint8), cv2.COLOR_RGB2GRAY) im = im_output * 255.0 if im_channel == 1: next_frame = im.astype(numpy.uint8) elif im_channel == 3: next_frame = cv2.cvtColor(im.astype(numpy.uint8), cv2.COLOR_RGB2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs_frame, next_frame, None, 0.5, 5, 5, 3, 5, 1.1, 0) optical_flow = flowlib.visualize_flow(flow) else: gt_motion = gt_motion[idx].cpu().data.numpy().transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(gt_motion) x1, y1, x2, y2 = self.get_img_coordinate(3, 2) img[y1:y2, x1:x2, :] = optical_flow / 255.0 depth = depth[idx].cpu().data.numpy().squeeze() depth = depth * 1.0 / depth.max() cmap = plt.get_cmap('jet') depth_map = cmap(depth)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(3, 4) img[y1:y2, x1:x2, :] = depth_map if self.save_display: img = img * 255.0 img = img.astype(numpy.uint8) img = Image.fromarray(img) img.save(os.path.join(self.save_display_dir, file_name)) else: plt.figure(1) plt.imshow(img) plt.axis('off') plt.show()
def visualize_result_bidirect(self, im_input_f, im_input_b, im_output, im_pred, pred_motion_f, gt_motion_f, depth_f, gt_depth_f, appear_f, conflict_f, attn_f, pred_motion_b, gt_motion_b, depth_b, gt_depth_b, appear_b, conflict_b, attn_b, file_name='tmp.png'): width, height = self.get_img_size(4, max(self.num_frame + 1, 7)) img = numpy.ones((height, width, 3)) prev_im = None for i in range(self.num_inputs): curr_im = im_input_f[0, i * 3:(i + 1) * 3, :, :].cpu().data.numpy().transpose( 1, 2, 0) x1, y1, x2, y2 = self.get_img_coordinate(1, i + 1) img[y1:y2, x1:x2, :] = curr_im if i > 0: im_diff = abs(curr_im - prev_im) x1, y1, x2, y2 = self.get_img_coordinate(2, i) img[y1:y2, x1:x2, :] = im_diff prev_im = curr_im im_output = im_output[0].cpu().data.numpy().transpose(1, 2, 0) x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_inputs + 1) img[y1:y2, x1:x2, :] = im_output im_diff = numpy.abs(im_output - prev_im) x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_inputs) img[y1:y2, x1:x2, :] = im_diff for i in range(self.num_inputs): curr_im = im_input_b[0, i * 3:(i + 1) * 3, :, :].cpu().data.numpy().transpose( 1, 2, 0) x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame - i) img[y1:y2, x1:x2, :] = curr_im if i > 0: im_diff = abs(curr_im - prev_im) x1, y1, x2, y2 = self.get_img_coordinate( 2, self.num_frame - i + 1) img[y1:y2, x1:x2, :] = im_diff prev_im = curr_im im_diff = numpy.abs(im_output - prev_im) x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_inputs + 2) img[y1:y2, x1:x2, :] = im_diff pred = im_pred[0].cpu().data.numpy().transpose(1, 2, 0) x1, y1, x2, y2 = self.get_img_coordinate(1, self.num_frame + 1) img[y1:y2, x1:x2, :] = pred im_diff = numpy.abs(pred - im_output) x1, y1, x2, y2 = self.get_img_coordinate(2, self.num_frame + 1) img[y1:y2, x1:x2, :] = im_diff pred_motion = pred_motion_f[0].cpu().data.numpy().transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(pred_motion, self.m_range) x1, y1, x2, y2 = self.get_img_coordinate(3, 1) img[y1:y2, x1:x2, :] = optical_flow / 255.0 gt_motion = gt_motion_f[0].cpu().data.numpy().transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(gt_motion, self.m_range) x1, y1, x2, y2 = self.get_img_coordinate(3, 2) img[y1:y2, x1:x2, :] = optical_flow / 255.0 appear = appear_f[0].cpu().data.numpy().squeeze() cmap = plt.get_cmap('jet') appear_map = cmap(appear)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(3, 3) img[y1:y2, x1:x2, :] = appear_map conflict = conflict_f[0].cpu().data.numpy().squeeze() cmap = plt.get_cmap('jet') conflict_map = cmap(conflict)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(3, 4) img[y1:y2, x1:x2, :] = conflict_map depth = depth_f[0].cpu().data.numpy().squeeze() cmap = plt.get_cmap('jet') depth_map = cmap(depth)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(3, 5) img[y1:y2, x1:x2, :] = depth_map depth = gt_depth_f[0].cpu().data.numpy().squeeze() cmap = plt.get_cmap('jet') depth_map = cmap(depth)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(3, 6) img[y1:y2, x1:x2, :] = depth_map attn = attn_f[0].cpu().data.numpy().squeeze() cmap = plt.get_cmap('jet') attn_map = cmap(attn)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(3, 7) img[y1:y2, x1:x2, :] = attn_map pred_motion = pred_motion_b[0].cpu().data.numpy().transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(pred_motion, self.m_range) x1, y1, x2, y2 = self.get_img_coordinate(4, 1) img[y1:y2, x1:x2, :] = optical_flow / 255.0 gt_motion = gt_motion_b[0].cpu().data.numpy().transpose(1, 2, 0) optical_flow = flowlib.visualize_flow(gt_motion, self.m_range) x1, y1, x2, y2 = self.get_img_coordinate(4, 2) img[y1:y2, x1:x2, :] = optical_flow / 255.0 appear = appear_b[0].cpu().data.numpy().squeeze() cmap = plt.get_cmap('jet') appear_map = cmap(appear)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(4, 3) img[y1:y2, x1:x2, :] = appear_map conflict = conflict_b[0].cpu().data.numpy().squeeze() cmap = plt.get_cmap('jet') conflict_map = cmap(conflict)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(4, 4) img[y1:y2, x1:x2, :] = conflict_map depth = depth_b[0].cpu().data.numpy().squeeze() depth = depth * 1.0 / depth.max() cmap = plt.get_cmap('jet') depth_map = cmap(depth)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(4, 5) img[y1:y2, x1:x2, :] = depth_map depth = gt_depth_b[0].cpu().data.numpy().squeeze() depth = depth * 1.0 / depth.max() cmap = plt.get_cmap('jet') depth_map = cmap(depth)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(4, 6) img[y1:y2, x1:x2, :] = depth_map attn = attn_b[0].cpu().data.numpy().squeeze() cmap = plt.get_cmap('jet') attn_map = cmap(attn)[:, :, 0:3] x1, y1, x2, y2 = self.get_img_coordinate(4, 7) img[y1:y2, x1:x2, :] = attn_map if self.save_display: img = img * 255.0 img = img.astype(numpy.uint8) img = Image.fromarray(img) img.save(os.path.join(self.save_display_dir, file_name)) else: plt.figure(1) plt.imshow(img) plt.axis('off') plt.show()