def add_hand_viz(ax, hand_df, joint_idxs=False, score_thresh=0.2): for hand_idx, hand_det in hand_df.iterrows(): joints2d = hand_det.joints2d links = [ (0, 1, 2, 3, 4), (0, 5, 6, 7, 8), (0, 9, 10, 11, 12), (0, 13, 14, 15, 16), (0, 17, 18, 19, 20), ] scores = hand_det.joints2d_scores if scores.shape[0] > 21: kept_links = [] for finger_idxs in links: score_finger_links = [] for link_idx in finger_idxs: if scores[link_idx] > score_thresh: score_finger_links.append(link_idx) kept_links.append(score_finger_links) joint_labels = [ f"{scores[idx]:.2f}" for idx in range(len(joints2d)) ] else: joint_labels = None viz2d.visualize_joints_2d( ax, joints2d, joint_labels=joint_labels, joint_idxs=joint_idxs, links=kept_links, )
def get_hands( hoa_df, img_path, crop_size=256, img_resize_factor=1, hand_mode="opp", scale_factor=1.5, debug=True, ): hand_dicts = [] for det_idx, det in hoa_df.iterrows(): if det.det_type == "hand": det_bbox = [ val * img_resize_factor for val in [det.left, det.top, det.right, det.bottom] ] square_det = preprocess.squarify(det_bbox, scale_factor) img = Image.open(img_path) crop = img.crop(square_det) crop_radius = square_det[2] - square_det[0] # Resize and convert to BGR hand_crop = cv2.resize(np.array(crop), (crop_size, crop_size))[:, :, ::-1] if det.side == "left": # Process all hands as right hands joints2d, peak_scores = hand_opp(np.flip(hand_crop, axis=1)) joints2d = postprocess.flip_coords( joints2d, crop_size=crop_size, axis=0 ) else: joints2d, peak_scores = hand_opp(hand_crop) mean_scores = np.stack(peak_scores).mean(0)[0] # joints2d_abs = joints2d / crop_size * crop_radius + np.array([det_bbox[0], det_bbox[1]]) joints2d_abs = joints2d / crop_size * crop_radius + np.array( [square_det[0], square_det[1]] ) if debug: fig, ax = plt.subplots(1) ax.imshow(img) # scatter(joints2d_abs[:, 0], joints2d_abs[:, 1], s=1) viz2d.visualize_joints_2d(ax, joints2d_abs, joint_idxs=False) fig.savefig("tmp.png") plt.clf() fig, ax = plt.subplots(1) ax.imshow(hand_crop[:, :, ::-1]) viz2d.visualize_joints_2d(ax, joints2d, joint_idxs=False) fig.savefig("tmp_crop.png") det_dict = det.to_dict() det_dict["joints2d"] = joints2d_abs det_dict["joints2d_scores"] = mean_scores hand_dicts.append(det_dict) return pd.DataFrame(hand_dicts)
def sample_vis(sample, results, save_img_path, fig=None, max_rows=5, display_centered=False): fig.clf() images = sample[TransQueries.IMAGE].permute(0, 2, 3, 1).cpu() + 0.5 batch_size = images.shape[0] # pred_handverts2d = get_check_none(results, "verts2d") gt_objverts2d = get_check_none(sample, TransQueries.OBJVERTS2D) pred_objverts2d = get_check_none(results, "obj_verts2d") gt_objcorners2d = get_check_none(sample, TransQueries.OBJCORNERS2D) pred_objcorners2d = get_check_none(results, "obj_corners2d") gt_objcorners3dw = get_check_none(sample, BaseQueries.OBJCORNERS3D) pred_objcorners3d = get_check_none(results, "obj_corners3d") gt_objverts3d = get_check_none(sample, TransQueries.OBJVERTS3D) pred_objverts3d = get_check_none(results, "obj_verts3d") gt_canobjverts3d = get_check_none(sample, TransQueries.OBJCANROTVERTS) pred_objverts3dw = get_check_none(results, "recov_objverts3d") gt_canobjcorners3d = get_check_none(sample, TransQueries.OBJCANROTCORNERS) pred_objcorners3dw = get_check_none(results, "recov_objcorners3d") gt_handjoints2d = get_check_none(sample, TransQueries.JOINTS2D) pred_handjoints2d = get_check_none(results, "joints2d") gt_handjoints3d = get_check_none(sample, TransQueries.JOINTS3D) pred_handjoints3d = get_check_none(results, "joints3d") gt_handverts3d = get_check_none(sample, TransQueries.HANDVERTS3D) pred_handverts3d = get_check_none(results, "verts3d") gt_objverts3dw = get_check_none(sample, BaseQueries.OBJVERTS3D) pred_handjoints3dw = get_check_none(results, "recov_joints3d") gt_handjoints3dw = get_check_none(sample, BaseQueries.JOINTS3D) pred_objfps2d = get_check_none(results, "kpt_2d") gt_objfps2d = get_check_none(sample, BaseQueries.OBJFPS2D) pred_objvar2d = get_check_none(results, "var") # gt_drill_angle_X, gt_drill_angle_Y, tmp = compute_gt_drill_angles(sample) row_nb = min(max_rows, batch_size) if display_centered: col_nb = 7 else: col_nb = 5 axes = fig.subplots(row_nb, col_nb) for row_idx in range(row_nb): # Column 0 col_idx = 0 axes[row_idx, col_idx].imshow(images[row_idx]) axes[row_idx, col_idx].axis("off") # Visualize 2D hand joints if pred_handjoints2d is not None: visualize_joints_2d(axes[row_idx, col_idx], pred_handjoints2d[row_idx], alpha=1, joint_idxs=False) if gt_handjoints2d is not None: visualize_joints_2d(axes[row_idx, col_idx], gt_handjoints2d[row_idx], alpha=0.5, joint_idxs=False) # Column 1 col_idx = 1 axes[row_idx, col_idx].imshow(images[row_idx]) axes[row_idx, col_idx].axis("off") # axes[row_idx, col_idx].set_title("dvec: {:.2f},{:.2f},{:.2f}".format(tmp[row_idx, 0, 0], tmp[row_idx, 0, 1], tmp[row_idx, 0, 2])) if gt_objfps2d is not None and pred_objfps2d is not None: arrow_nb = gt_objfps2d.shape[1] idxs = range(arrow_nb) arrows = torch.cat([gt_objfps2d[:, idxs].float(), pred_objfps2d[:, idxs].float()], 1) links = [[i, i + arrow_nb] for i in idxs] visualize_joints_2d( axes[row_idx, col_idx], arrows[row_idx], alpha=0.5, joint_idxs=False, links=links, color=["k"] * arrow_nb, ) if pred_objvar2d is not None: ells = compute_confidence_ellipses(pred_objfps2d[row_idx], pred_objvar2d[row_idx]) for e in ells: axes[row_idx, col_idx].add_artist(e) if pred_objfps2d is not None: axes[row_idx, col_idx].scatter( pred_objfps2d[row_idx, :, 0], pred_objfps2d[row_idx, :, 1], c="r", s=2, marker="X", alpha=0.7 ) if gt_objfps2d is not None: axes[row_idx, col_idx].scatter( gt_objfps2d[row_idx, :, 0], gt_objfps2d[row_idx, :, 1], c="b", s=2, marker="X", alpha=0.7 ) # if gt_objfps2d is not None: # axes[row_idx, col_idx].scatter( # gt_objfps2d[row_idx, [0,5], 0], gt_objfps2d[row_idx, [0,5], 1], c="g", s=4, marker="o", alpha=0.7 # ) # Column 2 col_idx = 2 axes[row_idx, col_idx].imshow(images[row_idx]) axes[row_idx, col_idx].axis("off") # Visualize 2D object vertices if pred_objverts2d is not None: axes[row_idx, col_idx].scatter( pred_objverts2d[row_idx, :, 0], pred_objverts2d[row_idx, :, 1], c="r", s=1, alpha=0.1 ) if gt_objverts2d is not None: axes[row_idx, col_idx].scatter( gt_objverts2d[row_idx, :, 0], gt_objverts2d[row_idx, :, 1], c="b", s=1, alpha=0.02 ) # Visualize 2D object bounding box # if pred_objcorners2d is not None: # visualize_joints_2d( # axes[row_idx, col_idx], # pred_objcorners2d[row_idx], # alpha=1, # joint_idxs=False, # links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], # ) # if gt_objcorners2d is not None: # visualize_joints_2d( # axes[row_idx, col_idx], # gt_objcorners2d[row_idx], # alpha=0.5, # joint_idxs=False, # links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], # ) # Visualize some (vertex position) errors for the 2D object vertices if (gt_objfps2d is None or pred_objfps2d is None) and gt_objverts2d is not None and pred_objverts2d is not None: idxs = list(range(6)) arrow_nb = len(idxs) arrows = torch.cat([gt_objverts2d[:, idxs].float(), pred_objverts2d[:, idxs].float()], 1) links = [[i, i + arrow_nb] for i in range(arrow_nb)] visualize_joints_2d( axes[row_idx, col_idx], arrows[row_idx], alpha=0.5, joint_idxs=False, links=links, color=["k"] * arrow_nb, ) # Column 3 # view from the top col_idx = 3 #axes[row_idx, col_idx].set_title("rotY: {:.1f}".format(gt_drill_angle_Y[row_idx])) if gt_objverts3dw is not None: axes[row_idx, col_idx].scatter( gt_objverts3dw[row_idx, :, 2], gt_objverts3dw[row_idx, :, 0], c="b", s=1, alpha=0.02 ) if pred_objverts3dw is not None: axes[row_idx, col_idx].scatter( pred_objverts3dw[row_idx, :, 2], pred_objverts3dw[row_idx, :, 0], c="r", s=1, alpha=0.02 ) if pred_handjoints3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_handjoints3dw[row_idx, :, [2,0]], alpha=1, joint_idxs=False ) if gt_handjoints3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_handjoints3dw[row_idx, :, [2,0]], alpha=0.5, joint_idxs=False ) axes[row_idx, col_idx].invert_yaxis() # if pred_objcorners3dw is not None: # visualize_joints_2d( # axes[row_idx, col_idx], # pred_objcorners3dw[row_idx], # alpha=1, # joint_idxs=False, # links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], # ) # if gt_objcorners3dw is not None: # visualize_joints_2d( # axes[row_idx, col_idx], # gt_objcorners3dw[row_idx], # alpha=0.5, # joint_idxs=False, # links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], # ) # if pred_objverts3dw is not None and gt_objverts3dw is not None: # arrow_nb = 6 # arrows = torch.cat([gt_objverts3dw[:, :arrow_nb], pred_objverts3dw[:, :arrow_nb]], 1) # links = [[i, i + arrow_nb] for i in range(arrow_nb)] # visualize_joints_2d( # axes[row_idx, col_idx], # arrows[row_idx], # alpha=0.5, # joint_idxs=False, # links=links, # color=["k"] * arrow_nb, # ) # Column 4 # view from the right col_idx = 4 #axes[row_idx, col_idx].set_title("rotX: {:.1f}".format(gt_drill_angle_X[row_idx])) # invert second axis here for more consistent viewpoints if gt_objverts3dw is not None: axes[row_idx, col_idx].scatter( gt_objverts3dw[row_idx, :, 2], -gt_objverts3dw[row_idx, :, 1], c="b", s=1, alpha=0.02 ) if pred_objverts3dw is not None: axes[row_idx, col_idx].scatter( pred_objverts3dw[row_idx, :, 2], -pred_objverts3dw[row_idx, :, 1], c="r", s=1, alpha=0.02 ) if pred_handjoints3dw is not None: pred_handjoints3dw_inv = np.stack([pred_handjoints3dw[:, :, 2], -pred_handjoints3dw[:, :, 1]], axis=-1) visualize_joints_2d( axes[row_idx, col_idx], pred_handjoints3dw_inv[row_idx, :, :], alpha=1, joint_idxs=False ) if gt_handjoints3dw is not None: gt_handjoints3dw_inv = np.stack([gt_handjoints3dw[:, :, 2], -gt_handjoints3dw[:, :, 1]], axis=-1) visualize_joints_2d( axes[row_idx, col_idx], gt_handjoints3dw_inv[row_idx, :, :], alpha=0.5, joint_idxs=False ) # if pred_objcorners3dw is not None: # visualize_joints_2d( # axes[row_idx, col_idx], # pred_objcorners3dw[row_idx, :, 1:], # alpha=1, # joint_idxs=False, # links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], # ) # if gt_objcorners3dw is not None: # visualize_joints_2d( # axes[row_idx, col_idx], # gt_objcorners3dw[row_idx, :, 1:], # alpha=0.5, # joint_idxs=False, # links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], # ) # if pred_objverts3dw is not None and gt_objverts3dw is not None: # arrow_nb = 6 # arrows = torch.cat([gt_objverts3dw[:, :arrow_nb, 1:], pred_objverts3dw[:, :arrow_nb, 1:]], 1) # links = [[i, i + arrow_nb] for i in range(arrow_nb)] # visualize_joints_2d( # axes[row_idx, col_idx], # arrows[row_idx], # alpha=0.5, # joint_idxs=False, # links=links, # color=["k"] * arrow_nb, # ) if display_centered: # Column 5 col_idx = 5 if gt_canobjverts3d is not None: axes[row_idx, col_idx].scatter( gt_canobjverts3d[row_idx, :, 0], gt_canobjverts3d[row_idx, :, 1], c="b", s=1, alpha=0.02 ) if pred_objverts3d is not None: axes[row_idx, col_idx].scatter( pred_objverts3d[row_idx, :, 0], pred_objverts3d[row_idx, :, 1], c="r", s=1, alpha=0.02 ) if pred_objcorners3d is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_objcorners3d[row_idx], alpha=1, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if gt_canobjcorners3d is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_canobjcorners3d[row_idx], alpha=0.5, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if pred_objcorners3d is not None and gt_canobjcorners3d is not None: arrow_nb = 6 arrows = torch.cat([gt_canobjcorners3d[:, :arrow_nb], pred_objcorners3d[:, :arrow_nb]], 1) links = [[i, i + arrow_nb] for i in range(arrow_nb)] visualize_joints_2d( axes[row_idx, col_idx], arrows[row_idx], alpha=0.5, joint_idxs=False, links=links, color=["k"] * arrow_nb, ) axes[row_idx, col_idx].set_aspect("equal") axes[row_idx, col_idx].invert_yaxis() # Column 6 col_idx = 6 if gt_objverts3d is not None: axes[row_idx, col_idx].scatter( gt_objverts3d[row_idx, :, 0], gt_objverts3d[row_idx, :, 1], c="b", s=1, alpha=0.02 ) # if pred_objverts3d is not None: # axes[row_idx, 2].scatter( # pred_objverts3d[row_idx, :, 0], pred_objverts3d[row_idx, :, 1], c="r", s=1, alpha=0.02 # ) if gt_handverts3d is not None: axes[row_idx, col_idx].scatter( gt_handverts3d[row_idx, :, 0], gt_handverts3d[row_idx, :, 1], c="g", s=1, alpha=0.2 ) if pred_handverts3d is not None: axes[row_idx, col_idx].scatter( pred_handverts3d[row_idx, :, 0], pred_handverts3d[row_idx, :, 1], c="c", s=1, alpha=0.2 ) if pred_handjoints3d is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_handjoints3d[row_idx], alpha=1, joint_idxs=False ) if gt_handjoints3d is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_handjoints3d[row_idx], alpha=0.5, joint_idxs=False ) axes[row_idx, col_idx].invert_yaxis() # Column 7 col_idx = 7 if gt_objverts3d is not None: axes[row_idx, col_idx].scatter( gt_objverts3d[row_idx, :, 1], gt_objverts3d[row_idx, :, 2], c="b", s=1, alpha=0.02 ) # if pred_objverts3d is not None: # axes[row_idx, 3].scatter( # pred_objverts3d[row_idx, :, 1], pred_objverts3d[row_idx, :, 2], c="r", s=1, alpha=0.02 # ) if gt_handverts3d is not None: axes[row_idx, col_idx].scatter( gt_handverts3d[row_idx, :, 1], gt_handverts3d[row_idx, :, 2], c="g", s=1, alpha=0.2 ) if pred_handverts3d is not None: axes[row_idx, col_idx].scatter( pred_handverts3d[row_idx, :, 1], pred_handverts3d[row_idx, :, 2], c="c", s=1, alpha=0.2 ) if pred_handjoints3d is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_handjoints3d[row_idx][:, 1:], alpha=1, joint_idxs=False ) if gt_handjoints3d is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_handjoints3d[row_idx][:, 1:], alpha=0.5, joint_idxs=False ) consistdisplay.squashfig(fig) fig.savefig(save_img_path, dpi=300)
def sample_vis(sample, results, save_img_path, fig=None, max_rows=5, display_centered=False): fig.clf() images = sample[TransQueries.IMAGE].permute(0, 2, 3, 1).cpu() + 0.5 batch_size = images.shape[0] # pred_handverts2d = get_check_none(results, "verts2d") gt_objverts2d = get_check_none(sample, TransQueries.OBJVERTS2D) pred_objverts2d = get_check_none(results, "obj_verts2d") gt_objcorners2d = get_check_none(sample, TransQueries.OBJCORNERS2D) pred_objcorners2d = get_check_none(results, "obj_corners2d") gt_objcorners3dw = get_check_none(sample, BaseQueries.OBJCORNERS3D) pred_objcorners3d = get_check_none(results, "obj_corners3d") gt_objverts3d = get_check_none(sample, TransQueries.OBJVERTS3D) gt_canobjverts3d = get_check_none(sample, TransQueries.OBJCANROTVERTS) gt_canobjcorners3d = get_check_none(sample, TransQueries.OBJCANROTCORNERS) pred_objverts3d = get_check_none(results, "obj_verts3d") gt_handjoints2d = get_check_none(sample, TransQueries.JOINTS2D) pred_handjoints2d = get_check_none(results, "joints2d") gt_handjoints3d = get_check_none(sample, TransQueries.JOINTS3D) pred_handjoints3d = get_check_none(results, "joints3d") gt_handverts3d = get_check_none(sample, TransQueries.HANDVERTS3D) gt_objverts3dw = get_check_none(sample, BaseQueries.OBJVERTS3D) pred_handjoints3dw = get_check_none(results, "recov_joints3d") gt_handjoints3dw = get_check_none(sample, BaseQueries.JOINTS3D) pred_objverts3dw = get_check_none(results, "recov_objverts3d") pred_objcorners3dw = get_check_none(results, "recov_objcorners3d") pred_handverts3d = get_check_none(results, "verts3d") row_nb = min(max_rows, batch_size) if display_centered: col_nb = 7 else: col_nb = 4 axes = fig.subplots(row_nb, col_nb) for row_idx in range(row_nb): # Column 0 axes[row_idx, 0].imshow(images[row_idx]) axes[row_idx, 0].axis("off") if pred_handjoints2d is not None: visualize_joints_2d(axes[row_idx, 0], pred_handjoints2d[row_idx], alpha=1, joint_idxs=False) if gt_handjoints2d is not None: visualize_joints_2d(axes[row_idx, 0], gt_handjoints2d[row_idx], alpha=0.5, joint_idxs=False) # Column 1 axes[row_idx, 1].imshow(images[row_idx]) axes[row_idx, 1].axis("off") if pred_objverts2d is not None: axes[row_idx, 1].scatter( pred_objverts2d[row_idx, :, 0], pred_objverts2d[row_idx, :, 1], c="r", s=1, alpha=0.2 ) if gt_objverts2d is not None: axes[row_idx, 1].scatter( gt_objverts2d[row_idx, :, 0], gt_objverts2d[row_idx, :, 1], c="b", s=1, alpha=0.02 ) if pred_objcorners2d is not None: visualize_joints_2d( axes[row_idx, 1], pred_objcorners2d[row_idx], alpha=1, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if gt_objcorners2d is not None: visualize_joints_2d( axes[row_idx, 1], gt_objcorners2d[row_idx], alpha=0.5, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if gt_objverts2d is not None and pred_objverts2d is not None: idxs = list(range(6)) arrow_nb = len(idxs) arrows = torch.cat([gt_objverts2d[:, idxs].float(), pred_objverts2d[:, idxs].float()], 1) links = [[i, i + arrow_nb] for i in range(arrow_nb)] visualize_joints_2d( axes[row_idx, 1], arrows[row_idx], alpha=0.5, joint_idxs=False, links=links, color=["k"] * arrow_nb, ) # Column 2 col_idx = 2 if gt_objverts3dw is not None: axes[row_idx, col_idx].scatter( gt_objverts3dw[row_idx, :, 0], gt_objverts3dw[row_idx, :, 1], c="b", s=1, alpha=0.02 ) if pred_objverts3dw is not None: axes[row_idx, col_idx].scatter( pred_objverts3dw[row_idx, :, 0], pred_objverts3dw[row_idx, :, 1], c="r", s=1, alpha=0.02 ) if pred_handjoints3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_handjoints3dw[row_idx][:], alpha=1, joint_idxs=False ) if gt_handjoints3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_handjoints3dw[row_idx][:], alpha=0.5, joint_idxs=False ) axes[row_idx, col_idx].invert_yaxis() if pred_objcorners3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_objcorners3dw[row_idx], alpha=1, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if gt_objcorners3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_objcorners3dw[row_idx], alpha=0.5, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if pred_objverts3dw is not None and gt_objverts3dw is not None: arrow_nb = 6 arrows = torch.cat([gt_objverts3dw[:, :arrow_nb], pred_objverts3dw[:, :arrow_nb]], 1) links = [[i, i + arrow_nb] for i in range(arrow_nb)] visualize_joints_2d( axes[row_idx, col_idx], arrows[row_idx], alpha=0.5, joint_idxs=False, links=links, color=["k"] * arrow_nb, ) # Column 3 col_idx = 3 if gt_objverts3dw is not None: axes[row_idx, col_idx].scatter( gt_objverts3dw[row_idx, :, 1], gt_objverts3dw[row_idx, :, 2], c="b", s=1, alpha=0.02 ) if pred_objverts3dw is not None: axes[row_idx, col_idx].scatter( pred_objverts3dw[row_idx, :, 1], pred_objverts3dw[row_idx, :, 2], c="r", s=1, alpha=0.02 ) if pred_handjoints3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_handjoints3dw[row_idx][:, 1:], alpha=1, joint_idxs=False ) if gt_handjoints3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_handjoints3dw[row_idx][:, 1:], alpha=0.5, joint_idxs=False ) if pred_objcorners3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_objcorners3dw[row_idx, :, 1:], alpha=1, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if gt_objcorners3dw is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_objcorners3dw[row_idx, :, 1:], alpha=0.5, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if pred_objverts3dw is not None and gt_objverts3dw is not None: arrow_nb = 6 arrows = torch.cat([gt_objverts3dw[:, :arrow_nb, 1:], pred_objverts3dw[:, :arrow_nb, 1:]], 1) links = [[i, i + arrow_nb] for i in range(arrow_nb)] visualize_joints_2d( axes[row_idx, col_idx], arrows[row_idx], alpha=0.5, joint_idxs=False, links=links, color=["k"] * arrow_nb, ) if display_centered: # Column 4 col_idx = 4 if gt_canobjverts3d is not None: axes[row_idx, col_idx].scatter( gt_canobjverts3d[row_idx, :, 0], gt_canobjverts3d[row_idx, :, 1], c="b", s=1, alpha=0.02 ) if pred_objverts3d is not None: axes[row_idx, col_idx].scatter( pred_objverts3d[row_idx, :, 0], pred_objverts3d[row_idx, :, 1], c="r", s=1, alpha=0.02 ) if pred_objcorners3d is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_objcorners3d[row_idx], alpha=1, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if gt_canobjcorners3d is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_canobjcorners3d[row_idx], alpha=0.5, joint_idxs=False, links=[[0, 1, 3, 2], [4, 5, 7, 6], [1, 5], [3, 7], [4, 0], [0, 2, 6, 4]], ) if pred_objcorners3d is not None and gt_canobjcorners3d is not None: arrow_nb = 6 arrows = torch.cat([gt_canobjcorners3d[:, :arrow_nb], pred_objcorners3d[:, :arrow_nb]], 1) links = [[i, i + arrow_nb] for i in range(arrow_nb)] visualize_joints_2d( axes[row_idx, col_idx], arrows[row_idx], alpha=0.5, joint_idxs=False, links=links, color=["k"] * arrow_nb, ) axes[row_idx, col_idx].set_aspect("equal") axes[row_idx, col_idx].invert_yaxis() # Column 5 col_idx = 5 if gt_objverts3d is not None: axes[row_idx, col_idx].scatter( gt_objverts3d[row_idx, :, 0], gt_objverts3d[row_idx, :, 1], c="b", s=1, alpha=0.02 ) # if pred_objverts3d is not None: # axes[row_idx, 2].scatter( # pred_objverts3d[row_idx, :, 0], pred_objverts3d[row_idx, :, 1], c="r", s=1, alpha=0.02 # ) if gt_handverts3d is not None: axes[row_idx, col_idx].scatter( gt_handverts3d[row_idx, :, 0], gt_handverts3d[row_idx, :, 1], c="g", s=1, alpha=0.2 ) if pred_handverts3d is not None: axes[row_idx, col_idx].scatter( pred_handverts3d[row_idx, :, 0], pred_handverts3d[row_idx, :, 1], c="c", s=1, alpha=0.2 ) if pred_handjoints3d is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_handjoints3d[row_idx], alpha=1, joint_idxs=False ) if gt_handjoints3d is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_handjoints3d[row_idx], alpha=0.5, joint_idxs=False ) axes[row_idx, col_idx].invert_yaxis() # Column 6 col_idx = 6 if gt_objverts3d is not None: axes[row_idx, col_idx].scatter( gt_objverts3d[row_idx, :, 1], gt_objverts3d[row_idx, :, 2], c="b", s=1, alpha=0.02 ) # if pred_objverts3d is not None: # axes[row_idx, 3].scatter( # pred_objverts3d[row_idx, :, 1], pred_objverts3d[row_idx, :, 2], c="r", s=1, alpha=0.02 # ) if gt_handverts3d is not None: axes[row_idx, col_idx].scatter( gt_handverts3d[row_idx, :, 1], gt_handverts3d[row_idx, :, 2], c="g", s=1, alpha=0.2 ) if pred_handverts3d is not None: axes[row_idx, col_idx].scatter( pred_handverts3d[row_idx, :, 1], pred_handverts3d[row_idx, :, 2], c="c", s=1, alpha=0.2 ) if pred_handjoints3d is not None: visualize_joints_2d( axes[row_idx, col_idx], pred_handjoints3d[row_idx][:, 1:], alpha=1, joint_idxs=False ) if gt_handjoints3d is not None: visualize_joints_2d( axes[row_idx, col_idx], gt_handjoints3d[row_idx][:, 1:], alpha=0.5, joint_idxs=False ) consistdisplay.squashfig(fig) fig.savefig(save_img_path, dpi=300)
frame = tarutils.cv2_imread_tar(tarf, frame_subpath) else: video_folder = os.path.join(args.epic_root, video_full_id[:3], video_full_id) frame = cv2.imread(os.path.join(video_folder, frame_name)) if not args.without_image: ax.imshow(frame[:, :, ::-1]) else: ax.imshow(np.zeros_like(frame[:, :, ::-1])) ax.set_title(action_segms[frame_idx]) ax.axis("off") box_annots = box_props[frame_idx] for hand in hands: viz2d.visualize_joints_2d(ax, hand, joint_idxs=False) for bboxes, noun in box_annots: bboxes_norm = [ epic_box_to_norm(bbox, resize_factor=resize_factor) for bbox in bboxes ] label_color = "w" detect2d.visualize_bboxes( ax, bboxes_norm, labels=[ noun, ] * len(bboxes), label_color=label_color, ) if args.rename: