def preprocess(self, frame, pose): trans_matrix, image = rotate_and_crop(frame, self.aug_conf["angle"], self.center, self.window_size) size_after_rotate = torch.FloatTensor([image.shape[1], image.shape[0]]) image = resize(image, (self.final_size, self.final_size), preserve_range=True) trans_matrix = scale(trans_matrix, self.final_size / size_after_rotate[0], self.final_size / size_after_rotate[1]) # randomly flip horizontal if self.aug_conf["flip"]: image = np.fliplr(image) trans_matrix = translate(trans_matrix, -image.shape[1] / 2, -image.shape[0] / 2) trans_matrix = flip_h(trans_matrix) trans_matrix = translate(trans_matrix, image.shape[1] / 2, image.shape[0] / 2) # normalize pose to [0,1] and image to [-1, 1] normalized_image = normalize_channels(image) trans_matrix = scale(trans_matrix, 1.0 / self.final_size, 1.0 / self.final_size) transformed_pose = transform_pose(trans_matrix, pose) trans_matrix = torch.from_numpy(trans_matrix).float() normalized_image = torch.from_numpy(normalized_image).float() transformed_pose = torch.from_numpy(transformed_pose).float() assert normalized_image.max() <= 1 and normalized_image.min() >= -1 return trans_matrix, normalized_image, transformed_pose
def eval_pckh_batch(predictions, poses, headsizes, matrices, return_perjoint=False): scores_05 = [] scores_02 = [] matches_05 = [] matches_02 = [] valids_02 = [] valids_05 = [] for i, prediction in enumerate(predictions): pred_pose = prediction[:, 0:2] ground_pose = poses[i, :, 0:2] # transform coordinates back to original pred_pose = transform_pose(matrices[i], pred_pose, inverse=True) ground_pose = transform_pose(matrices[i], ground_pose, inverse=True) match, valid, scores = pckh(ground_pose, pred_pose, headsizes[i]) if torch.cuda.is_available(): matches_05.append(match.cpu().numpy()) valids_05.append(valid.cpu().numpy()) else: matches_05.append(match.numpy()) valids_05.append(valid.numpy()) scores_05.append(scores) match, scores = pckh(ground_pose, pred_pose, headsizes[i], distance_threshold=0.2) if torch.cuda.is_available(): matches_02.append(match.cpu().numpy()) valids_02.append(valid.cpu().numpy()) else: matches_02.append(match.numpy()) valids_02.append(valid.numpy()) scores_02.append(scores) if return_perjoint: return matches_05, matches_02, valids_05, valids_02 else: return scores_05, scores_02
def eval_pck_batch(predictions, poses, matrices, distance_meassures, threshold=0.2, return_perjoint=False): scores_02 = [] matches = [] valids = [] for i, prediction in enumerate(predictions): # transform coordinates back to original pred_pose = torch.from_numpy(transform_pose(matrices[i], predictions[i], inverse=True)) ground_pose = torch.from_numpy(transform_pose(matrices[i], poses[i], inverse=True)) match, valid, scores = pck_bounding_box(ground_pose, pred_pose, distance_meassures[i], distance_threshold=threshold) scores_02.append(scores) if torch.cuda.is_available(): matches.append(match.cpu().numpy()) valids.append(valid.cpu().numpy()) else: valids.append(valid.numpy()) matches.append(match.numpy()) if return_perjoint: return matches, valids else: return scores_02
def plot(pose, matrix, path): # plt.tick_params( # axis='both', # changes apply to the x-axis # which='both', # both major and minor ticks are affected # bottom=False, # ticks along the bottom edge are off # right=False, # left=False, # top=False, # ticks along the top edge are off # labelbottom=False) # labels along the bottom edge are off plt.axis("off") vis = pose[:, 2] > 0.5 pose = pose[:, 0:2] pose = torch.from_numpy(transform_pose(matrix, pose, inverse=True)).float() pred_x = pose[:, 0] pred_y = pose[:, 1] s = 10 for _, (src, dst) in enumerate(joint_mapping): if not vis[src]: continue if vis[dst]: plt.plot([pred_x[src], pred_x[dst]], [pred_y[src], pred_y[dst]], lw=1, c="#00FF00") plt.scatter(x=pred_x[0:3], y=pred_y[0:3], c="#FF00FF", s=s) # right plt.scatter(x=pred_x[10:13], y=pred_y[10:13], c="#FF00FF", s=s) # right plt.scatter(x=pred_x[3:6], y=pred_y[3:6], c="#FFFF00", s=s) # left plt.scatter(x=pred_x[13:16], y=pred_y[13:16], c="#FFFF00", s=s) # left plt.scatter(x=pred_x[6:10], y=pred_y[6:10], c="#00FFFF", s=s) # rest plt.tight_layout() plt.savefig(path) plt.close()
def show_predictions_ontop(ground_truth, image, poses, path, matrix, bbox=None, save=True): plt.xticks([]) plt.yticks([]) ground_pose = ground_truth[:, 0:2] predicted_pose = poses[:, 0:2] plt.imshow(image) orig_gt_coordinates = transform_pose(matrix, ground_pose, inverse=True) orig_pred_coordinates = transform_pose(matrix, predicted_pose, inverse=True) gt_color = "r" pred_color = "b" bbox_color = "#ff00ff" for i, (src, dst) in enumerate(joint_mapping): if not ground_truth[src, 2]: continue if ground_truth[dst, 2]: #print("{} => {}".format(mpii_joint_order[src], mpii_joint_order[dst])) plt.plot( [orig_gt_coordinates[src][0], orig_gt_coordinates[dst][0]], [orig_gt_coordinates[src][1], orig_gt_coordinates[dst][1]], lw=1, c=gt_color) plt.plot( [orig_pred_coordinates[src][0], orig_pred_coordinates[dst][0]], [orig_pred_coordinates[src][1], orig_pred_coordinates[dst][1]], lw=1, c=pred_color) plt.scatter(orig_gt_coordinates[src][0], orig_gt_coordinates[src][1], label="{}".format(mpii_joint_order[src]), c=joint_colors[src]) plt.scatter(orig_pred_coordinates[src][0], orig_pred_coordinates[src][1], c=joint_colors[src]) plt.scatter(orig_gt_coordinates[dst][0], orig_gt_coordinates[dst][1], label="{}".format(mpii_joint_order[dst]), c=joint_colors[dst]) plt.scatter(orig_pred_coordinates[dst][0], orig_pred_coordinates[dst][1], c=joint_colors[dst]) else: #print("{}".format(mpii_joint_order[src])) plt.scatter(orig_gt_coordinates[src][0], orig_gt_coordinates[src][1], label="{}".format(mpii_joint_order[src]), c=joint_colors[src]) plt.scatter(orig_pred_coordinates[src][0], orig_pred_coordinates[src][1], c=joint_colors[src]) if bbox is not None: bbox_rect_original = Rectangle((bbox[0], bbox[1]), abs(bbox[0] - bbox[2]), abs(bbox[1] - bbox[3]), linewidth=1, facecolor='none', edgecolor=bbox_color) ax = plt.gca() ax.add_patch(bbox_rect_original) # remove duplicate handles in legend (src: https://stackoverflow.com/questions/13588920/stop-matplotlib-repeating-labels-in-legend) handles, labels = plt.gca().get_legend_handles_labels() by_label = OrderedDict(zip(labels, handles)) plt.legend(by_label.values(), by_label.keys()) if save: if os.path.isfile(path): os.remove(path) plt.savefig(path) else: plt.show() plt.close()
plt.subplot(122) matrix = entry["trans_matrices"][frame] index = int(entry["index"].item()) item_path = "/data/mjakobs/data/pennaction/frames/" + ds.items[ ds.indices[idx]] all_frames = sorted(glob.glob(item_path + "/*.jpg")) image_path = all_frames[frame] image = io.imread(image_path) plt.imshow(image) pose = torch.from_numpy( transform_pose(matrix, pose[:, 0:2], inverse=True)).float() x = (pose[:, 0] * vis).numpy() y = (pose[:, 1] * vis).numpy() x[x == 0] = None y[y == 0] = None plt.scatter(x=x, y=y, c="#FF00FF", s=10) for i, (src, dst) in enumerate(joint_mapping): if not vis[src]: continue if vis[dst]: plt.plot([x[src], x[dst]], [y[src], y[dst]], lw=1,
if vis[dst]: plt.plot([x[src], x[dst]], [y[src], y[dst]], lw=1, c="#00FFFF") plt.subplot(122) matrix = entry["trans_matrices"][frame] indices = entry["indices"].int() item_path = ds_full.items[indices[2]] all_frames = sorted(glob.glob(item_path + "/*.png")) image_path = all_frames[indices[0] + frame] image = io.imread(image_path) plt.imshow(image) pose = torch.from_numpy(transform_pose(matrix, pose[:, 0:2], inverse=True)).float() x = (pose[:, 0] * vis).numpy() y = (pose[:, 1] * vis).numpy() x[x == 0] = None y[y == 0] = None plt.scatter(x=x, y=y, c="#FF00FF", s=10) for i, (src, dst) in enumerate(joint_mapping): if not vis[src]: continue if vis[dst]: plt.plot([x[src], x[dst]], [y[src], y[dst]], lw=1, c="#00FFFF")
def plot_mpii_augmented(): ds = MPIIDataset("/data/mjakobs/data/mpii/", train=True, val=False, use_random_parameters=False, use_saved_tensors=False) ds_augmented = MPIIDataset("/data/mjakobs/data/mpii/", train=True, val=False, use_random_parameters=True, use_saved_tensors=False) ds_augmented.angles = np.array([-15]) ds_augmented.scales = np.array([1.3]) ds_augmented.flip_horizontal = np.array([1]) entry = ds[20] image = entry["normalized_image"] image = image.permute(1, 2, 0) image = (image + 1) / 2.0 plt.subplot(1, 4, 3) plt.imshow(image) pose = entry["normalized_pose"] scaled_pose = pose * 255.0 for i, (src, dst) in enumerate(joint_mapping): plt.plot([scaled_pose[src][0], scaled_pose[dst][0]], [scaled_pose[src][1], scaled_pose[dst][1]], lw=1, c="#00FFFF") plt.scatter(scaled_pose[src][0], scaled_pose[src][1], s=10, c="#FF00FF") plt.scatter(scaled_pose[dst][0], scaled_pose[dst][1], s=10, c="#FF00FF") plt.xticks([]) plt.yticks([]) image_number = "{}".format(int(entry["image_path"][0].item())) image_name = "{}.jpg".format(image_number.zfill(9)) image = io.imread("/data/mjakobs/data/mpii/images/{}".format(image_name)) plt.subplot(1, 4, 1) plt.imshow(image) plt.xticks([]) plt.yticks([]) plt.subplot(1, 4, 2) plt.imshow(image) matrix = entry["trans_matrix"] pose = transform_pose(matrix, entry["normalized_pose"][:, 0:2], inverse=True) scaled_pose = pose for i, (src, dst) in enumerate(joint_mapping): plt.plot([scaled_pose[src][0], scaled_pose[dst][0]], [scaled_pose[src][1], scaled_pose[dst][1]], lw=1, c="#00FFFF") plt.scatter(scaled_pose[src][0], scaled_pose[src][1], s=10, c="#FF00FF") plt.scatter(scaled_pose[dst][0], scaled_pose[dst][1], s=10, c="#FF00FF") plt.xticks([]) plt.yticks([]) entry = ds_augmented[20] image = entry["normalized_image"] image = image.permute(1, 2, 0) image = (image + 1) / 2.0 plt.subplot(1, 4, 4) plt.imshow(image) pose = entry["normalized_pose"] scaled_pose = pose * 255.0 for i, (src, dst) in enumerate(joint_mapping): plt.plot([scaled_pose[src][0], scaled_pose[dst][0]], [scaled_pose[src][1], scaled_pose[dst][1]], lw=1, c="#00FFFF") plt.scatter(scaled_pose[src][0], scaled_pose[src][1], s=10, c="#FF00FF") plt.scatter(scaled_pose[dst][0], scaled_pose[dst][1], s=10, c="#FF00FF") plt.xticks([]) plt.yticks([]) plt.show()