def preprocess(self, frame, pose):
trans_matrix, image = rotate_and_crop(frame, self.aug_conf["angle"],
self.center, self.window_size)
size_after_rotate = torch.FloatTensor([image.shape[1], image.shape[0]])
image = resize(image, (self.final_size, self.final_size),
preserve_range=True)
trans_matrix = scale(trans_matrix,
self.final_size / size_after_rotate[0],
self.final_size / size_after_rotate[1])
# randomly flip horizontal
if self.aug_conf["flip"]:
image = np.fliplr(image)
trans_matrix = translate(trans_matrix, -image.shape[1] / 2,
-image.shape[0] / 2)
trans_matrix = flip_h(trans_matrix)
trans_matrix = translate(trans_matrix, image.shape[1] / 2,
image.shape[0] / 2)
# normalize pose to [0,1] and image to [-1, 1]
normalized_image = normalize_channels(image)
trans_matrix = scale(trans_matrix, 1.0 / self.final_size,
1.0 / self.final_size)
transformed_pose = transform_pose(trans_matrix, pose)
trans_matrix = torch.from_numpy(trans_matrix).float()
normalized_image = torch.from_numpy(normalized_image).float()
transformed_pose = torch.from_numpy(transformed_pose).float()
assert normalized_image.max() <= 1 and normalized_image.min() >= -1
return trans_matrix, normalized_image, transformed_pose
def eval_pckh_batch(predictions, poses, headsizes, matrices, return_perjoint=False):
scores_05 = []
scores_02 = []
matches_05 = []
matches_02 = []
valids_02 = []
valids_05 = []
for i, prediction in enumerate(predictions):
pred_pose = prediction[:, 0:2]
ground_pose = poses[i, :, 0:2]
# transform coordinates back to original
pred_pose = transform_pose(matrices[i], pred_pose, inverse=True)
ground_pose = transform_pose(matrices[i], ground_pose, inverse=True)
match, valid, scores = pckh(ground_pose, pred_pose, headsizes[i])
if torch.cuda.is_available():
matches_05.append(match.cpu().numpy())
valids_05.append(valid.cpu().numpy())
else:
matches_05.append(match.numpy())
valids_05.append(valid.numpy())
scores_05.append(scores)
match, scores = pckh(ground_pose, pred_pose, headsizes[i], distance_threshold=0.2)
if torch.cuda.is_available():
matches_02.append(match.cpu().numpy())
valids_02.append(valid.cpu().numpy())
else:
matches_02.append(match.numpy())
valids_02.append(valid.numpy())
scores_02.append(scores)
if return_perjoint:
return matches_05, matches_02, valids_05, valids_02
else:
return scores_05, scores_02
def eval_pck_batch(predictions, poses, matrices, distance_meassures, threshold=0.2, return_perjoint=False):
scores_02 = []
matches = []
valids = []
for i, prediction in enumerate(predictions):
# transform coordinates back to original
pred_pose = torch.from_numpy(transform_pose(matrices[i], predictions[i], inverse=True))
ground_pose = torch.from_numpy(transform_pose(matrices[i], poses[i], inverse=True))
match, valid, scores = pck_bounding_box(ground_pose, pred_pose, distance_meassures[i], distance_threshold=threshold)
scores_02.append(scores)
if torch.cuda.is_available():
matches.append(match.cpu().numpy())
valids.append(valid.cpu().numpy())
else:
valids.append(valid.numpy())
matches.append(match.numpy())
if return_perjoint:
return matches, valids
else:
return scores_02
def plot(pose, matrix, path):
# plt.tick_params(
# axis='both', # changes apply to the x-axis
# which='both', # both major and minor ticks are affected
# bottom=False, # ticks along the bottom edge are off
# right=False,
# left=False,
# top=False, # ticks along the top edge are off
# labelbottom=False) # labels along the bottom edge are off
plt.axis("off")
vis = pose[:, 2] > 0.5
pose = pose[:, 0:2]
pose = torch.from_numpy(transform_pose(matrix, pose, inverse=True)).float()
pred_x = pose[:, 0]
pred_y = pose[:, 1]
s = 10
for _, (src, dst) in enumerate(joint_mapping):
if not vis[src]:
continue
if vis[dst]:
plt.plot([pred_x[src], pred_x[dst]], [pred_y[src], pred_y[dst]], lw=1, c="#00FF00")
plt.scatter(x=pred_x[0:3], y=pred_y[0:3], c="#FF00FF", s=s) # right
plt.scatter(x=pred_x[10:13], y=pred_y[10:13], c="#FF00FF", s=s) # right
plt.scatter(x=pred_x[3:6], y=pred_y[3:6], c="#FFFF00", s=s) # left
plt.scatter(x=pred_x[13:16], y=pred_y[13:16], c="#FFFF00", s=s) # left
plt.scatter(x=pred_x[6:10], y=pred_y[6:10], c="#00FFFF", s=s) # rest
plt.tight_layout()
plt.savefig(path)
plt.close()
def show_predictions_ontop(ground_truth,
image,
poses,
path,
matrix,
bbox=None,
save=True):
plt.xticks([])
plt.yticks([])
ground_pose = ground_truth[:, 0:2]
predicted_pose = poses[:, 0:2]
plt.imshow(image)
orig_gt_coordinates = transform_pose(matrix, ground_pose, inverse=True)
orig_pred_coordinates = transform_pose(matrix,
predicted_pose,
inverse=True)
gt_color = "r"
pred_color = "b"
bbox_color = "#ff00ff"
for i, (src, dst) in enumerate(joint_mapping):
if not ground_truth[src, 2]:
continue
if ground_truth[dst, 2]:
#print("{} => {}".format(mpii_joint_order[src], mpii_joint_order[dst]))
plt.plot(
[orig_gt_coordinates[src][0], orig_gt_coordinates[dst][0]],
[orig_gt_coordinates[src][1], orig_gt_coordinates[dst][1]],
lw=1,
c=gt_color)
plt.plot(
[orig_pred_coordinates[src][0], orig_pred_coordinates[dst][0]],
[orig_pred_coordinates[src][1], orig_pred_coordinates[dst][1]],
lw=1,
c=pred_color)
plt.scatter(orig_gt_coordinates[src][0],
orig_gt_coordinates[src][1],
label="{}".format(mpii_joint_order[src]),
c=joint_colors[src])
plt.scatter(orig_pred_coordinates[src][0],
orig_pred_coordinates[src][1],
c=joint_colors[src])
plt.scatter(orig_gt_coordinates[dst][0],
orig_gt_coordinates[dst][1],
label="{}".format(mpii_joint_order[dst]),
c=joint_colors[dst])
plt.scatter(orig_pred_coordinates[dst][0],
orig_pred_coordinates[dst][1],
c=joint_colors[dst])
else:
#print("{}".format(mpii_joint_order[src]))
plt.scatter(orig_gt_coordinates[src][0],
orig_gt_coordinates[src][1],
label="{}".format(mpii_joint_order[src]),
c=joint_colors[src])
plt.scatter(orig_pred_coordinates[src][0],
orig_pred_coordinates[src][1],
c=joint_colors[src])
if bbox is not None:
bbox_rect_original = Rectangle((bbox[0], bbox[1]),
abs(bbox[0] - bbox[2]),
abs(bbox[1] - bbox[3]),
linewidth=1,
facecolor='none',
edgecolor=bbox_color)
ax = plt.gca()
ax.add_patch(bbox_rect_original)
# remove duplicate handles in legend (src: https://stackoverflow.com/questions/13588920/stop-matplotlib-repeating-labels-in-legend)
handles, labels = plt.gca().get_legend_handles_labels()
by_label = OrderedDict(zip(labels, handles))
plt.legend(by_label.values(), by_label.keys())
if save:
if os.path.isfile(path):
os.remove(path)
plt.savefig(path)
else:
plt.show()
plt.close()
plt.subplot(122)
matrix = entry["trans_matrices"][frame]
index = int(entry["index"].item())
item_path = "/data/mjakobs/data/pennaction/frames/" + ds.items[
ds.indices[idx]]
all_frames = sorted(glob.glob(item_path + "/*.jpg"))
image_path = all_frames[frame]
image = io.imread(image_path)
plt.imshow(image)
pose = torch.from_numpy(
transform_pose(matrix, pose[:, 0:2], inverse=True)).float()
x = (pose[:, 0] * vis).numpy()
y = (pose[:, 1] * vis).numpy()
x[x == 0] = None
y[y == 0] = None
plt.scatter(x=x, y=y, c="#FF00FF", s=10)
for i, (src, dst) in enumerate(joint_mapping):
if not vis[src]:
continue
if vis[dst]:
plt.plot([x[src], x[dst]], [y[src], y[dst]],
lw=1,
if vis[dst]:
plt.plot([x[src], x[dst]], [y[src], y[dst]], lw=1, c="#00FFFF")
plt.subplot(122)
matrix = entry["trans_matrices"][frame]
indices = entry["indices"].int()
item_path = ds_full.items[indices[2]]
all_frames = sorted(glob.glob(item_path + "/*.png"))
image_path = all_frames[indices[0] + frame]
image = io.imread(image_path)
plt.imshow(image)
pose = torch.from_numpy(transform_pose(matrix, pose[:, 0:2], inverse=True)).float()
x = (pose[:, 0] * vis).numpy()
y = (pose[:, 1] * vis).numpy()
x[x == 0] = None
y[y == 0] = None
plt.scatter(x=x, y=y, c="#FF00FF", s=10)
for i, (src, dst) in enumerate(joint_mapping):
if not vis[src]:
continue
if vis[dst]:
plt.plot([x[src], x[dst]], [y[src], y[dst]], lw=1, c="#00FFFF")
def plot_mpii_augmented():
ds = MPIIDataset("/data/mjakobs/data/mpii/",
train=True,
val=False,
use_random_parameters=False,
use_saved_tensors=False)
ds_augmented = MPIIDataset("/data/mjakobs/data/mpii/",
train=True,
val=False,
use_random_parameters=True,
use_saved_tensors=False)
ds_augmented.angles = np.array([-15])
ds_augmented.scales = np.array([1.3])
ds_augmented.flip_horizontal = np.array([1])
entry = ds[20]
image = entry["normalized_image"]
image = image.permute(1, 2, 0)
image = (image + 1) / 2.0
plt.subplot(1, 4, 3)
plt.imshow(image)
pose = entry["normalized_pose"]
scaled_pose = pose * 255.0
for i, (src, dst) in enumerate(joint_mapping):
plt.plot([scaled_pose[src][0], scaled_pose[dst][0]],
[scaled_pose[src][1], scaled_pose[dst][1]],
lw=1,
c="#00FFFF")
plt.scatter(scaled_pose[src][0],
scaled_pose[src][1],
s=10,
c="#FF00FF")
plt.scatter(scaled_pose[dst][0],
scaled_pose[dst][1],
s=10,
c="#FF00FF")
plt.xticks([])
plt.yticks([])
image_number = "{}".format(int(entry["image_path"][0].item()))
image_name = "{}.jpg".format(image_number.zfill(9))
image = io.imread("/data/mjakobs/data/mpii/images/{}".format(image_name))
plt.subplot(1, 4, 1)
plt.imshow(image)
plt.xticks([])
plt.yticks([])
plt.subplot(1, 4, 2)
plt.imshow(image)
matrix = entry["trans_matrix"]
pose = transform_pose(matrix,
entry["normalized_pose"][:, 0:2],
inverse=True)
scaled_pose = pose
for i, (src, dst) in enumerate(joint_mapping):
plt.plot([scaled_pose[src][0], scaled_pose[dst][0]],
[scaled_pose[src][1], scaled_pose[dst][1]],
lw=1,
c="#00FFFF")
plt.scatter(scaled_pose[src][0],
scaled_pose[src][1],
s=10,
c="#FF00FF")
plt.scatter(scaled_pose[dst][0],
scaled_pose[dst][1],
s=10,
c="#FF00FF")
plt.xticks([])
plt.yticks([])
entry = ds_augmented[20]
image = entry["normalized_image"]
image = image.permute(1, 2, 0)
image = (image + 1) / 2.0
plt.subplot(1, 4, 4)
plt.imshow(image)
pose = entry["normalized_pose"]
scaled_pose = pose * 255.0
for i, (src, dst) in enumerate(joint_mapping):
plt.plot([scaled_pose[src][0], scaled_pose[dst][0]],
[scaled_pose[src][1], scaled_pose[dst][1]],
lw=1,
c="#00FFFF")
plt.scatter(scaled_pose[src][0],
scaled_pose[src][1],
s=10,
c="#FF00FF")
plt.scatter(scaled_pose[dst][0],
scaled_pose[dst][1],
s=10,
c="#FF00FF")
plt.xticks([])
plt.yticks([])
plt.show()