def extract_har_jhmdb_pose(model_path, output_path):
ds = JHMDBDataset("/data/mjakobs/data/jhmdb/", train=False)
model = DeepHar(num_actions=21, use_gt=True, nr_context=2, model_path="/data/mjakobs/data/pretrained_mixed_pose", use_timedistributed=True).to("cpu")
model.load_state_dict(torch.load(model_path, map_location="cpu"))
model.eval()
for i in range(len(ds)):
entry = ds[i]
sequence_length = entry["sequence_length"][0].item()
frames = entry["normalized_frames"][:sequence_length]
matrices = entry["trans_matrices"][:sequence_length]
all_frames = entry["all_frames"]
frames = frames.unsqueeze(0)
_, predicted_poses, _, _, _ = model(frames, gt_pose=None)
for frame in range(sequence_length):
path = '{}/{}_{}.png'.format(output_path, i, frame)
image = io.imread(all_frames[frame])
plt.imshow(image)
pose = predicted_poses.squeeze(0)
pose = pose[frame]
plot(pose, matrices[frame], path)
def extract_jhmdb_pose(model_path, output_path):
ds = JHMDBDataset("/data/mjakobs/data/jhmdb/", train=False)
model = Mpii_8(num_context=2)
model.load_state_dict(torch.load(model_path, map_location="cpu"))
model.eval()
for i in [100]:
entry = ds[i]
sequence_length = entry["sequence_length"][0].item()
frames = entry["normalized_frames"][:sequence_length]
matrices = entry["trans_matrices"][:sequence_length]
all_frames = entry["all_frames"]
_, predicted_poses, _, _ = model(frames)
for frame in range(sequence_length):
path = '{}/{}_{}.png'.format(output_path, i, frame)
image = io.imread(all_frames[frame])
plt.imshow(image)
pose = predicted_poses.squeeze(0)
pose = pose[frame]
plot(pose, matrices[frame], path)
def create_fragments_jhmdb(train=False,
val=False,
split=1,
use_random=False,
subprefix="1"):
ds = JHMDBDataset("/data/mjakobs/data/jhmdb/",
use_random_parameters=use_random,
use_saved_tensors=False,
train=train,
val=val,
split=split)
ds_bbox_gt = JHMDBDataset("/data/mjakobs/data/jhmdb/",
use_random_parameters=use_random,
use_saved_tensors=False,
train=train,
val=val,
split=split,
use_gt_bb=True)
if use_random:
assert train
print("-" * 50)
print("Train: {}, Val: {}, Split: {}, Random: {}".format(
train, val, split, use_random))
print("-" * 50)
length = len(ds)
current = 0
root_dir = "/data/mjakobs/data/jhmdb_fragments/"
if use_random:
prefix = "rand{}_".format(subprefix)
else:
prefix = ""
all_indices = list(range(len(ds)))
if train:
if val:
train_test_folder = "val"
else:
train_test_folder = "train"
else:
train_test_folder = "test"
for counter, idx in enumerate(all_indices):
entry = ds[idx]
frames = entry["normalized_frames"]
poses = entry["normalized_poses"]
actions = entry["action_1h"]
sequence_length = entry["sequence_length"]
matrices = entry["trans_matrices"]
bbox = entry["bbox"]
index = entry["index"]
parameters = entry["parameters"]
original_window_size = entry["original_window_size"]
frames = ((frames + 1) / 2.0) * 255.0
frames = frames.byte()
poses[:, :, 0:2] = poses[:, :, 0:2] * 255.0
poses = poses.int()
assert len(frames) == 40
assert len(poses) == 40
num_frames = 16
num_frames_total = sequence_length[0]
if num_frames_total < 16:
print("less than 16 frames")
continue
original_image = ds.indices[idx]
padded_original_image = str(original_image).zfill(8)
torch.save(
frames, root_dir + prefix + "images/" + padded_original_image +
".frames.pt")
torch.save(
actions, root_dir + prefix + "annotations/" +
padded_original_image + ".action_1h.pt")
torch.save(
poses, root_dir + prefix + "annotations/" + padded_original_image +
".poses.pt")
torch.save(
matrices, root_dir + prefix + "annotations/" +
padded_original_image + ".matrices.pt")
torch.save(
index, root_dir + prefix + "annotations/" + padded_original_image +
".index.pt")
torch.save(
bbox, root_dir + prefix + "annotations/" + padded_original_image +
".bbox.pt")
torch.save(
parameters, root_dir + prefix + "annotations/" +
padded_original_image + ".parameters.pt")
torch.save(
original_window_size, root_dir + prefix + "annotations/" +
padded_original_image + ".original_window_size.pt")
entry = ds_bbox_gt[idx]
frames_gt_bb = entry["normalized_frames"]
poses_gt_bb = entry["normalized_poses"]
matrices_gt_bb = entry["trans_matrices"]
bbox_gt_bb = entry["bbox"]
frames_gt_bb = ((frames_gt_bb + 1) / 2.0) * 255.0
frames_gt_bb = frames_gt_bb.byte()
poses_gt_bb[:, :, 0:2] = poses_gt_bb[:, :, 0:2] * 255.0
poses_gt_bb = poses_gt_bb.int()
torch.save(
frames_gt_bb, root_dir + prefix + "images/" +
padded_original_image + ".frames_gt_bb.pt")
torch.save(
poses_gt_bb, root_dir + prefix + "annotations/" +
padded_original_image + ".poses_gt_bb.pt")
torch.save(
matrices_gt_bb, root_dir + prefix + "annotations/" +
padded_original_image + ".matrices_gt_bb.pt")
torch.save(
bbox_gt_bb, root_dir + prefix + "annotations/" +
padded_original_image + ".bbox_gt_bb.pt")
indices = torch.zeros((num_frames_total - num_frames), 2)
for i in range(num_frames_total - num_frames):
start = i
end = min(i + num_frames, num_frames_total)
padded = str(current).zfill(8)
current = current + 1
indices = torch.zeros(3)
indices[0] = start
indices[1] = end
indices[2] = original_image
torch.save(
indices,
root_dir + prefix + "indices/{}/{}/{}.indices.pt".format(
train_test_folder, str(split), padded))
assert indices.shape == (3, )
print("{} - {}: {} / {}".format(train_test_folder, split, counter + 1,
len(all_indices)))
def check_e2e(model_path, confusion_classes):
model = DeepHar(num_actions=21,
use_gt=False,
nr_context=2,
use_timedistributed=True).to(device)
with torch.no_grad():
test_ds = JHMDBDataset("/data/mjakobs/data/jhmdb/",
train=False,
use_gt_bb=True)
model.load_state_dict(torch.load(model_path, map_location=device))
model.eval()
nr_confusions = 0
for i in range(190, len(test_ds)):
print(i)
test_objects = test_ds[i]
frames = test_objects["normalized_frames"].to(device)
actions = test_objects["action_1h"].to(device)
ground_poses = test_objects["normalized_poses"].to(device)
sequence_length = test_objects["sequence_length"].to(device).item()
padding = int((sequence_length - 16) / 2.0)
if sequence_length < 16:
print("length smaller than 16")
continue
ground_class = torch.argmax(actions, 0)
single_clip = frames[padding:(16 + padding)]
ground_vis = ground_poses[padding:(16 + padding)]
ground_vis = ground_vis[:, :, 2].cpu()
bboxes = test_objects["bbox"][padding:(16 + padding)]
trans_matrices = test_objects["trans_matrices"][padding:(16 +
padding)]
ground_poses = ground_poses[padding:(16 + padding)]
assert len(single_clip) == 16
single_clip = single_clip.unsqueeze(0).to(device)
_, predicted_poses, _, _, single_result = model(single_clip)
pred_class_single = torch.argmax(single_result.squeeze(1))
predicted_poses = predicted_poses.squeeze(0).cpu()
print(ground_class.item(), confusion_classes[1],
pred_class_single.item(), confusion_classes[0])
if ground_class.item() == confusion_classes[
1] and pred_class_single.item() == confusion_classes[0]:
print("confusion detected")
single_clip = single_clip.squeeze(0)
images = single_clip[::4].cpu()
assert len(images) == 4
poses = predicted_poses[::4].cpu()
for i in range(4):
plt.subplot(2, 2, i + 1)
plt.imshow((images[i].permute(1, 2, 0) + 1) / 2.0)
plt.title("Frame {}".format(i * 4))
#plt.scatter(x=predicted_poses[i, :, 0] * 255.0 * ground_vis[i], y=predicted_poses[i, :, 1] * 255.0 * ground_vis[i], s=10, c="#00FFFF")
plt.axis("off")
plt.tight_layout()
plt.savefig("confusion_e2e/said_{}_was_{}_{}.png".format(
jhmdb_actions[confusion_classes[0]],
jhmdb_actions[confusion_classes[1]], nr_confusions))
plt.close()
nr_confusions += 1
import torch
import numpy as np
from datasets.JHMDBDataset import JHMDBDataset
from deephar.models import Mpii_4
from deephar.evaluation import eval_pck_batch
pretrained_model = "/data/mjakobs/data/pose_jhmdb_perjoint"
if torch.cuda.is_available():
device = "cuda"
else:
device = "cpu"
with torch.no_grad():
test_ds = JHMDBDataset("/data/mjakobs/data/jhmdb/",
train=False,
use_gt_bb=True)
model = Mpii_4(num_context=0).to(device)
model.load_state_dict(torch.load(pretrained_model, map_location=device))
model.eval()
per_joint_accuracy = np.zeros(16)
number_valids = np.zeros(16)
for i in range(len(test_ds)):
print(i)
test_objects = test_ds[i]
frames = test_objects["normalized_frames"].to(device)
ground_poses = test_objects["normalized_poses"].to(device)
sequence_length = test_objects["sequence_length"].to(device).item()
scenarios = [[True, False, False, False], [True, False, True, False],
[False, False, False, False],
[True, True, False, False]] # train, val, random, saved
for path in subfolders:
os.makedirs(output_folder + "/" + path)
for scenario_idx, scenario in enumerate(scenarios):
train = scenario[0]
val = scenario[1]
use_random_parameters = scenario[2]
use_saved_tensors = scenario[3]
ds = JHMDBDataset("/data/mjakobs/data/jhmdb/",
train=train,
use_random_parameters=use_random_parameters,
val=val,
use_saved_tensors=False)
random.seed(1)
np.random.seed(1)
torch.manual_seed(1)
all_indices = list(range(len(ds)))
random.shuffle(all_indices)
test_indices = all_indices[:10]
for idx in test_indices:
entry = ds[idx]
subfolders = ["train_norandom_saved", "train_random_saved", "test_norandom_saved", "val_norandom_saved"]
scenarios = [[True, False, False, True], [True, False, True, True], [False, False, False, True], [True, True, False, True]] # train, val, random, saved
for path in subfolders:
os.makedirs(output_folder + "/" + path)
for scenario_idx, scenario in enumerate(scenarios):
train = scenario[0]
val = scenario[1]
use_random_parameters = scenario[2]
#use_saved_tensors = scenario[3]
ds = JHMDBFragmentsDataset("/data/mjakobs/data/jhmdb_fragments/", train=train, val=val, use_random_parameters=use_random_parameters)
ds_full = JHMDBDataset("/data/mjakobs/data/jhmdb/", train=train, val=val)
random.seed(1)
np.random.seed(1)
torch.manual_seed(1)
all_indices = list(range(len(ds)))
random.shuffle(all_indices)
test_indices = all_indices[:10]
for idx in test_indices:
entry = ds[idx]
for frame in range(16):
from datasets.JHMDBDataset import JHMDBDataset
ds_train = JHMDBDataset("/data/mjakobs/data/jhmdb/",
train=True,
val=False,
use_random_parameters=False)
ds_val = JHMDBDataset("/data/mjakobs/data/jhmdb/",
train=True,
val=True,
use_random_parameters=False)
print("train length", len(ds_train))
print("val length", len(ds_val))
train_classes = {}
val_classes = {}
for i in range(len(ds_train)):
entry = ds_train[i]
action = entry["action_label"]
if not action in train_classes:
train_classes[action] = 1.0 / len(ds_train)
else:
train_classes[action] = train_classes[action] + 1.0 / len(ds_train)
for i in range(len(ds_val)):
entry = ds_val[i]
action = entry["action_label"]
if not action in val_classes:
val_classes[action] = 1.0 / len(ds_val)
else: