def mpjpe_error(outputs, all_seq, input_n, dim_used, dct_n):
"""
:param outputs:
:param all_seq:
:param input_n:
:param dim_used:
:param data_mean:
:param data_std:
:return:
"""
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len,
seq_len).transpose(1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(-1, dim_full_len).clone()
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(-1, dim_full_len)
pred_expmap[:, 0:6] = 0
targ_expmap[:, 0:6] = 0
targ_p3d = data_utils.expmap2xyz_torch(targ_expmap).view(-1, 3)
pred_p3d = data_utils.expmap2xyz_torch(pred_expmap).view(-1, 3)
mean_3d_err = torch.mean(torch.norm(targ_p3d - pred_p3d, 2, 1))
return mean_3d_err
def mpjpe_error(outputs, all_seq, input_n, dim_used, N):
"""
:param outputs:
:param all_seq:
:param input_n:
:param dim_used:
:param data_mean:
:param data_std:
:return:
"""
n, seq_len, dim_full_len = all_seq.data.shape # batch, N, 99
dim_used_len = len(dim_used) # 48
t = np.arange(1, N + 1, 1)
A = chebyshev.chebvander(t, N - 1)
A = Variable(torch.from_numpy(A)).float().cuda()
outputs_t = torch.mm(A, outputs.view(-1, N).transpose(0, 1))
outputs_exp = outputs_t.transpose(0, 1).view(-1, dim_used_len,
seq_len).transpose(1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(
-1, dim_full_len).clone()
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(
-1, dim_full_len)
pred_expmap[:, 0:6] = 0
targ_expmap[:, 0:6] = 0
targ_p3d = data_utils.expmap2xyz_torch(targ_expmap).view(-1, 3)
pred_p3d = data_utils.expmap2xyz_torch(pred_expmap).view(-1, 3)
mean_3d_err = torch.mean(torch.norm(targ_p3d - pred_p3d, 2, 1))
return mean_3d_err
def __init__(self, opt, actions=None, split=0):
"""
:param path_to_data:
:param actions:
:param input_n:
:param output_n:
:param dct_used:
:param split: 0 train, 1 testing, 2 validation
:param sample_rate:
"""
self.path_to_data = "./datasets/h3.6m/"
self.split = split
self.in_n = opt.input_n
self.out_n = opt.output_n
self.sample_rate = 2
self.p3d = {}
self.data_idx = []
seq_len = self.in_n + self.out_n
subs = np.array([[1, 6, 7, 8, 9], [11], [5]])
# acts = data_utils.define_actions(actions)
if actions is None:
acts = [
"walking", "eating", "smoking", "discussion", "directions",
"greeting", "phoning", "posing", "purchases", "sitting",
"sittingdown", "takingphoto", "waiting", "walkingdog",
"walkingtogether"
]
else:
acts = actions
# subs = np.array([[1], [11], [5]])
# acts = ['walking']
# 32 human3.6 joint name:
joint_name = [
"Hips", "RightUpLeg", "RightLeg", "RightFoot", "RightToeBase",
"Site", "LeftUpLeg", "LeftLeg", "LeftFoot", "LeftToeBase", "Site",
"Spine", "Spine1", "Neck", "Head", "Site", "LeftShoulder",
"LeftArm", "LeftForeArm", "LeftHand", "LeftHandThumb", "Site",
"L_Wrist_End", "Site", "RightShoulder", "RightArm", "RightForeArm",
"RightHand", "RightHandThumb", "Site", "R_Wrist_End", "Site"
]
subs = subs[split]
key = 0
for subj in subs:
for action_idx in np.arange(len(acts)):
action = acts[action_idx]
if self.split <= 1:
for subact in [1, 2]: # subactions
print("Reading subject {0}, action {1}, subaction {2}".
format(subj, action, subact))
filename = '{0}/S{1}/{2}_{3}.txt'.format(
self.path_to_data, subj, action, subact)
the_sequence = data_utils.readCSVasFloat(filename)
n, d = the_sequence.shape
even_list = range(0, n, self.sample_rate)
num_frames = len(even_list)
the_sequence = np.array(the_sequence[even_list, :])
the_sequence = torch.from_numpy(
the_sequence).float().cuda()
# remove global rotation and translation
the_sequence[:, 0:6] = 0
p3d = data_utils.expmap2xyz_torch(the_sequence)
# self.p3d[(subj, action, subact)] = p3d.view(num_frames, -1).cpu().data.numpy()
self.p3d[key] = p3d.view(num_frames,
-1).cpu().data.numpy()
valid_frames = np.arange(0, num_frames - seq_len + 1,
opt.skip_rate)
# tmp_data_idx_1 = [(subj, action, subact)] * len(valid_frames)
tmp_data_idx_1 = [key] * len(valid_frames)
tmp_data_idx_2 = list(valid_frames)
self.data_idx.extend(
zip(tmp_data_idx_1, tmp_data_idx_2))
key += 1
else:
print("Reading subject {0}, action {1}, subaction {2}".
format(subj, action, 1))
filename = '{0}/S{1}/{2}_{3}.txt'.format(
self.path_to_data, subj, action, 1)
the_sequence1 = data_utils.readCSVasFloat(filename)
n, d = the_sequence1.shape
even_list = range(0, n, self.sample_rate)
num_frames1 = len(even_list)
the_sequence1 = np.array(the_sequence1[even_list, :])
the_seq1 = torch.from_numpy(the_sequence1).float().cuda()
the_seq1[:, 0:6] = 0
p3d1 = data_utils.expmap2xyz_torch(the_seq1)
# self.p3d[(subj, action, 1)] = p3d1.view(num_frames1, -1).cpu().data.numpy()
self.p3d[key] = p3d1.view(num_frames1,
-1).cpu().data.numpy()
print("Reading subject {0}, action {1}, subaction {2}".
format(subj, action, 2))
filename = '{0}/S{1}/{2}_{3}.txt'.format(
self.path_to_data, subj, action, 2)
the_sequence2 = data_utils.readCSVasFloat(filename)
n, d = the_sequence2.shape
even_list = range(0, n, self.sample_rate)
num_frames2 = len(even_list)
the_sequence2 = np.array(the_sequence2[even_list, :])
the_seq2 = torch.from_numpy(the_sequence2).float().cuda()
the_seq2[:, 0:6] = 0
p3d2 = data_utils.expmap2xyz_torch(the_seq2)
# self.p3d[(subj, action, 2)] = p3d2.view(num_frames2, -1).cpu().data.numpy()
self.p3d[key + 1] = p3d2.view(num_frames2,
-1).cpu().data.numpy()
# print("action:{}".format(action))
# print("subact1:{}".format(num_frames1))
# print("subact2:{}".format(num_frames2))
fs_sel1, fs_sel2 = data_utils.find_indices_256(
num_frames1, num_frames2, seq_len, input_n=self.in_n)
valid_frames = fs_sel1[:, 0]
tmp_data_idx_1 = [key] * len(valid_frames)
tmp_data_idx_2 = list(valid_frames)
self.data_idx.extend(zip(tmp_data_idx_1, tmp_data_idx_2))
valid_frames = fs_sel2[:, 0]
tmp_data_idx_1 = [key + 1] * len(valid_frames)
tmp_data_idx_2 = list(valid_frames)
self.data_idx.extend(zip(tmp_data_idx_1, tmp_data_idx_2))
key += 2
# ignore constant joints and joints at same position with other joints
joint_to_ignore = np.array([0, 1, 6, 11, 16, 20, 23, 24, 28, 31])
dimensions_to_ignore = np.concatenate(
(joint_to_ignore * 3, joint_to_ignore * 3 + 1,
joint_to_ignore * 3 + 2))
self.dimensions_to_use = np.setdiff1d(np.arange(96),
dimensions_to_ignore)
def test(self, train_loader, dataset='h3.6m', input_n=20, output_n=50, dct_n=20, cartesian=False,
dim_used=[]):
N = 0
# t_l = 0
if output_n >= 25:
eval_frame = [1, 3, 7, 9, 13, 24]
elif output_n == 10:
eval_frame = [1, 3, 7, 9]
t_e = np.zeros(len(eval_frame))
t_3d = np.zeros(len(eval_frame))
self.model.eval()
st = time.time()
bar = Bar('>>>', fill='>', max=len(train_loader))
for i, (inputs, targets, all_seq) in enumerate(train_loader):
bt = time.time()
if self.is_cuda:
inputs = Variable(inputs.cuda()).float()
all_seq = Variable(all_seq.cuda(non_blocking=True)).float()
outputs, reconstructions, log_var, z = self.model(inputs.float())
n = outputs.shape[0]
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
all_seq[:, :, 0:6] = 0
# inverse dct transformation
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
if cartesian == False:
outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1,
dim_used_len,
seq_len).transpose(
1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(-1, dim_full_len)
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(-1, dim_full_len)
pred_expmap[:, 0:6] = 0
targ_expmap[:, 0:6] = 0
pred_expmap = pred_expmap.view(-1, 3)
targ_expmap = targ_expmap.view(-1, 3)
# get euler angles from expmap
pred_eul = data_utils.rotmat2euler_torch(data_utils.expmap2rotmat_torch(pred_expmap))
pred_eul = pred_eul.view(-1, dim_full_len).view(-1, output_n, dim_full_len)
targ_eul = data_utils.rotmat2euler_torch(data_utils.expmap2rotmat_torch(targ_expmap))
targ_eul = targ_eul.view(-1, dim_full_len).view(-1, output_n, dim_full_len)
if dataset == 'h3.6m':
# get 3d coordinates
targ_p3d = data_utils.expmap2xyz_torch(targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
pred_p3d = data_utils.expmap2xyz_torch(pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
elif dataset == 'cmu_mocap':
# get 3d coordinates
targ_p3d = data_utils.expmap2xyz_torch_cmu(targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1,
3)
pred_p3d = data_utils.expmap2xyz_torch_cmu(pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1,
3)
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
t_e[k] += torch.mean(torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2, 1)).cpu().data.numpy() * n
t_3d[k] += torch.mean(torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3),
2, 1)).cpu().data.numpy() * n
elif cartesian:
outputs_3d = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1,
dim_used_len,
seq_len).transpose(
1, 2)
pred_3d = all_seq.clone()
dim_used = np.array(dim_used)
# deal with joints at same position
joint_to_ignore = np.array([16, 20, 29, 24, 27, 33, 36])
index_to_ignore = np.concatenate(
(joint_to_ignore * 3, joint_to_ignore * 3 + 1, joint_to_ignore * 3 + 2))
joint_equal = np.array([15, 15, 15, 23, 23, 32, 32])
index_to_equal = np.concatenate((joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2))
pred_3d[:, :, dim_used] = outputs_3d
pred_3d[:, :, index_to_ignore] = pred_3d[:, :, index_to_equal]
pred_p3d = pred_3d.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :]
targ_p3d = all_seq.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :]
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
t_e[k] += torch.mean(torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3),
2, 1)).cpu().data.numpy()[0] * n
t_3d[k] += torch.mean(torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3),
2, 1)).cpu().data.numpy()[0] * n
N += n
bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(i + 1, len(train_loader), time.time() - bt,
time.time() - st)
bar.next()
bar.finish()
return t_e / N, t_3d / N
def test(train_loader,
model,
stepsize,
input_n=20,
output_n=50,
dct_n=20,
is_cuda=False,
dim_used=[]):
N = 0
# t_l = 0
if output_n == 25:
eval_frame = [1, 3, 7, 9, 13, 24]
elif output_n == 50:
eval_frame = [1, 3, 7, 9, 13, 24, 35]
elif output_n == 100:
eval_frame = [1, 3, 7, 9, 13, 24, 35, 49]
elif output_n == 10:
eval_frame = [1, 3, 7, 9]
t_e = np.zeros(len(eval_frame))
t_3d = np.zeros(len(eval_frame))
model.eval()
# calculate no of iterations in auto regression to perform
iterations = int(output_n / stepsize)
print('iterations: {}'.format(iterations))
st = time.time()
bar = Bar('>>>', fill='>', max=len(train_loader))
for i, (inputs, targets, all_seq) in enumerate(train_loader):
bt = time.time()
all_seq = Variable(all_seq).float()
dim_used_len = len(dim_used)
if is_cuda:
all_seq = all_seq.cuda()
dct_m_in, _ = data_utils.get_dct_matrix(dct_n)
dct_m_in = Variable(torch.from_numpy(dct_m_in)).float().cuda()
_, idct_m = data_utils.get_dct_matrix(dct_n)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
n, seq_len, dim_full_len = all_seq.data.shape
y_hat = None
# Auto regression
for idx in range(iterations):
# start index of the input sequence
start = input_n + idx * stepsize
# end index of the input sequence
stop = start + stepsize
if y_hat is None:
# slice the sequence of length = (input_n + output_n) in iteration 1
input_seq = all_seq[:, :dct_n, dim_used]
else:
# stack output from prev iteration and next frames to form the next input seq
input_seq = torch.cat(
(y_hat, all_seq[:, start:stop, dim_used]), 1)
# calculate DCT of the input seq
input_dct_seq = torch.matmul(dct_m_in, input_seq).transpose(1, 2)
if is_cuda:
input_dct_seq = input_dct_seq.cuda()
y = model(input_dct_seq)
y_t = y.view(-1, dct_n).transpose(0, 1)
y_exp = torch.matmul(idct_m,
y_t).transpose(0, 1).contiguous().view(
-1, dim_used_len, dct_n).transpose(1, 2)
y_hat = y_exp[:, stepsize:, :]
# accumulate the output frames in a single tensor
if idx == 0:
outputs = y_exp
else:
outputs = torch.cat((outputs, y_exp[:, input_n:, :]), 1)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs
pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(
-1, dim_full_len)
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(
-1, dim_full_len)
pred_expmap[:, 0:6] = 0
targ_expmap[:, 0:6] = 0
pred_expmap = pred_expmap.view(-1, 3)
targ_expmap = targ_expmap.view(-1, 3)
# get euler angles from expmap
pred_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(pred_expmap))
pred_eul = pred_eul.view(-1,
dim_full_len).view(-1, output_n, dim_full_len)
targ_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(targ_expmap))
targ_eul = targ_eul.view(-1,
dim_full_len).view(-1, output_n, dim_full_len)
# get 3d coordinates
targ_p3d = data_utils.expmap2xyz_torch(
targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
pred_p3d = data_utils.expmap2xyz_torch(
pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
# update loss and testing errors
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
'''
t_e[k] += torch.mean(torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2, 1)).cpu().data.numpy()[0] * n
t_3d[k] += torch.mean(torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2,
1)).cpu().data.numpy()[0] * n
'''
t_e[k] += torch.mean(
torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2,
1)).item() * n
t_3d[k] += torch.mean(
torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) -
pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2,
1)).item() * n
# t_l += loss.cpu().data.numpy()[0] * n
N += n
bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(
i + 1, len(train_loader),
time.time() - bt,
time.time() - st)
bar.next()
bar.finish()
return t_e / N, t_3d / N
def test(train_loader,
model,
input_n=20,
output_n=50,
is_cuda=False,
dim_used=[]):
N = 0
# t_l = 0
if output_n >= 25:
eval_frame = [1, 3, 7, 9, 13, 24]
elif output_n == 10:
eval_frame = [1, 3, 7, 9]
else:
eval_frame = [1, 2, 3, 4]
t_e = np.zeros(len(eval_frame))
t_3d = np.zeros(len(eval_frame))
model.eval()
st = time.time()
bar = Bar('>>>', fill='>', max=len(train_loader))
for i, (inputs, targets, all_seq) in enumerate(train_loader):
bt = time.time()
if is_cuda:
inputs = Variable(inputs.cuda()).float()
# targets = Variable(targets.cuda(async=True)).float()
all_seq = Variable(all_seq.cuda()).float()
outputs = model(inputs)
# inverse dct transformation
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
N = input_n + output_n
t = np.arange(1, N + 1, 1)
A = chebyshev.chebvander(t, N - 1)
A = Variable(torch.from_numpy(A)).float().cuda()
outputs_t = torch.matmul(A, outputs.view(-1, N).transpose(0, 1))
outputs_expmap = outputs_t.transpose(0,
1).view(-1, dim_used_len,
seq_len).transpose(1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_expmap
pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(
-1, dim_full_len)
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(
-1, dim_full_len)
pred_expmap[:, 0:6] = 0
targ_expmap[:, 0:6] = 0
pred_expmap = pred_expmap.view(-1, 3)
targ_expmap = targ_expmap.view(-1, 3)
# get euler angles from expmap
pred_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(pred_expmap))
pred_eul = pred_eul.view(-1,
dim_full_len).view(-1, output_n, dim_full_len)
targ_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(targ_expmap))
targ_eul = targ_eul.view(-1,
dim_full_len).view(-1, output_n, dim_full_len)
# get 3d coordinates
targ_p3d = data_utils.expmap2xyz_torch(
targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
pred_p3d = data_utils.expmap2xyz_torch(
pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
# update loss and testing errors
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
t_e[k] += torch.mean(
torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2,
1)).cpu().data.numpy() * n
t_3d[k] += torch.mean(
torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) -
pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2,
1)).cpu().data.numpy() * n
# t_l += loss.cpu().data.numpy()[0] * n
N += n
bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(
i + 1, len(train_loader),
time.time() - bt,
time.time() - st)
bar.next()
bar.finish()
return t_e / N, t_3d / N