def __init__(self,
path_to_data,
actions,
input_n=10,
output_n=10,
split=0,
data_mean=0,
data_std=0,
dim_used=0,
dct_n=15):
self.path_to_data = path_to_data
self.split = split
actions = data_utils.define_actions_cmu(actions)
# actions = ['walking']
if split == 0:
path_to_data = path_to_data + '/train/'
is_test = False
else:
path_to_data = path_to_data + '/test/'
is_test = True
all_seqs, dim_ignore, dim_use, data_mean, data_std = data_utils.load_data_cmu_3d(
path_to_data,
actions,
input_n,
output_n,
data_std=data_std,
data_mean=data_mean,
is_test=is_test)
if not is_test:
dim_used = dim_use
self.all_seqs = all_seqs
self.dim_used = dim_used
all_seqs = all_seqs[:, :, dim_used]
all_seqs = all_seqs.transpose(0, 2, 1)
all_seqs = all_seqs.reshape(-1, input_n + output_n)
all_seqs = all_seqs.transpose()
dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n)
dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n)
pad_idx = np.repeat([input_n - 1], output_n)
i_idx = np.append(np.arange(0, input_n), pad_idx)
input_dct_seq = np.matmul(dct_m_in[0:dct_n, :], all_seqs[i_idx, :])
input_dct_seq = input_dct_seq.transpose().reshape(
-1, len(dim_used), dct_n)
# input_dct_seq = input_dct_seq.reshape(-1, len(dim_used) * (input_n + output_n))
output_dct_seq = np.matmul(dct_m_out[0:dct_n, :], all_seqs)
output_dct_seq = output_dct_seq.transpose().reshape(
-1, len(dim_used), dct_n)
self.input_dct_seq = input_dct_seq
self.output_dct_seq = output_dct_seq
self.data_mean = data_mean
self.data_std = data_std
def __init__(self,
path_to_data,
actions,
input_n=20,
output_n=10,
dct_used=15,
split=0,
sample_rate=2):
"""
:param path_to_data:
:param actions:
:param input_n:
:param output_n:
:param dct_used:
:param split: 0 train, 1 testing, 2 validation
:param sample_rate:
"""
self.path_to_data = path_to_data
self.split = split
self.dct_used = dct_used
subs = np.array([[1, 6, 7, 8, 9], [5], [11]])
acts = data_utils.define_actions(actions)
# subs = np.array([[1], [5], [11]])
# acts = ['walking']
subjs = subs[split]
all_seqs, dim_ignore, dim_used = data_utils.load_data_3d(
path_to_data, subjs, acts, sample_rate, input_n + output_n)
self.all_seqs = all_seqs
self.dim_used = dim_used
all_seqs = all_seqs[:, :, dim_used]
all_seqs = all_seqs.transpose(0, 2, 1)
all_seqs = all_seqs.reshape(-1, input_n + output_n)
all_seqs = all_seqs.transpose()
dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n)
dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n)
pad_idx = np.repeat([input_n - 1], output_n)
i_idx = np.append(np.arange(0, input_n), pad_idx)
input_dct_seq = np.matmul(dct_m_in[0:dct_used, :], all_seqs[i_idx, :])
input_dct_seq = input_dct_seq.transpose().reshape(
[-1, len(dim_used), dct_used])
# input_dct_seq = input_dct_seq.reshape(-1, len(dim_used) * dct_used)
output_dct_seq = np.matmul(dct_m_out[0:dct_used, :], all_seqs)
output_dct_seq = output_dct_seq.transpose().reshape(
[-1, len(dim_used), dct_used])
# output_dct_seq = output_dct_seq.reshape(-1, len(dim_used) * dct_used)
self.input_dct_seq = input_dct_seq
self.output_dct_seq = output_dct_seq
def mpjpe_error_p3d(outputs, all_seq, dct_n, dim_used):
"""
:param outputs:n*66*dct_n
:param all_seq:
:param dct_n:
:param dim_used:
:return:
"""
n, seq_len, dim_full_len = all_seq.data.shape
dim_used = np.array(dim_used)
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_p3d = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len,
seq_len).transpose(1,
2)
pred_3d = outputs_p3d.contiguous().view(-1, dim_used_len).view(-1, 3)
targ_3d = all_seq[:, :, dim_used].contiguous().view(-1, dim_used_len).view(-1, 3)
mean_3d_err = torch.mean(torch.norm(pred_3d - targ_3d, 2, 1))
return mean_3d_err
def mpjpe_error(outputs, all_seq, input_n, dim_used, dct_n):
"""
:param outputs:
:param all_seq:
:param input_n:
:param dim_used:
:param data_mean:
:param data_std:
:return:
"""
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len,
seq_len).transpose(1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(-1, dim_full_len).clone()
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(-1, dim_full_len)
pred_expmap[:, 0:6] = 0
targ_expmap[:, 0:6] = 0
targ_p3d = data_utils.expmap2xyz_torch(targ_expmap).view(-1, 3)
pred_p3d = data_utils.expmap2xyz_torch(pred_expmap).view(-1, 3)
mean_3d_err = torch.mean(torch.norm(targ_p3d - pred_p3d, 2, 1))
return mean_3d_err
def test(train_loader,
model,
input_n=20,
dct_n=20,
dim_used=[],
output_n=50,
is_cuda=False):
N = 0
if output_n == 15:
eval_frame = [2, 5, 8, 11, 14]
elif output_n == 30:
eval_frame = [2, 5, 8, 11, 14, 17, 20, 23, 26, 29]
t_err = np.zeros(len(eval_frame))
model.eval()
st = time.time()
bar = Bar('>>>', fill='>', max=len(train_loader))
for i, (inputs, targets, all_seq) in enumerate(train_loader):
bt = time.time()
if is_cuda:
inputs = Variable(inputs.cuda()).float()
# targets = Variable(targets.cuda(async=True)).float()
all_seq = Variable(all_seq.cuda(async=True)).float()
else:
inputs = Variable(inputs).float()
# targets = Variable(targets).float()
all_seq = Variable(all_seq).float()
outputs = model(inputs)
n, seq_len, dim_full_len = all_seq.data.shape
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(
0, 1).contiguous().view(-1, dim_full_len - 3,
seq_len).transpose(1, 2)
pred_exp = all_seq.clone()
pred_exp[:, :, dim_used] = outputs_exp
pred_exp = pred_exp.contiguous().view(n, seq_len, -1)[:, input_n:, :]
targ_exp = all_seq.contiguous().view(n, seq_len, -1)[:, input_n:, :]
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
t_err[k] += torch.mean(
torch.norm(targ_exp[:, j, :] - pred_exp[:, j, :], p=2,
dim=1)).cpu().data.numpy()[0] * n
# update the training loss
N += n
bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(
i, len(train_loader),
time.time() - bt,
time.time() - st)
bar.next()
bar.finish()
return t_err / N
def test(train_loader, model, input_n=20, output_n=50, is_cuda=False, dim_used=[], dct_n=15):
N = 0
t_l = 0
if output_n == 25:
eval_frame = [1, 3, 7, 9, 13, 24]
elif output_n == 10:
eval_frame = [1, 3, 7, 9]
t_3d = np.zeros(len(eval_frame))
model.eval()
st = time.time()
bar = Bar('>>>', fill='>', max=len(train_loader))
for i, (inputs, targets, all_seq) in enumerate(train_loader):
bt = time.time()
if is_cuda:
inputs = Variable(inputs.cuda()).float()
all_seq = Variable(all_seq.cuda(async=True)).float()
outputs = model(inputs)
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_3d = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len,
seq_len).transpose(1,
2)
pred_3d = all_seq.clone()
dim_used = np.array(dim_used)
# joints at same loc
joint_to_ignore = np.array([16, 20, 23, 24, 28, 31])
index_to_ignore = np.concatenate((joint_to_ignore * 3, joint_to_ignore * 3 + 1, joint_to_ignore * 3 + 2))
joint_equal = np.array([13, 19, 22, 13, 27, 30])
index_to_equal = np.concatenate((joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2))
pred_3d[:, :, dim_used] = outputs_3d
pred_3d[:, :, index_to_ignore] = pred_3d[:, :, index_to_equal]
pred_p3d = pred_3d.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :]
targ_p3d = all_seq.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :]
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
t_3d[k] += torch.mean(torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2,
1)).cpu().data.numpy() * n
N += n
bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(i+1, len(train_loader), time.time() - bt,
time.time() - st)
bar.next()
bar.finish()
return t_l / N, t_3d / N
def train(train_loader,
model,
optimizer,
lr_now=None,
max_norm=True,
is_cuda=False,
dim_used=[],
dct_n=15):
t_l = utils.AccumLoss()
t_e = utils.AccumLoss()
t_3d = utils.AccumLoss()
model.train()
st = time.time()
bar = Bar('>>>', fill='>', max=len(train_loader))
for i, (inputs, targets, all_seq) in enumerate(train_loader):
bt = time.time()
if is_cuda:
inputs = Variable(inputs.cuda()).float()
all_seq = Variable(all_seq.cuda(async=True)).float()
outputs = model(inputs)
n, seq_len, dim_full_len = all_seq.data.shape
dim_used = np.array(dim_used)
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_p3d = torch.matmul(idct_m[:, :dct_n],
outputs_t).transpose(0, 1).contiguous()
outputs_p3d = outputs_p3d.view(-1, dim_used_len, seq_len).transpose(
1, 2).contiguous().view(-1, 3)
targ_p3d = all_seq[:, :, dim_used].clone().contiguous().view(-1, 3)
loss = torch.mean(torch.norm(targ_p3d - outputs_p3d, 2, 1))
# calculate loss and backward
optimizer.zero_grad()
loss.backward()
if max_norm:
nn.utils.clip_grad_norm(model.parameters(), max_norm=1)
optimizer.step()
t_l.update(loss.cpu().data.numpy()[0] * n, n)
bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(
i + 1, len(train_loader),
time.time() - bt,
time.time() - st)
bar.next()
bar.finish()
return lr_now, t_l.avg, t_e.avg, t_3d.avg
def mpjpe_error_3dpw(outputs, all_seq, dct_n, dim_used):
n, seq_len, dim_full_len = all_seq.data.shape
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_exp = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose(
0, 1).contiguous().view(-1, dim_full_len - 3, seq_len).transpose(1, 2)
pred_3d = all_seq.clone()
pred_3d[:, :, dim_used] = outputs_exp
pred_3d = pred_3d.contiguous().view(-1, dim_full_len).view(-1, 3)
targ_3d = all_seq.contiguous().view(-1, dim_full_len).view(-1, 3)
mean_3d_err = torch.mean(torch.norm(pred_3d - targ_3d, 2, 1))
return mean_3d_err
def dct(model, in_tensor, inverse=False):
"""
:param model: the model instance
:param in_tensor: tensor to have it's temproal component DCT'ed assumes
that the last dim of in_tenosr is the temporal one
:param inverse: toggle to inverse the transformation (boole)
"""
t_n = in_tensor.size()[-1]
dct_matrix, idct_matrix = data_utils.get_dct_matrix(t_n)
if not inverse:
dct_matrix = torch.from_numpy(dct_matrix).to(model.device).float()
tensor_dct = torch.matmul(in_tensor, dct_matrix)
return tensor_dct
else:
idct_matrix = torch.from_numpy(idct_matrix).to(model.device).float()
tensor_idct = torch.matmul(in_tensor, idct_matrix)
return tensor_idct
def euler_error(outputs, all_seq, input_n, dim_used, dct_n):
"""
:param outputs:
:param all_seq:
:param input_n:
:param dim_used:
:return:
"""
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_exp = torch.matmul(idct_m[:, :dct_n],
outputs_t).transpose(0, 1).contiguous().view(
-1, dim_used_len, seq_len).transpose(1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
pred_expmap = pred_expmap[:,
input_n:, :].contiguous().view(-1, dim_full_len)
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(
-1, dim_full_len)
# pred_expmap[:, 0:6] = 0
# targ_expmap[:, 0:6] = 0
pred_expmap = pred_expmap.view(-1, 3)
targ_expmap = targ_expmap.view(-1, 3)
pred_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(pred_expmap))
pred_eul = pred_eul.view(-1, dim_full_len)
targ_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(targ_expmap))
targ_eul = targ_eul.view(-1, dim_full_len)
mean_errors = torch.mean(torch.norm(pred_eul - targ_eul, 2, 1))
return mean_errors
def sen_loss(outputs, all_seq, dim_used, dct_n, inputs, cartesian=False, lambda_ = 0.01, KL=None, reconstructions=None, log_var=None):
"""
:param outputs: N * (seq_len*dim_used_len)
:param all_seq: N * seq_len * dim_full_len
:param input_n:
:param dim_used:
:return:
"""
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
dim_used = np.array(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
if cartesian:
pred_cart = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous()
pred_cart = pred_cart.view(-1, dim_used_len, seq_len).transpose(1, 2).contiguous().view(-1, 3)
targ_cart= all_seq.clone()[:, :, dim_used].contiguous().view(-1, 3)
joint_loss = torch.mean(torch.norm(targ_cart - pred_cart, 2, 1))
else:
pred_expmap = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2)
targ_expmap = all_seq.clone()[:, :, dim_used]
joint_loss = torch.mean(torch.abs(pred_expmap - targ_expmap))
if KL == None:
neg_gauss_log_lik = torch.zeros(1)
latent_loss = torch.zeros(1)
loss = joint_loss
else:
latent_loss = torch.mean(KL)
mse = torch.pow((reconstructions - inputs), 2)
gauss_log_lik = -mse#0.5*(log_var + np.log(2*np.pi) + (mse/(1e-8 + torch.exp(log_var))))
neg_gauss_log_lik = -torch.mean(torch.sum(gauss_log_lik, axis=(1, 2)))
loss = joint_loss + lambda_*(neg_gauss_log_lik + latent_loss)
return loss, joint_loss, neg_gauss_log_lik, latent_loss
def sen_loss(outputs, all_seq, dim_used, dct_n):
"""
:param outputs: N * (seq_len*dim_used_len)
:param all_seq: N * seq_len * dim_full_len
:param input_n:
:param dim_used:
:return:
"""
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
dim_used = np.array(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
pred_expmap = torch.matmul(idct_m[:, :dct_n],
outputs_t).transpose(0, 1).contiguous().view(
-1, dim_used_len, seq_len).transpose(1, 2)
targ_expmap = all_seq.clone()[:, :, dim_used]
loss = torch.mean(
torch.sum(torch.abs(pred_expmap - targ_expmap), dim=2).view(-1))
return loss
def main(opt):
is_cuda = torch.cuda.is_available()
desired_acts = ['eating', 'posing', 'sitting', 'posing', 'walkingdog']
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
dct_n = opt.dct_n
#calculate stepsize for auto regression based on input fames and DCT coefficients
stepsize = dct_n - input_n
sample_rate = opt.sample_rate
model = nnmodel.GCN(input_feature=(input_n + stepsize),
hidden_feature=opt.linear_size,
p_dropout=opt.dropout,
num_stage=opt.num_stage,
node_n=48)
if is_cuda:
model.cuda()
model_path_len = "checkpoint/pretrained/h36m_in{}_out{}_dctn{}.pth.tar".format(
input_n, stepsize, dct_n)
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
err_best = ckpt['err']
start_epoch = ckpt['epoch']
model.load_state_dict(ckpt['state_dict'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
# data loading
print(">>> loading data")
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion(path_to_data=opt.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
dct_n=dct_n,
split=1,
sample_rate=sample_rate)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
dim_used = test_dataset.dim_used
print(">>> data loaded !")
model.eval()
fig = plt.figure()
ax = plt.gca(projection='3d')
#calculate no of iterations in auto regression to perform
iterations = int(output_n / stepsize)
print('iterations: {}'.format(iterations))
for act in acts:
for i, (_, targets, all_seq) in enumerate(test_data[act]):
all_seq = Variable(all_seq).float()
dim_used_len = len(dim_used)
if is_cuda:
all_seq = all_seq.cuda()
dct_m_in, _ = data_utils.get_dct_matrix(dct_n)
dct_m_in = Variable(torch.from_numpy(dct_m_in)).float().cuda()
_, idct_m = data_utils.get_dct_matrix(dct_n)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
targ_expmap = all_seq.cpu().data.numpy()
y_hat = None
#Auto regression
for idx in range(iterations):
#start index of the input sequence
start = input_n + idx * stepsize
#end index of the input sequence
stop = start + stepsize
if y_hat is None:
#slice the sequence of length = (input_n + output_n) in iteration 1
input_seq = all_seq[:, :dct_n, dim_used]
else:
#stack output from prev iteration and next frames to form the next input seq
input_seq = torch.cat(
(y_hat, all_seq[:, start:stop, dim_used]), 1)
#calculate DCT of the input seq
input_dct_seq = torch.matmul(dct_m_in,
input_seq).transpose(1, 2)
if is_cuda:
input_dct_seq = input_dct_seq.cuda()
y = model(input_dct_seq)
y_t = y.view(-1, dct_n).transpose(0, 1)
y_exp = torch.matmul(idct_m,
y_t).transpose(0, 1).contiguous().view(
-1, dim_used_len,
dct_n).transpose(1, 2)
y_hat = y_exp[:, stepsize:, :]
#accumulate the output frames in a single tensor
if idx == 0:
outputs = y_exp
else:
outputs = torch.cat((outputs, y_exp[:, input_n:, :]), 1)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs
pred_expmap = pred_expmap.cpu().data.numpy()
#calculate loss and save to a file for later use
#save_loss_file(act, pred_expmap, targ_expmap, input_n, output_n)
if act in desired_acts:
for k in range(8):
plt.cla()
figure_title = "action:{}, seq:{},".format(act, (k + 1))
viz.plot_predictions(targ_expmap[k, :, :],
pred_expmap[k, :, :], fig, ax,
figure_title)
plt.pause(1)
def main(opt):
is_cuda = torch.cuda.is_available()
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
sample_rate = opt.sample_rate
model = nnmodel.GCN(input_feature=(input_n + output_n), hidden_feature=opt.linear_size, p_dropout=opt.dropout,
num_stage=opt.num_stage, node_n=48)
if is_cuda:
model.cuda()
model_path_len = './checkpoint/pretrained/h36m_in10_out25.pth.tar'
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
err_best = ckpt['err']
start_epoch = ckpt['epoch']
model.load_state_dict(ckpt['state_dict'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(start_epoch, err_best))
# data loading
print(">>> loading data")
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1,
sample_rate=sample_rate)
test_data[act] = DataLoader(
dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
dim_used = test_dataset.dim_used
print(">>> data loaded !")
model.eval()
fig = plt.figure()
ax = plt.gca(projection='3d')
for act in acts:
for i, (inputs, targets, all_seq) in enumerate(test_data[act]):
inputs = Variable(inputs).float()
all_seq = Variable(all_seq).float()
if is_cuda:
inputs = inputs.cuda()
all_seq = all_seq.cuda()
outputs = model(inputs)
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, seq_len).transpose(0, 1)
outputs_exp = torch.matmul(idct_m, outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len,
seq_len).transpose(1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
targ_expmap = all_seq
pred_expmap = pred_expmap.cpu().data.numpy()
targ_expmap = targ_expmap.cpu().data.numpy()
for k in range(8):
plt.cla()
figure_title = "action:{}, seq:{},".format(act, (k + 1))
viz.plot_predictions(targ_expmap[k, :, :], pred_expmap[k, :, :], fig, ax, figure_title)
plt.pause(1)
def test(train_loader,
model,
input_n=20,
output_n=50,
is_cuda=False,
dim_used=[],
dct_n=20):
N = 0
if output_n == 25:
eval_frame = [1, 3, 7, 9, 13, 24]
elif output_n == 10:
eval_frame = [1, 3, 7, 9]
t_e = np.zeros(len(eval_frame))
t_3d = np.zeros(len(eval_frame))
model.eval()
st = time.time()
bar = Bar('>>>', fill='>', max=len(train_loader))
for i, (inputs, targets, all_seq) in enumerate(train_loader):
bt = time.time()
if is_cuda:
inputs = Variable(inputs.cuda()).float()
all_seq = Variable(all_seq.cuda(async=True)).float()
outputs = model(inputs)
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
all_seq[:, :, 0:6] = 0
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, seq_len).transpose(0, 1)
outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view \
(-1, dim_used_len, seq_len).transpose(1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(
-1, dim_full_len)
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(
-1, dim_full_len)
pred_expmap = pred_expmap.view(-1, 3)
targ_expmap = targ_expmap.view(-1, 3)
pred_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(pred_expmap))
pred_eul = pred_eul.view(-1, dim_full_len).view(
-1, output_n, dim_full_len) # [:, :, dim_used]
targ_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(targ_expmap))
targ_eul = targ_eul.view(-1, dim_full_len).view(
-1, output_n, dim_full_len) # [:, :, dim_used]
targ_p3d = data_utils.expmap2xyz_torch_cmu(
targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
pred_p3d = data_utils.expmap2xyz_torch_cmu(
pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
t_e[k] += torch.mean(
torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2,
1)).cpu().data.numpy()[0] * n
t_3d[k] += torch.mean(
torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) -
pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2,
1)).cpu().data.numpy()[0] * n
# update the training loss
N += n
bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(
i + 1, len(train_loader),
time.time() - bt,
time.time() - st)
bar.next()
bar.finish()
return t_e / N, t_3d / N
def __init__(self,
path_to_data,
actions,
input_n=10,
output_n=10,
dct_n=20,
split=0,
sample_rate=2,
data_mean=0,
data_std=0):
"""
read h36m data to get the dct coefficients.
:param path_to_data:
:param actions: actions to read
:param input_n: past frame length
:param output_n: future frame length
:param dct_n: number of dct coeff. used
:param split: 0 train, 1 test, 2 validation
:param sample_rate: 2
:param data_mean: mean of expmap
:param data_std: standard deviation of expmap
"""
self.path_to_data = path_to_data
self.split = split
subs = np.array([[1, 6, 7, 8, 9], [5], [11]])
acts = data_utils.define_actions(actions)
# subs = np.array([[1], [5], [11]])
# acts = ['walking']
subjs = subs[split]
all_seqs, dim_ignore, dim_use, data_mean, data_std = data_utils.load_data(
path_to_data,
subjs,
acts,
sample_rate,
input_n + output_n,
data_mean=data_mean,
data_std=data_std,
input_n=input_n)
self.data_mean = data_mean
self.data_std = data_std
# first 6 elements are global translation and global rotation
dim_used = dim_use[6:]
self.all_seqs = all_seqs
self.dim_used = dim_used
all_seqs = all_seqs[:, :, dim_used]
all_seqs = all_seqs.transpose(0, 2, 1)
all_seqs = all_seqs.reshape(-1, input_n + output_n)
all_seqs = all_seqs.transpose()
dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n)
dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n)
# padding the observed sequence so that it has the same length as observed + future sequence
pad_idx = np.repeat([input_n - 1], output_n)
i_idx = np.append(np.arange(0, input_n), pad_idx)
input_dct_seq = np.matmul(dct_m_in[:dct_n, :], all_seqs[i_idx, :])
input_dct_seq = input_dct_seq.transpose().reshape(
[-1, len(dim_used), dct_n])
output_dct_seq = np.matmul(dct_m_out[:dct_n], all_seqs)
output_dct_seq = output_dct_seq.transpose().reshape(
[-1, len(dim_used), dct_n])
self.input_dct_seq = input_dct_seq
self.output_dct_seq = output_dct_seq
def __init__(self,
path_to_data,
input_n=20,
output_n=10,
dct_n=15,
split=0):
"""
:param path_to_data:
:param input_n:
:param output_n:
:param dct_n:
:param split:
"""
self.path_to_data = path_to_data
self.split = split
self.dct_n = dct_n
# since baselines (http://arxiv.org/abs/1805.00655.pdf and https://arxiv.org/pdf/1705.02445.pdf)
# use observed 50 frames but our method use 10 past frames in order to make sure all methods are evaluated
# on same sequences, we first crop the sequence with 50 past frames and then use the last 10 frame as input
if split == 1:
their_input_n = 50
else:
their_input_n = input_n
seq_len = their_input_n + output_n
if split == 0:
self.data_path = path_to_data + '/train/'
elif split == 1:
self.data_path = path_to_data + '/test/'
elif split == 2:
self.data_path = path_to_data + '/validation/'
all_seqs = []
files = []
for (dirpath, dirnames, filenames) in walk(self.data_path):
files.extend(filenames)
for f in files:
with open(self.data_path + f, 'rb') as f:
data = pkl.load(f, encoding='latin1')
joint_pos = data['jointPositions']
for i in range(len(joint_pos)):
seqs = joint_pos[i]
seqs = seqs - seqs[:, 0:3].repeat(24, axis=0).reshape(
-1, 72)
n_frames = seqs.shape[0]
fs = np.arange(0, n_frames - seq_len + 1)
fs_sel = fs
for j in np.arange(seq_len - 1):
fs_sel = np.vstack((fs_sel, fs + j + 1))
fs_sel = fs_sel.transpose()
seq_sel = seqs[fs_sel, :]
if len(all_seqs) == 0:
all_seqs = seq_sel
else:
all_seqs = np.concatenate((all_seqs, seq_sel), axis=0)
self.all_seqs = all_seqs[:, (their_input_n - input_n):, :]
self.dim_used = np.array(range(3, all_seqs.shape[2]))
all_seqs = all_seqs[:, (their_input_n - input_n):, 3:]
n, seq_len, dim_len = all_seqs.shape
all_seqs = all_seqs.transpose(0, 2, 1)
all_seqs = all_seqs.reshape(-1, input_n + output_n)
all_seqs = all_seqs.transpose()
dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n)
dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n)
pad_idx = np.repeat([input_n - 1], output_n)
i_idx = np.append(np.arange(0, input_n), pad_idx)
input_dct_seq = np.matmul(dct_m_in[0:dct_n, :], all_seqs[i_idx, :])
input_dct_seq = input_dct_seq.transpose().reshape(-1, dim_len, dct_n)
# input_dct_seq = input_dct_seq.reshape(-1, dim_len * dct_used)
output_dct_seq = np.matmul(dct_m_out[0:dct_n, :], all_seqs)
output_dct_seq = output_dct_seq.transpose().reshape(-1, dim_len, dct_n)
# output_dct_seq = output_dct_seq.reshape(-1, dim_len * dct_used)
self.input_dct_seq = input_dct_seq
self.output_dct_seq = output_dct_seq
def main(opt):
is_cuda = torch.cuda.is_available()
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
sample_rate = opt.sample_rate
dct_n = opt.dct_n
model = nnmodel.GCN(input_feature=dct_n,
hidden_feature=opt.linear_size,
p_dropout=opt.dropout,
num_stage=opt.num_stage,
node_n=66)
if is_cuda:
model.to('cuda' if torch.cuda.is_available else 'cpu')
model_path_len = './checkpoint/pretrained/h36m3D_in10_out10_dctn15.pth.tar'
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
err_best = ckpt['err']
start_epoch = ckpt['epoch']
model.load_state_dict(ckpt['state_dict'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
# data loading
print(">>> loading data")
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion3D(path_to_data=opt.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=1,
sample_rate=sample_rate,
dct_used=dct_n)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
dim_used = test_dataset.dim_used
print(">>> data loaded !")
model.eval()
fig = plt.figure()
ax = plt.gca(projection='3d')
for act in acts:
for i, (inputs, targets, all_seq) in enumerate(test_data[act]):
inputs = Variable(inputs).float()
all_seq = Variable(all_seq).float()
if is_cuda:
inputs = inputs.to(
'cuda' if torch.cuda.is_available else 'cpu')
all_seq = all_seq.to(
'cuda' if torch.cuda.is_available else 'cpu')
outputs = model(inputs)
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float()
if is_cuda:
idct_m = idct_m.to(
'cuda' if torch.cuda.is_available else 'cpu')
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_3d = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose(
0, 1).contiguous().view(-1, dim_used_len,
seq_len).transpose(1, 2)
pred_3d = all_seq.clone()
dim_used = np.array(dim_used)
# joints at same loc
joint_to_ignore = np.array([16, 20, 23, 24, 28, 31])
index_to_ignore = np.concatenate(
(joint_to_ignore * 3, joint_to_ignore * 3 + 1,
joint_to_ignore * 3 + 2))
joint_equal = np.array([13, 19, 22, 13, 27, 30])
index_to_equal = np.concatenate(
(joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2))
pred_3d[:, :, dim_used] = outputs_3d
pred_3d[:, :, index_to_ignore] = pred_3d[:, :, index_to_equal]
pred_p3d = pred_3d.contiguous().view(
n, seq_len, -1, 3)[:, input_n:, :, :].cpu().data.numpy()
targ_p3d = all_seq.contiguous().view(
n, seq_len, -1, 3)[:, input_n:, :, :].cpu().data.numpy()
for k in range(8):
plt.cla()
figure_title = "action:{}, seq:{},".format(act, (k + 1))
viz.plot_predictions_direct(targ_p3d[k], pred_p3d[k], fig, ax,
figure_title)
plt.pause(1)
def test(train_loader,
model,
stepsize,
input_n=20,
output_n=50,
dct_n=20,
is_cuda=False,
dim_used=[]):
N = 0
# t_l = 0
if output_n == 25:
eval_frame = [1, 3, 7, 9, 13, 24]
elif output_n == 50:
eval_frame = [1, 3, 7, 9, 13, 24, 35]
elif output_n == 100:
eval_frame = [1, 3, 7, 9, 13, 24, 35, 49]
elif output_n == 10:
eval_frame = [1, 3, 7, 9]
t_e = np.zeros(len(eval_frame))
t_3d = np.zeros(len(eval_frame))
model.eval()
# calculate no of iterations in auto regression to perform
iterations = int(output_n / stepsize)
print('iterations: {}'.format(iterations))
st = time.time()
bar = Bar('>>>', fill='>', max=len(train_loader))
for i, (inputs, targets, all_seq) in enumerate(train_loader):
bt = time.time()
all_seq = Variable(all_seq).float()
dim_used_len = len(dim_used)
if is_cuda:
all_seq = all_seq.cuda()
dct_m_in, _ = data_utils.get_dct_matrix(dct_n)
dct_m_in = Variable(torch.from_numpy(dct_m_in)).float().cuda()
_, idct_m = data_utils.get_dct_matrix(dct_n)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
n, seq_len, dim_full_len = all_seq.data.shape
y_hat = None
# Auto regression
for idx in range(iterations):
# start index of the input sequence
start = input_n + idx * stepsize
# end index of the input sequence
stop = start + stepsize
if y_hat is None:
# slice the sequence of length = (input_n + output_n) in iteration 1
input_seq = all_seq[:, :dct_n, dim_used]
else:
# stack output from prev iteration and next frames to form the next input seq
input_seq = torch.cat(
(y_hat, all_seq[:, start:stop, dim_used]), 1)
# calculate DCT of the input seq
input_dct_seq = torch.matmul(dct_m_in, input_seq).transpose(1, 2)
if is_cuda:
input_dct_seq = input_dct_seq.cuda()
y = model(input_dct_seq)
y_t = y.view(-1, dct_n).transpose(0, 1)
y_exp = torch.matmul(idct_m,
y_t).transpose(0, 1).contiguous().view(
-1, dim_used_len, dct_n).transpose(1, 2)
y_hat = y_exp[:, stepsize:, :]
# accumulate the output frames in a single tensor
if idx == 0:
outputs = y_exp
else:
outputs = torch.cat((outputs, y_exp[:, input_n:, :]), 1)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs
pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(
-1, dim_full_len)
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(
-1, dim_full_len)
pred_expmap[:, 0:6] = 0
targ_expmap[:, 0:6] = 0
pred_expmap = pred_expmap.view(-1, 3)
targ_expmap = targ_expmap.view(-1, 3)
# get euler angles from expmap
pred_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(pred_expmap))
pred_eul = pred_eul.view(-1,
dim_full_len).view(-1, output_n, dim_full_len)
targ_eul = data_utils.rotmat2euler_torch(
data_utils.expmap2rotmat_torch(targ_expmap))
targ_eul = targ_eul.view(-1,
dim_full_len).view(-1, output_n, dim_full_len)
# get 3d coordinates
targ_p3d = data_utils.expmap2xyz_torch(
targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
pred_p3d = data_utils.expmap2xyz_torch(
pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
# update loss and testing errors
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
'''
t_e[k] += torch.mean(torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2, 1)).cpu().data.numpy()[0] * n
t_3d[k] += torch.mean(torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2,
1)).cpu().data.numpy()[0] * n
'''
t_e[k] += torch.mean(
torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2,
1)).item() * n
t_3d[k] += torch.mean(
torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) -
pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2,
1)).item() * n
# t_l += loss.cpu().data.numpy()[0] * n
N += n
bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(
i + 1, len(train_loader),
time.time() - bt,
time.time() - st)
bar.next()
bar.finish()
return t_e / N, t_3d / N
def test(self, train_loader, dataset='h3.6m', input_n=20, output_n=50, dct_n=20, cartesian=False,
dim_used=[]):
N = 0
# t_l = 0
if output_n >= 25:
eval_frame = [1, 3, 7, 9, 13, 24]
elif output_n == 10:
eval_frame = [1, 3, 7, 9]
t_e = np.zeros(len(eval_frame))
t_3d = np.zeros(len(eval_frame))
self.model.eval()
st = time.time()
bar = Bar('>>>', fill='>', max=len(train_loader))
for i, (inputs, targets, all_seq) in enumerate(train_loader):
bt = time.time()
if self.is_cuda:
inputs = Variable(inputs.cuda()).float()
all_seq = Variable(all_seq.cuda(non_blocking=True)).float()
outputs, reconstructions, log_var, z = self.model(inputs.float())
n = outputs.shape[0]
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
all_seq[:, :, 0:6] = 0
# inverse dct transformation
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
if cartesian == False:
outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1,
dim_used_len,
seq_len).transpose(
1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(-1, dim_full_len)
targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(-1, dim_full_len)
pred_expmap[:, 0:6] = 0
targ_expmap[:, 0:6] = 0
pred_expmap = pred_expmap.view(-1, 3)
targ_expmap = targ_expmap.view(-1, 3)
# get euler angles from expmap
pred_eul = data_utils.rotmat2euler_torch(data_utils.expmap2rotmat_torch(pred_expmap))
pred_eul = pred_eul.view(-1, dim_full_len).view(-1, output_n, dim_full_len)
targ_eul = data_utils.rotmat2euler_torch(data_utils.expmap2rotmat_torch(targ_expmap))
targ_eul = targ_eul.view(-1, dim_full_len).view(-1, output_n, dim_full_len)
if dataset == 'h3.6m':
# get 3d coordinates
targ_p3d = data_utils.expmap2xyz_torch(targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
pred_p3d = data_utils.expmap2xyz_torch(pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3)
elif dataset == 'cmu_mocap':
# get 3d coordinates
targ_p3d = data_utils.expmap2xyz_torch_cmu(targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1,
3)
pred_p3d = data_utils.expmap2xyz_torch_cmu(pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1,
3)
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
t_e[k] += torch.mean(torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2, 1)).cpu().data.numpy() * n
t_3d[k] += torch.mean(torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3),
2, 1)).cpu().data.numpy() * n
elif cartesian:
outputs_3d = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1,
dim_used_len,
seq_len).transpose(
1, 2)
pred_3d = all_seq.clone()
dim_used = np.array(dim_used)
# deal with joints at same position
joint_to_ignore = np.array([16, 20, 29, 24, 27, 33, 36])
index_to_ignore = np.concatenate(
(joint_to_ignore * 3, joint_to_ignore * 3 + 1, joint_to_ignore * 3 + 2))
joint_equal = np.array([15, 15, 15, 23, 23, 32, 32])
index_to_equal = np.concatenate((joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2))
pred_3d[:, :, dim_used] = outputs_3d
pred_3d[:, :, index_to_ignore] = pred_3d[:, :, index_to_equal]
pred_p3d = pred_3d.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :]
targ_p3d = all_seq.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :]
for k in np.arange(0, len(eval_frame)):
j = eval_frame[k]
t_e[k] += torch.mean(torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3),
2, 1)).cpu().data.numpy()[0] * n
t_3d[k] += torch.mean(torch.norm(
targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3),
2, 1)).cpu().data.numpy()[0] * n
N += n
bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(i + 1, len(train_loader), time.time() - bt,
time.time() - st)
bar.next()
bar.finish()
return t_e / N, t_3d / N
