def __init__(self, path_to_data, actions, input_n=20, output_n=10, split=0, sample_rate=2,
autoencoder=IdentityAutoencoder(), subset=False, treat_subj5_differently=True):
"""
:param split: 0 train, 1 testing, 2 validation
"""
# Note: the default autoencoder is an indentity mapping which is what is used in the paper
self.path_to_data = path_to_data
self.split = split
subs = np.array([[1, 6, 7, 8, 9], [5], [11]], dtype=object)
acts = data_utils.define_actions(actions)
if subset:
subs = np.array([[1], [5], [11]], dtype=object)
acts = ['walking']
subjs = subs[split]
all_seqs, dim_ignore, dim_used = data_utils.load_data_3d(path_to_data, subjs, acts, sample_rate,
input_n + output_n,
treat_subj5_differently=treat_subj5_differently)
self.all_seqs = all_seqs
self.dim_used = dim_used
# (nb_total_seq, len_seq, nb_joints)
all_seqs = torch.from_numpy(all_seqs[:, :, dim_used]).float()
# (nb_total_seq, nb_joints, hidden_dim)
self.all_seqs_encoded = autoencoder(all_seqs.transpose(2, 1))[1]
def __init__(self, path_to_data, actions, input_n=10, output_n=10, split=0, sample_rate=2, data_mean=0,
data_std=0):
"""
read h36m data to get the dct coefficients.
:param path_to_data:
:param actions: actions to read
:param input_n: past frame length
:param output_n: future frame length
:param dct_n: number of dct coeff. used
:param split: 0 train, 1 test, 2 validation
:param sample_rate: 2
:param data_mean: mean of expmap
:param data_std: standard deviation of expmap
"""
self.path_to_data = path_to_data
self.split = split
subs = np.array([[1, 6, 7, 8, 9], [5], [11]])
acts = data_utils.define_actions(actions)
# subs = np.array([[1], [5], [11]])
# acts = ['walking']
subjs = subs[split]
all_seqs, dim_ignore, dim_use, data_mean, data_std = data_utils.load_data(path_to_data, subjs, acts,
sample_rate,
input_n + output_n,
data_mean=data_mean,
data_std=data_std,
input_n=input_n)
self.data_mean = data_mean
self.data_std = data_std
# first 6 elements are global translation and global rotation
dim_used = dim_use[6:]
self.all_seqs = all_seqs
self.dim_used = dim_used
all_seqs = all_seqs[:, :, dim_used]
all_seqs = all_seqs.transpose(0, 2, 1)
all_seqs = all_seqs.reshape(-1, input_n + output_n)
all_seqs = all_seqs.transpose()
# padding the observed sequence so that it has the same length as observed + future sequence
pad_idx = np.repeat([input_n - 1], output_n)
i_idx = np.append(np.arange(0, input_n), pad_idx)
input_chebyshev_coef = data_utils.get_chebyshev_coef(input_n + output_n, all_seqs[i_idx, :])
input_chebyshev_coef = input_chebyshev_coef.transpose().reshape([-1, len(dim_used), input_n + output_n])
target_chebyshev_coef = data_utils.get_chebyshev_coef(input_n + output_n, all_seqs)
target_chebyshev_coef = target_chebyshev_coef.transpose().reshape([-1, len(dim_used), input_n + output_n])
self.input_chebyshev_coef = input_chebyshev_coef
#self.output_chebyshev_coef = target_chebyshev_coef
self.output_chebyshev_coef = target_chebyshev_coef-input_chebyshev_coef
def __init__(self,
path_to_data,
actions,
input_n=10,
output_n=10,
dct_n=20,
split=0,
sample_rate=2,
data_mean=0,
data_std=0):
"""
read h36m data to get the dct coefficients.
:param path_to_data:
:param actions: actions to read
:param input_n: past frame length
:param output_n: future frame length
:param dct_n: number of dct coeff. used
:param split: 0 train, 1 test, 2 validation
:param sample_rate: 2
:param data_mean: mean of expmap
:param data_std: standard deviation of expmap
"""
self.path_to_data = path_to_data
self.split = split
subs = np.array([[1, 6, 7, 8, 9], [5], [11]])
acts = data_utils.define_actions(actions)
# subs = np.array([[1], [5], [11]])
# acts = ['walking']
subjs = subs[split]
all_seqs, dim_ignore, dim_use, data_mean, data_std = data_utils.load_data(
path_to_data,
subjs,
acts,
sample_rate,
1,
data_mean=data_mean,
data_std=data_std,
input_n=1)
self.data_mean = data_mean
self.data_std = data_std
self.max = np.max(all_seqs)
self.min = np.min(all_seqs)
# first 6 elements are global translation and global rotation
dim_used = dim_use[6:]
self.all_seqs = all_seqs
self.dim_used = dim_used
m = self.all_seqs.shape[0]
n = self.all_seqs.shape[-1]
self.all_seqs = self.all_seqs.reshape((m, n))
#print("all seq_shape ", self.all_seqs.shape)
self.all_seqs = self.all_seqs[:, dim_used]
def __init__(self,
path_to_data,
actions,
input_n=20,
output_n=10,
dct_used=15,
split=0,
sample_rate=2):
"""
:param path_to_data:
:param actions:
:param input_n:
:param output_n:
:param dct_used:
:param split: 0 train, 1 testing, 2 validation
:param sample_rate:
"""
self.path_to_data = path_to_data
self.split = split
self.dct_used = dct_used
subs = np.array([[1, 6, 7, 8, 9], [5], [11]])
acts = data_utils.define_actions(actions)
# subs = np.array([[1], [5], [11]])
# acts = ['walking']
subjs = subs[split]
all_seqs, dim_ignore, dim_used = data_utils.load_data_3d(
path_to_data, subjs, acts, sample_rate, input_n + output_n)
self.all_seqs = all_seqs
self.dim_used = dim_used
all_seqs = all_seqs[:, :, dim_used]
all_seqs = all_seqs.transpose(0, 2, 1)
all_seqs = all_seqs.reshape(-1, input_n + output_n)
all_seqs = all_seqs.transpose()
dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n)
dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n)
pad_idx = np.repeat([input_n - 1], output_n)
i_idx = np.append(np.arange(0, input_n), pad_idx)
input_dct_seq = np.matmul(dct_m_in[0:dct_used, :], all_seqs[i_idx, :])
input_dct_seq = input_dct_seq.transpose().reshape(
[-1, len(dim_used), dct_used])
# input_dct_seq = input_dct_seq.reshape(-1, len(dim_used) * dct_used)
output_dct_seq = np.matmul(dct_m_out[0:dct_used, :], all_seqs)
output_dct_seq = output_dct_seq.transpose().reshape(
[-1, len(dim_used), dct_used])
# output_dct_seq = output_dct_seq.reshape(-1, len(dim_used) * dct_used)
self.input_dct_seq = input_dct_seq
self.output_dct_seq = output_dct_seq
def main(opt):
model = nnmodel.InceptionGCN(opt.linear_size,
opt.dropout,
num_stage=opt.num_stage,
node_n=66,
opt=opt)
# If you change the path of the model, change the configuration of opt accordingly e.g. if the model
# was trained for long term prediction, change opt.output_n to 25
model_path_len = './checkpoint/pretrained/ckpt_main_3d_3D_in10_out10_best.pth.tar'
ckpt = torch.load(model_path_len, map_location='cpu')
model.load_state_dict(ckpt['state_dict'])
# data loading
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion3D(path_to_data=opt.data_dir,
actions=act,
input_n=opt.input_n,
output_n=opt.output_n,
split=1,
sample_rate=opt.sample_rate)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
dim_used = test_dataset.dim_used
print(">>> data loaded !")
model.eval()
fig = plt.figure()
ax = plt.gca(projection='3d')
for act in acts:
print(act)
if opt.output_n == 25 and act not in [
'smoking', 'eating', 'discussion', 'walking'
]:
continue
for i, (inputs, all_seq) in enumerate(test_data[act]):
print(act)
preds = model(inputs)
pred_exmap = all_seq.clone()
pred_exmap[:, :, dim_used] = preds.detach().transpose(1, 2)
for k in range(0, 8):
plt.cla()
figure_title = "action:{}, seq:{},".format(act, (k + 1))
viz.plot_predictions(all_seq.numpy()[k, :, :],
pred_exmap.numpy()[k, :, :], fig, ax,
figure_title)
def __init__(self,
path_to_data,
actions,
input_n=20,
output_n=10,
dct_used=15,
split=0,
sample_rate=2):
"""
:param path_to_data:
:param actions:
:param input_n:
:param output_n:
:param dct_used:
:param split: 0 train, 1 testing, 2 validation
:param sample_rate:
"""
self.path_to_data = path_to_data
self.split = split
self.dct_used = dct_used
subs = np.array([[1, 6, 7, 8, 9], [5], [11]])
acts = data_utils.define_actions(actions)
# subs = np.array([[1], [5], [11]])
# acts = ['walking']
subjs = subs[split]
all_seqs, dim_ignore, dim_used = data_utils.load_data_3d(
path_to_data, subjs, acts, sample_rate, 20)
self.all_seqs = all_seqs
self.dim_used = dim_used
m = self.all_seqs.shape[0]
n = self.all_seqs.shape[-1]
#average pose over 20 contiguous timesteps
self.all_seqs = np.mean(self.all_seqs, axis=1)
#normalise to unit cube
self.all_seqs = self.all_seqs.reshape((m, n))
max_per_pose = np.amax(self.all_seqs, -1)
max_per_pose = np.repeat(max_per_pose.reshape(m, 1), 96, axis=1)
min_per_pose = np.amin(self.all_seqs, -1)
min_per_pose = np.repeat(min_per_pose.reshape(m, 1), 96, axis=1)
self.all_seqs = (self.all_seqs - min_per_pose) / (max_per_pose -
min_per_pose)
def main(opt):
is_cuda = torch.cuda.is_available()
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
sample_rate = opt.sample_rate
model = nnmodel.GCN(input_feature=(input_n + output_n), hidden_feature=opt.linear_size, p_dropout=opt.dropout,
num_stage=opt.num_stage, node_n=48)
if is_cuda:
model.cuda()
model_path_len = './checkpoint/pretrained/h36m_in10_out25.pth.tar'
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
err_best = ckpt['err']
start_epoch = ckpt['epoch']
model.load_state_dict(ckpt['state_dict'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(start_epoch, err_best))
# data loading
print(">>> loading data")
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1,
sample_rate=sample_rate)
test_data[act] = DataLoader(
dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
dim_used = test_dataset.dim_used
print(">>> data loaded !")
model.eval()
fig = plt.figure()
ax = plt.gca(projection='3d')
for act in acts:
for i, (inputs, targets, all_seq) in enumerate(test_data[act]):
inputs = Variable(inputs).float()
all_seq = Variable(all_seq).float()
if is_cuda:
inputs = inputs.cuda()
all_seq = all_seq.cuda()
outputs = model(inputs)
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
outputs_t = outputs.view(-1, seq_len).transpose(0, 1)
outputs_exp = torch.matmul(idct_m, outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len,
seq_len).transpose(1, 2)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
targ_expmap = all_seq
pred_expmap = pred_expmap.cpu().data.numpy()
targ_expmap = targ_expmap.cpu().data.numpy()
for k in range(8):
plt.cla()
figure_title = "action:{}, seq:{},".format(act, (k + 1))
viz.plot_predictions(targ_expmap[k, :, :], pred_expmap[k, :, :], fig, ax, figure_title)
plt.pause(1)
def main(opt):
is_cuda = torch.cuda.is_available()
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
sample_rate = opt.sample_rate
dct_n = opt.dct_n
model = nnmodel.GCN(input_feature=dct_n,
hidden_feature=opt.linear_size,
p_dropout=opt.dropout,
num_stage=opt.num_stage,
node_n=66)
if is_cuda:
model.to('cuda' if torch.cuda.is_available else 'cpu')
model_path_len = './checkpoint/pretrained/h36m3D_in10_out10_dctn15.pth.tar'
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
err_best = ckpt['err']
start_epoch = ckpt['epoch']
model.load_state_dict(ckpt['state_dict'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
# data loading
print(">>> loading data")
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion3D(path_to_data=opt.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=1,
sample_rate=sample_rate,
dct_used=dct_n)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
dim_used = test_dataset.dim_used
print(">>> data loaded !")
model.eval()
fig = plt.figure()
ax = plt.gca(projection='3d')
for act in acts:
for i, (inputs, targets, all_seq) in enumerate(test_data[act]):
inputs = Variable(inputs).float()
all_seq = Variable(all_seq).float()
if is_cuda:
inputs = inputs.to(
'cuda' if torch.cuda.is_available else 'cpu')
all_seq = all_seq.to(
'cuda' if torch.cuda.is_available else 'cpu')
outputs = model(inputs)
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
_, idct_m = data_utils.get_dct_matrix(seq_len)
idct_m = Variable(torch.from_numpy(idct_m)).float()
if is_cuda:
idct_m = idct_m.to(
'cuda' if torch.cuda.is_available else 'cpu')
outputs_t = outputs.view(-1, dct_n).transpose(0, 1)
outputs_3d = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose(
0, 1).contiguous().view(-1, dim_used_len,
seq_len).transpose(1, 2)
pred_3d = all_seq.clone()
dim_used = np.array(dim_used)
# joints at same loc
joint_to_ignore = np.array([16, 20, 23, 24, 28, 31])
index_to_ignore = np.concatenate(
(joint_to_ignore * 3, joint_to_ignore * 3 + 1,
joint_to_ignore * 3 + 2))
joint_equal = np.array([13, 19, 22, 13, 27, 30])
index_to_equal = np.concatenate(
(joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2))
pred_3d[:, :, dim_used] = outputs_3d
pred_3d[:, :, index_to_ignore] = pred_3d[:, :, index_to_equal]
pred_p3d = pred_3d.contiguous().view(
n, seq_len, -1, 3)[:, input_n:, :, :].cpu().data.numpy()
targ_p3d = all_seq.contiguous().view(
n, seq_len, -1, 3)[:, input_n:, :, :].cpu().data.numpy()
for k in range(8):
plt.cla()
figure_title = "action:{}, seq:{},".format(act, (k + 1))
viz.plot_predictions_direct(targ_p3d[k], pred_p3d[k], fig, ax,
figure_title)
plt.pause(1)
opt = parser.parse_args()
is_cuda = torch.cuda.is_available()
input_n = 10
output_n = 25
dct_n = 35
sample_rate = 2
cartesian = False
if opt.dataset == 'h3.6m':
node_n = 48
n_z = 384
actions = data_utils.define_actions('all', 'h3.6m', out_of_distribution=False)
elif opt.dataset == 'cmu_mocap':
node_n = 64
actions = data_utils.define_actions('all', 'cmu_mocap', out_of_distribution=False)
n_z = 512
model = nnmodel.GCN(input_feature=dct_n, hidden_feature=256, p_dropout=0.3,
num_stage=12, node_n=node_n, variational=True, n_z=32, num_decoder_stage=6)
if is_cuda:
model.cuda()
print(">>> total params: {:.2f}M".format(sum(p.numel() for p in model.parameters()) / 1000000.0))
# optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
if opt.model_path == None:
model_path_len = 'checkpoint/test/ckpt_main_cmu_mocap_in50_out25_dctn35_dropout_0.3_var_lambda_0.003_nz_8_lr_0.0005_n_layers_6_last.pth.tar'
else:
def main(opt):
start_epoch = 0
err_best = 10000
lr_now = opt.lr
is_cuda = torch.cuda.is_available()
opt.is_load = True
# save option in log
script_name = os.path.basename(__file__).split('.')[0]
script_name = script_name + '_3D_in{:d}_out{:d}_dct_n_{:d}'.format(
opt.input_n, opt.output_n, opt.dct_n)
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
dct_n = opt.dct_n
sample_rate = opt.sample_rate
model = nnmodel.GCN(input_feature=dct_n,
hidden_feature=opt.linear_size,
p_dropout=opt.dropout,
num_stage=opt.num_stage,
node_n=66)
if is_cuda:
model.cuda()
print(">>> total params: {:.2f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
if opt.is_load:
model_path_len = 'checkpoint/pretrained/h36m3D_in10_out25_dctn30.pth.tar'
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
start_epoch = ckpt['epoch']
err_best = ckpt['err']
lr_now = ckpt['lr']
model.load_state_dict(ckpt['state_dict'])
optimizer.load_state_dict(ckpt['optimizer'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
# data loading
print(">>> loading data")
# train_dataset = H36motion3D(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n,
# split=0, dct_used=dct_n, sample_rate=sample_rate)
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion3D(path_to_data=opt.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=1,
sample_rate=sample_rate,
dct_used=dct_n)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
# val_dataset = H36motion3D(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n,
# split=2, dct_used=dct_n, sample_rate=sample_rate)
# load dadasets for training
# train_loader = DataLoader(
# dataset=train_dataset,
# batch_size=opt.train_batch,
# shuffle=True,
# num_workers=opt.job,
# pin_memory=True)
# val_loader = DataLoader(
# dataset=val_dataset,
# batch_size=opt.test_batch,
# shuffle=False,
# num_workers=opt.job,
# pin_memory=True)
print(">>> data loaded !")
# print(">>> train data {}".format(train_dataset.__len__()))
# print(">>> test data {}".format(test_dataset.__len__()))
# print(">>> validation data {}".format(val_dataset.__len__()))
# for epoch in range(start_epoch, opt.epochs):
#
# if (epoch + 1) % opt.lr_decay == 0:
# lr_now = utils.lr_decay(optimizer, lr_now, opt.lr_gamma)
#
# print('==========================')
# print('>>> epoch: {} | lr: {:.5f}'.format(epoch + 1, lr_now))
ret_log = np.array([start_epoch])
head = np.array(['epoch'])
# per epoch
# lr_now, t_l = train(train_loader, model, optimizer, lr_now=lr_now, max_norm=opt.max_norm, is_cuda=is_cuda,
# dim_used=train_dataset.dim_used, dct_n=dct_n)
# ret_log = np.append(ret_log, [lr_now, t_l])
# head = np.append(head, ['lr', 't_l'])
#
# v_3d = val(val_loader, model, is_cuda=is_cuda, dim_used=train_dataset.dim_used, dct_n=dct_n)
#
# ret_log = np.append(ret_log, [v_3d])
# head = np.append(head, ['v_3d'])
test_3d_temp = np.array([])
test_3d_head = np.array([])
for act in acts:
test_l, test_3d = test(test_data[act],
model,
input_n=input_n,
output_n=output_n,
is_cuda=is_cuda,
dim_used=test_dataset.dim_used,
dct_n=dct_n)
# ret_log = np.append(ret_log, test_l)
ret_log = np.append(ret_log, test_3d)
head = np.append(
head, [act + '3d80', act + '3d160', act + '3d320', act + '3d400'])
if output_n > 10:
head = np.append(head, [act + '3d560', act + '3d1000'])
ret_log = np.append(ret_log, test_3d_temp)
head = np.append(head, test_3d_head)
# update log file and save checkpoint
df = pd.DataFrame(np.expand_dims(ret_log, axis=0))
# if epoch == start_epoch:
df.to_csv(opt.ckpt + '/' + script_name + '.csv', header=head, index=False)
def __init__(self,
path_to_data,
actions,
input_n=20,
dct_used=15,
split=0,
sample_rate=2):
"""
:param path_to_data:
:param actions:
:param input_n:
:param output_n:
:param dct_used:
:param split: 0 train, 1 testing, 2 validation
:param sample_rate:
"""
self.path_to_data = path_to_data
self.split = split
self.dct_used = dct_used
subs = np.array([[1, 6, 7, 8, 9], [5], [11]])
acts = data_utils.define_actions(actions)
subjs = subs[split]
labels = []
all_seqs = np.array([])
#print(acts)
for act in acts:
#print(act)
action_seq, dim_ignore, dim_used = data_utils.load_data_3d(
path_to_data, subjs, [act], sample_rate, input_n)
#print(action_seq.shape)
try:
all_seqs = np.concatenate((all_seqs, action_seq), axis=0)
except:
all_seqs = action_seq
label = [str(act)] * action_seq.shape[0]
labels = labels + label
#print(len(labels))
#print(all_seqs.shape)
#print(labels[0], labels[-1])
#all_seqs, dim_ignore, dim_used = data_utils.load_data_3d(path_to_data, subjs, acts, sample_rate, input_n )
self.labels = labels
self.all_seqs = all_seqs
t_n = input_n
b_n, f_n = self.all_seqs.shape[0], 96
self.all_seqs = self.all_seqs.reshape(b_n, f_n, t_n)
#normalise to unit cube
max_per_pose = np.amax(self.all_seqs, axis=(1, 2))
max_per_pose = np.repeat(max_per_pose.reshape(b_n, 1, 1), 96, axis=1)
max_per_pose = np.repeat(max_per_pose.reshape(b_n, 96, 1), t_n, axis=2)
min_per_pose = np.amin(self.all_seqs, axis=(1, 2))
min_per_pose = np.repeat(min_per_pose.reshape(b_n, 1, 1), 96, axis=1)
min_per_pose = np.repeat(min_per_pose.reshape(b_n, 96, 1), t_n, axis=2)
self.all_seqs = (self.all_seqs - min_per_pose) / (max_per_pose -
min_per_pose)
def main(opt):
start_epoch = 0
err_best = 10000
lr_now = opt.lr
is_cuda = torch.cuda.is_available()
# define log csv file
script_name = os.path.basename(__file__).split('.')[0]
script_name = script_name + "_in{:d}_out{:d}_dctn{:d}".format(
opt.input_n, opt.output_n, opt.dct_n)
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
dct_n = opt.dct_n
sample_rate = opt.sample_rate
# 48 nodes for angle prediction
model = nnmodel.GCN(input_feature=input_n + output_n,
hidden_feature=opt.linear_size,
p_dropout=opt.dropout,
num_stage=opt.num_stage,
node_n=48)
if is_cuda:
model.cuda()
print(">>> total params: {:.2f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
# continue from checkpoint
if opt.is_load:
model_path_len = 'checkpoint/test/ckpt_main_gcn_muti_att_best.pth.tar'
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
start_epoch = ckpt['epoch']
err_best = ckpt['err']
lr_now = ckpt['lr']
model.load_state_dict(ckpt['state_dict'])
optimizer.load_state_dict(ckpt['optimizer'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
# data loading
print(">>> loading data")
train_dataset = H36motion(path_to_data=opt.data_dir,
actions='all',
input_n=input_n,
output_n=output_n,
split=0,
sample_rate=sample_rate)
data_std = train_dataset.data_std
data_mean = train_dataset.data_mean
val_dataset = H36motion(path_to_data=opt.data_dir,
actions='all',
input_n=input_n,
output_n=output_n,
split=2,
sample_rate=sample_rate,
data_mean=data_mean,
data_std=data_std)
# load dadasets for training
train_loader = DataLoader(dataset=train_dataset,
batch_size=opt.train_batch,
shuffle=True,
num_workers=opt.job,
pin_memory=True)
val_loader = DataLoader(dataset=val_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion(path_to_data=opt.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=1,
sample_rate=sample_rate,
data_mean=data_mean,
data_std=data_std)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
print(">>> data loaded !")
print(">>> train data {}".format(train_dataset.__len__()))
print(">>> validation data {}".format(val_dataset.__len__()))
for epoch in range(start_epoch, opt.epochs):
if (epoch + 1) % opt.lr_decay == 0:
lr_now = utils.lr_decay(optimizer, lr_now, opt.lr_gamma)
print('==========================')
print('>>> epoch: {} | lr: {:.5f}'.format(epoch + 1, lr_now))
ret_log = np.array([epoch + 1])
head = np.array(['epoch'])
# per epoch
lr_now, t_l, t_e, t_3d = train(train_loader,
model,
optimizer,
input_n=input_n,
output_n=output_n,
lr_now=lr_now,
max_norm=opt.max_norm,
is_cuda=is_cuda,
dim_used=train_dataset.dim_used)
ret_log = np.append(ret_log, [lr_now, t_l, t_e, t_3d])
head = np.append(head, ['lr', 't_l', 't_e', 't_3d'])
v_e, v_3d = val(val_loader,
model,
input_n=input_n,
output_n=output_n,
is_cuda=is_cuda,
dim_used=train_dataset.dim_used)
ret_log = np.append(ret_log, [v_e, v_3d])
head = np.append(head, ['v_e', 'v_3d'])
test_3d_temp = np.array([])
test_3d_head = np.array([])
for act in acts:
test_e, test_3d = test(test_data[act],
model,
input_n=input_n,
output_n=output_n,
is_cuda=is_cuda,
dim_used=train_dataset.dim_used)
ret_log = np.append(ret_log, test_e)
test_3d_temp = np.append(test_3d_temp, test_3d)
test_3d_head = np.append(
test_3d_head,
[act + '3d80', act + '3d160', act + '3d320', act + '3d400'])
head = np.append(
head, [act + '80', act + '160', act + '320', act + '400'])
if output_n > 10:
head = np.append(head, [act + '560', act + '1000'])
test_3d_head = np.append(test_3d_head,
[act + '3d560', act + '3d1000'])
ret_log = np.append(ret_log, test_3d_temp)
head = np.append(head, test_3d_head)
# update log file and save checkpoint
df = pd.DataFrame(np.expand_dims(ret_log, axis=0))
if epoch == start_epoch:
df.to_csv(opt.ckpt + '/' + script_name + '.csv',
header=head,
index=False)
else:
with open(opt.ckpt + '/' + script_name + '.csv', 'a') as f:
df.to_csv(f, header=False, index=False)
if not np.isnan(v_e):
is_best = v_e < err_best
err_best = min(v_e, err_best)
else:
is_best = False
file_name = [
'ckpt_' + script_name + '_best.pth.tar',
'ckpt_' + script_name + '_last.pth.tar'
]
utils.save_ckpt(
{
'epoch': epoch + 1,
'lr': lr_now,
'err': test_e[0],
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict()
},
ckpt_path=opt.ckpt,
is_best=is_best,
file_name=file_name)
def get_poses(self,
input_n,
output_n,
sample_rate,
dct_n,
out_of_distribution_action=None,
val_categorise=False):
if out_of_distribution_action != None:
self.out_of_distribution = True
self.acts_train = data_utils.define_actions(
out_of_distribution_action,
self.dataset,
out_of_distribution=False)
self.acts_OoD = data_utils.define_actions(
out_of_distribution_action,
self.dataset,
out_of_distribution=True)
self.acts_test = data_utils.define_actions(
'all', self.dataset, out_of_distribution=False)
else:
self.out_of_distribution = False
self.acts_train = data_utils.define_actions(
'all', self.dataset, out_of_distribution=False)
self.acts_OoD = None
self.acts_test = data_utils.define_actions(
'all', self.dataset, out_of_distribution=False)
if self.dataset == 'h3.6m':
self.cartesian = False
self.node_n = 48
self.train_dataset = H36motion_pose(path_to_data=self.data_dir,
actions=self.acts_train,
input_n=input_n,
output_n=output_n,
split=0,
sample_rate=sample_rate,
dct_n=dct_n)
self.data_std = self.train_dataset.data_std
self.data_mean = self.train_dataset.data_mean
self.dim_used = self.train_dataset.dim_used
self.val_dataset = H36motion_pose(path_to_data=self.data_dir,
actions=self.acts_train,
input_n=input_n,
output_n=output_n,
split=2,
sample_rate=sample_rate,
data_mean=self.data_mean,
data_std=self.data_std,
dct_n=dct_n)
if self.out_of_distribution:
self.OoD_val_dataset = H36motion_pose(
path_to_data=self.data_dir,
actions=self.acts_OoD,
input_n=input_n,
output_n=output_n,
split=2,
sample_rate=sample_rate,
data_mean=self.data_mean,
data_std=self.data_std,
dct_n=dct_n)
else:
self.OoD_val_dataset = None
self.test_dataset = dict()
for act in self.acts_test:
self.test_dataset[act] = H36motion_pose(
path_to_data=self.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=1,
sample_rate=sample_rate,
data_mean=self.data_mean,
data_std=self.data_std,
dct_n=dct_n)
elif self.dataset == 'h3.6m_3d':
self.cartesian = False
self.node_n = 96
if val_categorise:
self.train_dataset = dict()
self.val_dataset = dict()
for act in self.acts_train:
self.train_dataset[act] = H36motion3D_pose(
path_to_data=self.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=0,
sample_rate=sample_rate,
dct_used=dct_n)
self.val_dataset[act] = H36motion3D_pose(
path_to_data=self.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=2,
sample_rate=sample_rate,
dct_used=dct_n)
else:
self.train_dataset = H36motion3D_pose(
path_to_data=self.data_dir,
actions=self.acts_train,
input_n=input_n,
output_n=output_n,
split=0,
sample_rate=sample_rate,
dct_used=dct_n)
self.val_dataset = H36motion3D_pose(path_to_data=self.data_dir,
actions=self.acts_train,
input_n=input_n,
output_n=output_n,
split=2,
sample_rate=sample_rate,
dct_used=dct_n)
if self.out_of_distribution:
self.OoD_val_dataset = H36motion3D_pose(
path_to_data=self.data_dir,
actions=self.acts_OoD,
input_n=input_n,
output_n=output_n,
split=2,
sample_rate=sample_rate,
dct_used=dct_n)
else:
self.OoD_val_dataset = None
self.test_dataset = dict()
for act in self.acts_test:
self.test_dataset[act] = H36motion3D_pose(
path_to_data=self.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=1,
sample_rate=sample_rate,
dct_used=dct_n)
else:
raise Exception("Dataset name ({}) is not valid!".format(dataset))
return self.out_of_distribution
def get_dct_and_sequences(self,
input_n,
output_n,
sample_rate,
dct_n,
out_of_distribution_action=None):
if out_of_distribution_action != None:
self.out_of_distribution = True
acts_train = data_utils.define_actions(out_of_distribution_action,
self.dataset,
out_of_distribution=False)
acts_OoD = data_utils.define_actions(out_of_distribution_action,
self.dataset,
out_of_distribution=True)
acts_test = data_utils.define_actions('all',
self.dataset,
out_of_distribution=False)
else:
self.out_of_distribution = False
acts_train = data_utils.define_actions('all',
self.dataset,
out_of_distribution=False)
acts_OoD = None
acts_test = data_utils.define_actions('all',
self.dataset,
out_of_distribution=False)
self.acts_test = acts_test
if self.dataset == 'h3.6m':
self.cartesian = False
self.node_n = 48
self.train_dataset = H36motion(path_to_data=self.data_dir,
actions=acts_train,
input_n=input_n,
output_n=output_n,
split=0,
sample_rate=sample_rate,
dct_n=dct_n)
self.data_std = self.train_dataset.data_std
self.data_mean = self.train_dataset.data_mean
self.dim_used = self.train_dataset.dim_used
self.val_dataset = H36motion(path_to_data=self.data_dir,
actions=acts_train,
input_n=input_n,
output_n=output_n,
split=2,
sample_rate=sample_rate,
data_mean=self.data_mean,
data_std=self.data_std,
dct_n=dct_n)
if self.out_of_distribution:
self.OoD_val_dataset = H36motion(path_to_data=self.data_dir,
actions=acts_OoD,
input_n=input_n,
output_n=output_n,
split=2,
sample_rate=sample_rate,
data_mean=self.data_mean,
data_std=self.data_std,
dct_n=dct_n)
else:
self.OoD_val_dataset = None
self.test_dataset = dict()
for act in acts_test:
self.test_dataset[act] = H36motion(path_to_data=self.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=1,
sample_rate=sample_rate,
data_mean=self.data_mean,
data_std=self.data_std,
dct_n=dct_n)
elif self.dataset == 'cmu_mocap':
self.cartesian = False
self.node_n = 64
self.train_dataset = CMU_Motion(path_to_data=self.data_dir,
actions=acts_train,
input_n=input_n,
output_n=output_n,
split=0,
dct_n=dct_n)
self.data_std = self.train_dataset.data_std
self.data_mean = self.train_dataset.data_mean
self.dim_used = self.train_dataset.dim_used
self.val_dataset = None
self.OoD_val_dataset = None
self.test_dataset = dict()
for act in acts_test:
self.test_dataset[act] = CMU_Motion(path_to_data=self.data_dir,
actions=[act],
input_n=input_n,
output_n=output_n,
split=1,
data_mean=self.data_mean,
data_std=self.data_std,
dim_used=self.dim_used,
dct_n=dct_n)
elif self.dataset == 'cmu_mocap_3d':
self.cartesian = True
self.node_n = 75
self.train_dataset = CMU_Motion3D(path_to_data=self.data_dir,
actions=acts_train,
input_n=input_n,
output_n=output_n,
split=0,
dct_n=dct_n)
self.data_std = self.train_dataset.data_std
self.data_mean = self.train_dataset.data_mean
self.dim_used = self.train_dataset.dim_used
self.val_dataset = None
self.OoD_val_dataset = None
self.test_dataset = dict()
for act in acts_test:
self.test_dataset[act] = CMU_Motion3D(
path_to_data=self.data_dir,
actions=[act],
input_n=input_n,
output_n=output_n,
split=1,
data_mean=self.data_mean,
data_std=self.data_std,
dim_used=self.dim_used,
dct_n=dct_n)
else:
raise Exception("Dataset name ({}) is not valid!".format(dataset))
return self.out_of_distribution
n_z=1)
print(">>> total params: {:.2f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
lr = 0.00003
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
if is_cuda:
model.cuda()
ckpt = torch.load(opt.ckpt)
start_epoch = ckpt['epoch']
err_best = ckpt['err']
lr_now = ckpt['lr']
model.load_state_dict(ckpt['state_dict'])
optimizer.load_state_dict(ckpt['optimizer'])
print(">>> loading data")
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion(path_to_data='h3.6m/dataset/',
actions=act,
input_n=10,
output_n=10,
split=1,
sample_rate=2)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
dim_used = test_dataset.dim_used
print(">>> data loaded !")
action='store_true',
help='toggle do degradation experiments')
parser.add_argument('--n_z',
type=int,
default=2,
help='Number of latent variables')
parser.add_argument('--algorithm',
type=str,
default="PCA",
help='choose PCA or UMAP')
parser.set_defaults(multi_dim_experiment=False)
parser.set_defaults(degradation_experiment=False)
opt = parser.parse_args()
acts = data_utils.define_actions("all")
train_dataset = H36motion3D_pose(path_to_data="h3.6m/dataset/",
actions=acts,
input_n=1,
output_n=1,
split=0,
sample_rate=2,
dct_used=2)
val_dataset = H36motion3D_pose(path_to_data="h3.6m/dataset/",
actions=acts,
input_n=1,
output_n=1,
split=2,
sample_rate=2,
dct_used=2)
def main(opt):
start_epoch = 0
err_best = 10000
lr_now = opt.lr
is_cuda = torch.cuda.is_available()
# define log csv file
script_name = os.path.basename(__file__).split('.')[0]
script_name = script_name + "_in{:d}_out{:d}_dctn{:d}".format(
opt.input_n, opt.output_n, opt.dct_n)
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
dct_n = opt.dct_n
sample_rate = opt.sample_rate
# 48 nodes for angle prediction
model = nnmodel.GCN(input_feature=dct_n,
hidden_feature=opt.linear_size,
p_dropout=opt.dropout,
num_stage=opt.num_stage,
node_n=48)
if is_cuda:
model.cuda()
print(">>> total params: {:.2f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
# continue from checkpoint
if opt.is_load:
model_path_len = 'checkpoint/test/ckpt_main_gcn_muti_att_best.pth.tar'
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
start_epoch = ckpt['epoch']
err_best = ckpt['err']
lr_now = ckpt['lr']
model.load_state_dict(ckpt['state_dict'])
optimizer.load_state_dict(ckpt['optimizer'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
# data loading
print(">>> loading data")
train_dataset = H36motion(path_to_data=opt.data_dir,
actions='all',
input_n=input_n,
output_n=output_n,
split=0,
sample_rate=sample_rate,
dct_n=dct_n)
data_std = train_dataset.data_std
data_mean = train_dataset.data_mean
val_dataset = H36motion(path_to_data=opt.data_dir,
actions='all',
input_n=input_n,
output_n=output_n,
split=2,
sample_rate=sample_rate,
data_mean=data_mean,
data_std=data_std,
dct_n=dct_n)
# load datasets for training
train_loader = DataLoader(dataset=train_dataset,
batch_size=opt.train_batch,
shuffle=True,
num_workers=opt.job,
pin_memory=True)
val_loader = DataLoader(dataset=val_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion(path_to_data=opt.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=1,
sample_rate=sample_rate,
data_mean=data_mean,
data_std=data_std,
dct_n=dct_n)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
print(">>> data loaded !")
print(">>> train data {}".format(train_dataset.__len__()))
print(">>> validation data {}".format(val_dataset.__len__()))
for epoch in range(start_epoch, opt.epochs):
if (epoch + 1) % opt.lr_decay == 0:
lr_now = utils.lr_decay(optimizer, lr_now, opt.lr_gamma)
print('==========================')
print('>>> epoch: {} | lr: {:.5f}'.format(epoch + 1, lr_now))
ret_log = np.array([epoch + 1])
head = np.array(['epoch'])
# per epoch
a = train_dataset.dim_used
def main(opt):
is_cuda = torch.cuda.is_available()
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
itera = 50
#model = nnmodel.GCN(input_feature=(input_n + output_n), hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=48)
model = AttModel.AttModelRef4(in_features=opt.in_features,
kernel_size=opt.kernel_size,
d_model=opt.d_model,
num_stage=opt.num_stage,
dct_n=opt.dct_n,
device=opt.device)
if is_cuda:
model.cuda()
model_path_len = '/home/costa/src/Transformer/checkpoint/trans_N6_last_pose_in50_out10_ks10_dctn20/ckpt_best.pth.tar'
model_path_len = '/home/costa/Desktop/ckpt_best.pth.tar'
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
err_best = ckpt['err']
start_epoch = ckpt['epoch']
model.load_state_dict(ckpt['state_dict'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
# data loading
print(">>> loading data")
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = datasets.Datasets(opt=opt,
actions=[act],
split=1,
itera=itera)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch_size,
shuffle=False,
pin_memory=True)
dim_used = test_dataset.dimensions_to_use
print(">>> data loaded !")
joint_to_ignore = np.array([16, 20, 23, 24, 28, 31])
index_to_ignore = np.concatenate(
(joint_to_ignore * 3, joint_to_ignore * 3 + 1,
joint_to_ignore * 3 + 2))
joint_equal = np.array([13, 19, 22, 13, 27, 30])
index_to_equal = np.concatenate(
(joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2))
model.eval()
fig = plt.figure()
ax = plt.gca(projection='3d')
for act in acts:
for i, all_seq in enumerate(test_data[act]):
inputs = Variable(all_seq[:, :opt.input_n, dim_used]).float()
all_seq = Variable(all_seq).float()
if is_cuda:
inputs = inputs.cuda()
all_seq = all_seq.cuda()
outputs = model(inputs, opt.output_n, opt.input_n, itera=itera)
n, seq_len, dim_full_len = all_seq.data.shape
dim_used_len = len(dim_used)
'''
_, idct_m = data_utils.get_dct_matrix(seq_len)
if is_cuda:
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
else:
idct_m = Variable(torch.from_numpy(idct_m)).float()
outputs_t = outputs.view(-1, seq_len).transpose(0, 1)
outputs_exp = torch.matmul(idct_m, outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2)
p3d_out = p3d_h36.clone()[:, in_n:in_n + out_n]
p3d_out[:, :, dim_used] = p3d_out_all[:, seq_in:, 0]
p3d_out[:, :, index_to_ignore] = p3d_out[:, :, index_to_equal]
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs_exp
'''
pred = all_seq.clone()
pred[:, opt.input_n:opt.input_n + opt.output_n * itera,
dim_used] = outputs[:, opt.kernel_size:, 0]
pred[:, opt.input_n:opt.input_n + opt.output_n * itera,
index_to_ignore] = pred[:, opt.input_n:opt.input_n +
opt.output_n * itera, index_to_equal]
targ = all_seq
pred = pred.cpu().data.numpy()
targ = targ.cpu().data.numpy()
for k in range(8):
plt.cla()
figure_title = "action:{}, seq:{},".format(act, (k + 1))
viz.plot_predictions_from_3d(
targ[k, opt.input_n:opt.input_n + opt.output_n * itera, :],
pred[k, opt.input_n:opt.input_n + opt.output_n * itera, :],
fig, ax, figure_title)
plt.pause(1)
def main(opt):
is_cuda = torch.cuda.is_available()
desired_acts = ['eating', 'posing', 'sitting', 'posing', 'walkingdog']
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
dct_n = opt.dct_n
#calculate stepsize for auto regression based on input fames and DCT coefficients
stepsize = dct_n - input_n
sample_rate = opt.sample_rate
model = nnmodel.GCN(input_feature=(input_n + stepsize),
hidden_feature=opt.linear_size,
p_dropout=opt.dropout,
num_stage=opt.num_stage,
node_n=48)
if is_cuda:
model.cuda()
model_path_len = "checkpoint/pretrained/h36m_in{}_out{}_dctn{}.pth.tar".format(
input_n, stepsize, dct_n)
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
err_best = ckpt['err']
start_epoch = ckpt['epoch']
model.load_state_dict(ckpt['state_dict'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
# data loading
print(">>> loading data")
acts = data_utils.define_actions('all')
test_data = dict()
for act in acts:
test_dataset = H36motion(path_to_data=opt.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
dct_n=dct_n,
split=1,
sample_rate=sample_rate)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
dim_used = test_dataset.dim_used
print(">>> data loaded !")
model.eval()
fig = plt.figure()
ax = plt.gca(projection='3d')
#calculate no of iterations in auto regression to perform
iterations = int(output_n / stepsize)
print('iterations: {}'.format(iterations))
for act in acts:
for i, (_, targets, all_seq) in enumerate(test_data[act]):
all_seq = Variable(all_seq).float()
dim_used_len = len(dim_used)
if is_cuda:
all_seq = all_seq.cuda()
dct_m_in, _ = data_utils.get_dct_matrix(dct_n)
dct_m_in = Variable(torch.from_numpy(dct_m_in)).float().cuda()
_, idct_m = data_utils.get_dct_matrix(dct_n)
idct_m = Variable(torch.from_numpy(idct_m)).float().cuda()
targ_expmap = all_seq.cpu().data.numpy()
y_hat = None
#Auto regression
for idx in range(iterations):
#start index of the input sequence
start = input_n + idx * stepsize
#end index of the input sequence
stop = start + stepsize
if y_hat is None:
#slice the sequence of length = (input_n + output_n) in iteration 1
input_seq = all_seq[:, :dct_n, dim_used]
else:
#stack output from prev iteration and next frames to form the next input seq
input_seq = torch.cat(
(y_hat, all_seq[:, start:stop, dim_used]), 1)
#calculate DCT of the input seq
input_dct_seq = torch.matmul(dct_m_in,
input_seq).transpose(1, 2)
if is_cuda:
input_dct_seq = input_dct_seq.cuda()
y = model(input_dct_seq)
y_t = y.view(-1, dct_n).transpose(0, 1)
y_exp = torch.matmul(idct_m,
y_t).transpose(0, 1).contiguous().view(
-1, dim_used_len,
dct_n).transpose(1, 2)
y_hat = y_exp[:, stepsize:, :]
#accumulate the output frames in a single tensor
if idx == 0:
outputs = y_exp
else:
outputs = torch.cat((outputs, y_exp[:, input_n:, :]), 1)
pred_expmap = all_seq.clone()
dim_used = np.array(dim_used)
pred_expmap[:, :, dim_used] = outputs
pred_expmap = pred_expmap.cpu().data.numpy()
#calculate loss and save to a file for later use
#save_loss_file(act, pred_expmap, targ_expmap, input_n, output_n)
if act in desired_acts:
for k in range(8):
plt.cla()
figure_title = "action:{}, seq:{},".format(act, (k + 1))
viz.plot_predictions(targ_expmap[k, :, :],
pred_expmap[k, :, :], fig, ax,
figure_title)
plt.pause(1)
try:
os.makedirs(opt.save_dir)
except OSError:
pass
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%Y/%m/%d %H:%M:%S',
filename=os.path.join(opt.save_dir, 'train_test.log'), level=logging.INFO)
logging.info('======================================================')
# load model
model['st_gcn'] = Model(opt).cuda()
model['post_refine'] = post_refine(opt).cuda()
dataset = Human36mDataset('data/data_3d_h36m.npz', opt)
actions = define_actions(opt.actions)
if opt.pro_train:
train_data = Fusion(opt=opt, train=True, dataset=dataset)
train_dataloader = DataLoader(train_data, batch_size=256, shuffle=True, num_workers=8, pin_memory=False)
if opt.pro_test:
test_data = Fusion(opt=opt, train=False, dataset=dataset)
test_dataloader = DataLoader(test_data, batch_size=256, shuffle=False, num_workers=8, pin_memory=False)
# 3. set optimizer
all_param = []
for i_model in model:
all_param += list(model[i_model].parameters())
if opt.optimizer == 'SGD':
optimizer_all = optim.SGD(all_param, lr=opt.learning_rate,
momentum=0.9, nesterov=True, weight_decay=opt.weight_decay)
def main(opt):
start_epoch = 0
err_best = 10000
lr_now = opt.lr
is_cuda = torch.cuda.is_available()
opt.is_load = True
# define log csv file
script_name = os.path.basename(__file__).split('.')[0]
script_name = script_name + "_in{:d}_out{:d}_dctn{:d}".format(
opt.input_n, opt.output_n, opt.dct_n)
desired_acts = ['eating', 'walkingdog']
# create model
print(">>> creating model")
input_n = opt.input_n
output_n = opt.output_n
dct_n = opt.dct_n
# calculate stepsize for auto regression based on input fames and DCT coefficients
stepsize = dct_n - input_n
sample_rate = opt.sample_rate
# 48 nodes for angle prediction
model = nnmodel.GCN(input_feature=dct_n,
hidden_feature=opt.linear_size,
p_dropout=opt.dropout,
num_stage=opt.num_stage,
node_n=48)
if is_cuda:
model.cuda()
print(">>> total params: {:.2f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
# continue from checkpoint
if opt.is_load:
model_path_len = "checkpoint/logs/ckpt_main_in{}_out{}_dctn{}_best.pth.tar".format(
input_n, stepsize, dct_n)
print(">>> loading ckpt len from '{}'".format(model_path_len))
if is_cuda:
ckpt = torch.load(model_path_len)
else:
ckpt = torch.load(model_path_len, map_location='cpu')
start_epoch = ckpt['epoch']
err_best = ckpt['err']
lr_now = ckpt['lr']
model.load_state_dict(ckpt['state_dict'])
optimizer.load_state_dict(ckpt['optimizer'])
print(">>> ckpt len loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
# data loading
print(">>> loading data")
# train_dataset = H36motion(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n,
# split=0, sample_rate=sample_rate, dct_n=dct_n)
# data_std = train_dataset.data_std
# data_mean = train_dataset.data_mean
dim_used = [
6, 7, 8, 9, 12, 13, 14, 15, 21, 22, 23, 24, 27, 28, 29, 30, 36, 37, 38,
39, 40, 41, 42, 43, 44, 45, 46, 47, 51, 52, 53, 54, 55, 56, 57, 60, 61,
62, 75, 76, 77, 78, 79, 80, 81, 84, 85, 86
]
# val_dataset = H36motion(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n,
# split=2, sample_rate=sample_rate, data_mean=data_mean, data_std=data_std, dct_n=dct_n)
# # load dadasets for training
# train_loader = DataLoader(
# dataset=train_dataset,
# batch_size=opt.train_batch,
# shuffle=True,
# num_workers=opt.job,
# pin_memory=True)
# val_loader = DataLoader(
# dataset=val_dataset,
# batch_size=opt.test_batch,
# shuffle=False,
# num_workers=opt.job,
# pin_memory=True)
acts = data_utils.define_actions('all')
print(acts)
test_data = dict()
for act in acts:
test_dataset = H36motion(path_to_data=opt.data_dir,
actions=act,
input_n=input_n,
output_n=output_n,
split=1,
sample_rate=sample_rate,
dct_n=dct_n)
test_data[act] = DataLoader(dataset=test_dataset,
batch_size=opt.test_batch,
shuffle=False,
num_workers=opt.job,
pin_memory=True)
print(">>> data loaded !")
# print(">>> train data {}".format(train_dataset.__len__()))
# print(">>> validation data {}".format(val_dataset.__len__()))
# for epoch in range(start_epoch, opt.epochs):
#
# if (epoch + 1) % opt.lr_decay == 0:
# lr_now = utils.lr_decay(optimizer, lr_now, opt.lr_gamma)
# print('==========================')
# print('>>> epoch: {} | lr: {:.5f}'.format(epoch + 1, lr_now))
ret_log = np.array([start_epoch])
head = np.array(['epoch'])
# per epoch
# lr_now, t_l, t_e, t_3d = train(train_loader, model, optimizer, input_n=input_n,
# lr_now=lr_now, max_norm=opt.max_norm, is_cuda=is_cuda,
# dim_used=train_dataset.dim_used, dct_n=dct_n)
# ret_log = np.append(ret_log, [lr_now, t_l, t_e, t_3d])
# head = np.append(head, ['lr', 't_l', 't_e', 't_3d'])
#
# v_e, v_3d = val(val_loader, model, input_n=input_n, is_cuda=is_cuda, dim_used=train_dataset.dim_used,
# dct_n=dct_n)
#
# ret_log = np.append(ret_log, [v_e, v_3d])
# head = np.append(head, ['v_e', 'v_3d'])
test_3d_temp = np.array([])
test_3d_head = np.array([])
for act in acts:
test_e, test_3d = test(test_data[act],
model,
stepsize,
input_n=input_n,
output_n=output_n,
is_cuda=is_cuda,
dim_used=dim_used,
dct_n=dct_n)
ret_log = np.append(ret_log, test_e)
test_3d_temp = np.append(test_3d_temp, test_3d)
test_3d_head = np.append(
test_3d_head,
[act + '3d80', act + '3d160', act + '3d320', act + '3d400'])
head = np.append(head,
[act + '80', act + '160', act + '320', act + '400'])
if output_n > 10:
if output_n == 25:
head = np.append(head, [act + '560', act + '1000'])
test_3d_head = np.append(test_3d_head,
[act + '3d560', act + '3d1000'])
if output_n == 50:
head = np.append(head,
[act + '560', act + '1000', act + '2000'])
test_3d_head = np.append(
test_3d_head,
[act + '3d560', act + '3d1000', act + '3d2000'])
if output_n == 100:
head = np.append(
head,
[act + '560', act + '1000', act + '2000', act + '4000'])
test_3d_head = np.append(test_3d_head, [
act + '3d560', act + '3d1000', act + '3d2000',
act + '3d4000'
])
ret_log = np.append(ret_log, test_3d_temp)
head = np.append(head, test_3d_head)
# update log file and save checkpoint
df = pd.DataFrame(np.expand_dims(ret_log, axis=0))
# if epoch == start_epoch:
df.to_csv(opt.ckpt + '/' + script_name + '.csv', header=head, index=False)
