def __init__(self, path_to_data, actions, input_n=10, output_n=10, split=0, data_mean=0, data_std=0, dim_used=0, dct_n=15): self.path_to_data = path_to_data self.split = split actions = data_utils.define_actions_cmu(actions) # actions = ['walking'] if split == 0: path_to_data = path_to_data + '/train/' is_test = False else: path_to_data = path_to_data + '/test/' is_test = True all_seqs, dim_ignore, dim_use, data_mean, data_std = data_utils.load_data_cmu_3d( path_to_data, actions, input_n, output_n, data_std=data_std, data_mean=data_mean, is_test=is_test) if not is_test: dim_used = dim_use self.all_seqs = all_seqs self.dim_used = dim_used all_seqs = all_seqs[:, :, dim_used] all_seqs = all_seqs.transpose(0, 2, 1) all_seqs = all_seqs.reshape(-1, input_n + output_n) all_seqs = all_seqs.transpose() dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n) dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n) pad_idx = np.repeat([input_n - 1], output_n) i_idx = np.append(np.arange(0, input_n), pad_idx) input_dct_seq = np.matmul(dct_m_in[0:dct_n, :], all_seqs[i_idx, :]) input_dct_seq = input_dct_seq.transpose().reshape( -1, len(dim_used), dct_n) # input_dct_seq = input_dct_seq.reshape(-1, len(dim_used) * (input_n + output_n)) output_dct_seq = np.matmul(dct_m_out[0:dct_n, :], all_seqs) output_dct_seq = output_dct_seq.transpose().reshape( -1, len(dim_used), dct_n) self.input_dct_seq = input_dct_seq self.output_dct_seq = output_dct_seq self.data_mean = data_mean self.data_std = data_std
def __init__(self, path_to_data, actions, input_n=20, output_n=10, dct_used=15, split=0, sample_rate=2): """ :param path_to_data: :param actions: :param input_n: :param output_n: :param dct_used: :param split: 0 train, 1 testing, 2 validation :param sample_rate: """ self.path_to_data = path_to_data self.split = split self.dct_used = dct_used subs = np.array([[1, 6, 7, 8, 9], [5], [11]]) acts = data_utils.define_actions(actions) # subs = np.array([[1], [5], [11]]) # acts = ['walking'] subjs = subs[split] all_seqs, dim_ignore, dim_used = data_utils.load_data_3d( path_to_data, subjs, acts, sample_rate, input_n + output_n) self.all_seqs = all_seqs self.dim_used = dim_used all_seqs = all_seqs[:, :, dim_used] all_seqs = all_seqs.transpose(0, 2, 1) all_seqs = all_seqs.reshape(-1, input_n + output_n) all_seqs = all_seqs.transpose() dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n) dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n) pad_idx = np.repeat([input_n - 1], output_n) i_idx = np.append(np.arange(0, input_n), pad_idx) input_dct_seq = np.matmul(dct_m_in[0:dct_used, :], all_seqs[i_idx, :]) input_dct_seq = input_dct_seq.transpose().reshape( [-1, len(dim_used), dct_used]) # input_dct_seq = input_dct_seq.reshape(-1, len(dim_used) * dct_used) output_dct_seq = np.matmul(dct_m_out[0:dct_used, :], all_seqs) output_dct_seq = output_dct_seq.transpose().reshape( [-1, len(dim_used), dct_used]) # output_dct_seq = output_dct_seq.reshape(-1, len(dim_used) * dct_used) self.input_dct_seq = input_dct_seq self.output_dct_seq = output_dct_seq
def mpjpe_error_p3d(outputs, all_seq, dct_n, dim_used): """ :param outputs:n*66*dct_n :param all_seq: :param dct_n: :param dim_used: :return: """ n, seq_len, dim_full_len = all_seq.data.shape dim_used = np.array(dim_used) dim_used_len = len(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) outputs_p3d = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2) pred_3d = outputs_p3d.contiguous().view(-1, dim_used_len).view(-1, 3) targ_3d = all_seq[:, :, dim_used].contiguous().view(-1, dim_used_len).view(-1, 3) mean_3d_err = torch.mean(torch.norm(pred_3d - targ_3d, 2, 1)) return mean_3d_err
def mpjpe_error(outputs, all_seq, input_n, dim_used, dct_n): """ :param outputs: :param all_seq: :param input_n: :param dim_used: :param data_mean: :param data_std: :return: """ n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2) pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs_exp pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(-1, dim_full_len).clone() targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(-1, dim_full_len) pred_expmap[:, 0:6] = 0 targ_expmap[:, 0:6] = 0 targ_p3d = data_utils.expmap2xyz_torch(targ_expmap).view(-1, 3) pred_p3d = data_utils.expmap2xyz_torch(pred_expmap).view(-1, 3) mean_3d_err = torch.mean(torch.norm(targ_p3d - pred_p3d, 2, 1)) return mean_3d_err
def test(train_loader, model, input_n=20, dct_n=20, dim_used=[], output_n=50, is_cuda=False): N = 0 if output_n == 15: eval_frame = [2, 5, 8, 11, 14] elif output_n == 30: eval_frame = [2, 5, 8, 11, 14, 17, 20, 23, 26, 29] t_err = np.zeros(len(eval_frame)) model.eval() st = time.time() bar = Bar('>>>', fill='>', max=len(train_loader)) for i, (inputs, targets, all_seq) in enumerate(train_loader): bt = time.time() if is_cuda: inputs = Variable(inputs.cuda()).float() # targets = Variable(targets.cuda(async=True)).float() all_seq = Variable(all_seq.cuda(async=True)).float() else: inputs = Variable(inputs).float() # targets = Variable(targets).float() all_seq = Variable(all_seq).float() outputs = model(inputs) n, seq_len, dim_full_len = all_seq.data.shape _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose( 0, 1).contiguous().view(-1, dim_full_len - 3, seq_len).transpose(1, 2) pred_exp = all_seq.clone() pred_exp[:, :, dim_used] = outputs_exp pred_exp = pred_exp.contiguous().view(n, seq_len, -1)[:, input_n:, :] targ_exp = all_seq.contiguous().view(n, seq_len, -1)[:, input_n:, :] for k in np.arange(0, len(eval_frame)): j = eval_frame[k] t_err[k] += torch.mean( torch.norm(targ_exp[:, j, :] - pred_exp[:, j, :], p=2, dim=1)).cpu().data.numpy()[0] * n # update the training loss N += n bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format( i, len(train_loader), time.time() - bt, time.time() - st) bar.next() bar.finish() return t_err / N
def test(train_loader, model, input_n=20, output_n=50, is_cuda=False, dim_used=[], dct_n=15): N = 0 t_l = 0 if output_n == 25: eval_frame = [1, 3, 7, 9, 13, 24] elif output_n == 10: eval_frame = [1, 3, 7, 9] t_3d = np.zeros(len(eval_frame)) model.eval() st = time.time() bar = Bar('>>>', fill='>', max=len(train_loader)) for i, (inputs, targets, all_seq) in enumerate(train_loader): bt = time.time() if is_cuda: inputs = Variable(inputs.cuda()).float() all_seq = Variable(all_seq.cuda(async=True)).float() outputs = model(inputs) n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) outputs_3d = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2) pred_3d = all_seq.clone() dim_used = np.array(dim_used) # joints at same loc joint_to_ignore = np.array([16, 20, 23, 24, 28, 31]) index_to_ignore = np.concatenate((joint_to_ignore * 3, joint_to_ignore * 3 + 1, joint_to_ignore * 3 + 2)) joint_equal = np.array([13, 19, 22, 13, 27, 30]) index_to_equal = np.concatenate((joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2)) pred_3d[:, :, dim_used] = outputs_3d pred_3d[:, :, index_to_ignore] = pred_3d[:, :, index_to_equal] pred_p3d = pred_3d.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :] targ_p3d = all_seq.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :] for k in np.arange(0, len(eval_frame)): j = eval_frame[k] t_3d[k] += torch.mean(torch.norm( targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2, 1)).cpu().data.numpy() * n N += n bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(i+1, len(train_loader), time.time() - bt, time.time() - st) bar.next() bar.finish() return t_l / N, t_3d / N
def train(train_loader, model, optimizer, lr_now=None, max_norm=True, is_cuda=False, dim_used=[], dct_n=15): t_l = utils.AccumLoss() t_e = utils.AccumLoss() t_3d = utils.AccumLoss() model.train() st = time.time() bar = Bar('>>>', fill='>', max=len(train_loader)) for i, (inputs, targets, all_seq) in enumerate(train_loader): bt = time.time() if is_cuda: inputs = Variable(inputs.cuda()).float() all_seq = Variable(all_seq.cuda(async=True)).float() outputs = model(inputs) n, seq_len, dim_full_len = all_seq.data.shape dim_used = np.array(dim_used) dim_used_len = len(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) outputs_p3d = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous() outputs_p3d = outputs_p3d.view(-1, dim_used_len, seq_len).transpose( 1, 2).contiguous().view(-1, 3) targ_p3d = all_seq[:, :, dim_used].clone().contiguous().view(-1, 3) loss = torch.mean(torch.norm(targ_p3d - outputs_p3d, 2, 1)) # calculate loss and backward optimizer.zero_grad() loss.backward() if max_norm: nn.utils.clip_grad_norm(model.parameters(), max_norm=1) optimizer.step() t_l.update(loss.cpu().data.numpy()[0] * n, n) bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format( i + 1, len(train_loader), time.time() - bt, time.time() - st) bar.next() bar.finish() return lr_now, t_l.avg, t_e.avg, t_3d.avg
def mpjpe_error_3dpw(outputs, all_seq, dct_n, dim_used): n, seq_len, dim_full_len = all_seq.data.shape _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) outputs_exp = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose( 0, 1).contiguous().view(-1, dim_full_len - 3, seq_len).transpose(1, 2) pred_3d = all_seq.clone() pred_3d[:, :, dim_used] = outputs_exp pred_3d = pred_3d.contiguous().view(-1, dim_full_len).view(-1, 3) targ_3d = all_seq.contiguous().view(-1, dim_full_len).view(-1, 3) mean_3d_err = torch.mean(torch.norm(pred_3d - targ_3d, 2, 1)) return mean_3d_err
def dct(model, in_tensor, inverse=False): """ :param model: the model instance :param in_tensor: tensor to have it's temproal component DCT'ed assumes that the last dim of in_tenosr is the temporal one :param inverse: toggle to inverse the transformation (boole) """ t_n = in_tensor.size()[-1] dct_matrix, idct_matrix = data_utils.get_dct_matrix(t_n) if not inverse: dct_matrix = torch.from_numpy(dct_matrix).to(model.device).float() tensor_dct = torch.matmul(in_tensor, dct_matrix) return tensor_dct else: idct_matrix = torch.from_numpy(idct_matrix).to(model.device).float() tensor_idct = torch.matmul(in_tensor, idct_matrix) return tensor_idct
def euler_error(outputs, all_seq, input_n, dim_used, dct_n): """ :param outputs: :param all_seq: :param input_n: :param dim_used: :return: """ n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view( -1, dim_used_len, seq_len).transpose(1, 2) pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs_exp pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(-1, dim_full_len) targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view( -1, dim_full_len) # pred_expmap[:, 0:6] = 0 # targ_expmap[:, 0:6] = 0 pred_expmap = pred_expmap.view(-1, 3) targ_expmap = targ_expmap.view(-1, 3) pred_eul = data_utils.rotmat2euler_torch( data_utils.expmap2rotmat_torch(pred_expmap)) pred_eul = pred_eul.view(-1, dim_full_len) targ_eul = data_utils.rotmat2euler_torch( data_utils.expmap2rotmat_torch(targ_expmap)) targ_eul = targ_eul.view(-1, dim_full_len) mean_errors = torch.mean(torch.norm(pred_eul - targ_eul, 2, 1)) return mean_errors
def sen_loss(outputs, all_seq, dim_used, dct_n, inputs, cartesian=False, lambda_ = 0.01, KL=None, reconstructions=None, log_var=None): """ :param outputs: N * (seq_len*dim_used_len) :param all_seq: N * seq_len * dim_full_len :param input_n: :param dim_used: :return: """ n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) dim_used = np.array(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) if cartesian: pred_cart = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous() pred_cart = pred_cart.view(-1, dim_used_len, seq_len).transpose(1, 2).contiguous().view(-1, 3) targ_cart= all_seq.clone()[:, :, dim_used].contiguous().view(-1, 3) joint_loss = torch.mean(torch.norm(targ_cart - pred_cart, 2, 1)) else: pred_expmap = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2) targ_expmap = all_seq.clone()[:, :, dim_used] joint_loss = torch.mean(torch.abs(pred_expmap - targ_expmap)) if KL == None: neg_gauss_log_lik = torch.zeros(1) latent_loss = torch.zeros(1) loss = joint_loss else: latent_loss = torch.mean(KL) mse = torch.pow((reconstructions - inputs), 2) gauss_log_lik = -mse#0.5*(log_var + np.log(2*np.pi) + (mse/(1e-8 + torch.exp(log_var)))) neg_gauss_log_lik = -torch.mean(torch.sum(gauss_log_lik, axis=(1, 2))) loss = joint_loss + lambda_*(neg_gauss_log_lik + latent_loss) return loss, joint_loss, neg_gauss_log_lik, latent_loss
def sen_loss(outputs, all_seq, dim_used, dct_n): """ :param outputs: N * (seq_len*dim_used_len) :param all_seq: N * seq_len * dim_full_len :param input_n: :param dim_used: :return: """ n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) dim_used = np.array(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) pred_expmap = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view( -1, dim_used_len, seq_len).transpose(1, 2) targ_expmap = all_seq.clone()[:, :, dim_used] loss = torch.mean( torch.sum(torch.abs(pred_expmap - targ_expmap), dim=2).view(-1)) return loss
def main(opt): is_cuda = torch.cuda.is_available() desired_acts = ['eating', 'posing', 'sitting', 'posing', 'walkingdog'] # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n dct_n = opt.dct_n #calculate stepsize for auto regression based on input fames and DCT coefficients stepsize = dct_n - input_n sample_rate = opt.sample_rate model = nnmodel.GCN(input_feature=(input_n + stepsize), hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=48) if is_cuda: model.cuda() model_path_len = "checkpoint/pretrained/h36m_in{}_out{}_dctn{}.pth.tar".format( input_n, stepsize, dct_n) print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') err_best = ckpt['err'] start_epoch = ckpt['epoch'] model.load_state_dict(ckpt['state_dict']) print(">>> ckpt len loaded (epoch: {} | err: {})".format( start_epoch, err_best)) # data loading print(">>> loading data") acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, dct_n=dct_n, split=1, sample_rate=sample_rate) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) dim_used = test_dataset.dim_used print(">>> data loaded !") model.eval() fig = plt.figure() ax = plt.gca(projection='3d') #calculate no of iterations in auto regression to perform iterations = int(output_n / stepsize) print('iterations: {}'.format(iterations)) for act in acts: for i, (_, targets, all_seq) in enumerate(test_data[act]): all_seq = Variable(all_seq).float() dim_used_len = len(dim_used) if is_cuda: all_seq = all_seq.cuda() dct_m_in, _ = data_utils.get_dct_matrix(dct_n) dct_m_in = Variable(torch.from_numpy(dct_m_in)).float().cuda() _, idct_m = data_utils.get_dct_matrix(dct_n) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() targ_expmap = all_seq.cpu().data.numpy() y_hat = None #Auto regression for idx in range(iterations): #start index of the input sequence start = input_n + idx * stepsize #end index of the input sequence stop = start + stepsize if y_hat is None: #slice the sequence of length = (input_n + output_n) in iteration 1 input_seq = all_seq[:, :dct_n, dim_used] else: #stack output from prev iteration and next frames to form the next input seq input_seq = torch.cat( (y_hat, all_seq[:, start:stop, dim_used]), 1) #calculate DCT of the input seq input_dct_seq = torch.matmul(dct_m_in, input_seq).transpose(1, 2) if is_cuda: input_dct_seq = input_dct_seq.cuda() y = model(input_dct_seq) y_t = y.view(-1, dct_n).transpose(0, 1) y_exp = torch.matmul(idct_m, y_t).transpose(0, 1).contiguous().view( -1, dim_used_len, dct_n).transpose(1, 2) y_hat = y_exp[:, stepsize:, :] #accumulate the output frames in a single tensor if idx == 0: outputs = y_exp else: outputs = torch.cat((outputs, y_exp[:, input_n:, :]), 1) pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs pred_expmap = pred_expmap.cpu().data.numpy() #calculate loss and save to a file for later use #save_loss_file(act, pred_expmap, targ_expmap, input_n, output_n) if act in desired_acts: for k in range(8): plt.cla() figure_title = "action:{}, seq:{},".format(act, (k + 1)) viz.plot_predictions(targ_expmap[k, :, :], pred_expmap[k, :, :], fig, ax, figure_title) plt.pause(1)
def main(opt): is_cuda = torch.cuda.is_available() # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n sample_rate = opt.sample_rate model = nnmodel.GCN(input_feature=(input_n + output_n), hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=48) if is_cuda: model.cuda() model_path_len = './checkpoint/pretrained/h36m_in10_out25.pth.tar' print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') err_best = ckpt['err'] start_epoch = ckpt['epoch'] model.load_state_dict(ckpt['state_dict']) print(">>> ckpt len loaded (epoch: {} | err: {})".format(start_epoch, err_best)) # data loading print(">>> loading data") acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate) test_data[act] = DataLoader( dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) dim_used = test_dataset.dim_used print(">>> data loaded !") model.eval() fig = plt.figure() ax = plt.gca(projection='3d') for act in acts: for i, (inputs, targets, all_seq) in enumerate(test_data[act]): inputs = Variable(inputs).float() all_seq = Variable(all_seq).float() if is_cuda: inputs = inputs.cuda() all_seq = all_seq.cuda() outputs = model(inputs) n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, seq_len).transpose(0, 1) outputs_exp = torch.matmul(idct_m, outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2) pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs_exp targ_expmap = all_seq pred_expmap = pred_expmap.cpu().data.numpy() targ_expmap = targ_expmap.cpu().data.numpy() for k in range(8): plt.cla() figure_title = "action:{}, seq:{},".format(act, (k + 1)) viz.plot_predictions(targ_expmap[k, :, :], pred_expmap[k, :, :], fig, ax, figure_title) plt.pause(1)
def test(train_loader, model, input_n=20, output_n=50, is_cuda=False, dim_used=[], dct_n=20): N = 0 if output_n == 25: eval_frame = [1, 3, 7, 9, 13, 24] elif output_n == 10: eval_frame = [1, 3, 7, 9] t_e = np.zeros(len(eval_frame)) t_3d = np.zeros(len(eval_frame)) model.eval() st = time.time() bar = Bar('>>>', fill='>', max=len(train_loader)) for i, (inputs, targets, all_seq) in enumerate(train_loader): bt = time.time() if is_cuda: inputs = Variable(inputs.cuda()).float() all_seq = Variable(all_seq.cuda(async=True)).float() outputs = model(inputs) n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) all_seq[:, :, 0:6] = 0 _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, seq_len).transpose(0, 1) outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view \ (-1, dim_used_len, seq_len).transpose(1, 2) pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs_exp pred_expmap = pred_expmap[:, input_n:, :].contiguous().view( -1, dim_full_len) targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view( -1, dim_full_len) pred_expmap = pred_expmap.view(-1, 3) targ_expmap = targ_expmap.view(-1, 3) pred_eul = data_utils.rotmat2euler_torch( data_utils.expmap2rotmat_torch(pred_expmap)) pred_eul = pred_eul.view(-1, dim_full_len).view( -1, output_n, dim_full_len) # [:, :, dim_used] targ_eul = data_utils.rotmat2euler_torch( data_utils.expmap2rotmat_torch(targ_expmap)) targ_eul = targ_eul.view(-1, dim_full_len).view( -1, output_n, dim_full_len) # [:, :, dim_used] targ_p3d = data_utils.expmap2xyz_torch_cmu( targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3) pred_p3d = data_utils.expmap2xyz_torch_cmu( pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3) for k in np.arange(0, len(eval_frame)): j = eval_frame[k] t_e[k] += torch.mean( torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2, 1)).cpu().data.numpy()[0] * n t_3d[k] += torch.mean( torch.norm( targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2, 1)).cpu().data.numpy()[0] * n # update the training loss N += n bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format( i + 1, len(train_loader), time.time() - bt, time.time() - st) bar.next() bar.finish() return t_e / N, t_3d / N
def __init__(self, path_to_data, actions, input_n=10, output_n=10, dct_n=20, split=0, sample_rate=2, data_mean=0, data_std=0): """ read h36m data to get the dct coefficients. :param path_to_data: :param actions: actions to read :param input_n: past frame length :param output_n: future frame length :param dct_n: number of dct coeff. used :param split: 0 train, 1 test, 2 validation :param sample_rate: 2 :param data_mean: mean of expmap :param data_std: standard deviation of expmap """ self.path_to_data = path_to_data self.split = split subs = np.array([[1, 6, 7, 8, 9], [5], [11]]) acts = data_utils.define_actions(actions) # subs = np.array([[1], [5], [11]]) # acts = ['walking'] subjs = subs[split] all_seqs, dim_ignore, dim_use, data_mean, data_std = data_utils.load_data( path_to_data, subjs, acts, sample_rate, input_n + output_n, data_mean=data_mean, data_std=data_std, input_n=input_n) self.data_mean = data_mean self.data_std = data_std # first 6 elements are global translation and global rotation dim_used = dim_use[6:] self.all_seqs = all_seqs self.dim_used = dim_used all_seqs = all_seqs[:, :, dim_used] all_seqs = all_seqs.transpose(0, 2, 1) all_seqs = all_seqs.reshape(-1, input_n + output_n) all_seqs = all_seqs.transpose() dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n) dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n) # padding the observed sequence so that it has the same length as observed + future sequence pad_idx = np.repeat([input_n - 1], output_n) i_idx = np.append(np.arange(0, input_n), pad_idx) input_dct_seq = np.matmul(dct_m_in[:dct_n, :], all_seqs[i_idx, :]) input_dct_seq = input_dct_seq.transpose().reshape( [-1, len(dim_used), dct_n]) output_dct_seq = np.matmul(dct_m_out[:dct_n], all_seqs) output_dct_seq = output_dct_seq.transpose().reshape( [-1, len(dim_used), dct_n]) self.input_dct_seq = input_dct_seq self.output_dct_seq = output_dct_seq
def __init__(self, path_to_data, input_n=20, output_n=10, dct_n=15, split=0): """ :param path_to_data: :param input_n: :param output_n: :param dct_n: :param split: """ self.path_to_data = path_to_data self.split = split self.dct_n = dct_n # since baselines (http://arxiv.org/abs/1805.00655.pdf and https://arxiv.org/pdf/1705.02445.pdf) # use observed 50 frames but our method use 10 past frames in order to make sure all methods are evaluated # on same sequences, we first crop the sequence with 50 past frames and then use the last 10 frame as input if split == 1: their_input_n = 50 else: their_input_n = input_n seq_len = their_input_n + output_n if split == 0: self.data_path = path_to_data + '/train/' elif split == 1: self.data_path = path_to_data + '/test/' elif split == 2: self.data_path = path_to_data + '/validation/' all_seqs = [] files = [] for (dirpath, dirnames, filenames) in walk(self.data_path): files.extend(filenames) for f in files: with open(self.data_path + f, 'rb') as f: data = pkl.load(f, encoding='latin1') joint_pos = data['jointPositions'] for i in range(len(joint_pos)): seqs = joint_pos[i] seqs = seqs - seqs[:, 0:3].repeat(24, axis=0).reshape( -1, 72) n_frames = seqs.shape[0] fs = np.arange(0, n_frames - seq_len + 1) fs_sel = fs for j in np.arange(seq_len - 1): fs_sel = np.vstack((fs_sel, fs + j + 1)) fs_sel = fs_sel.transpose() seq_sel = seqs[fs_sel, :] if len(all_seqs) == 0: all_seqs = seq_sel else: all_seqs = np.concatenate((all_seqs, seq_sel), axis=0) self.all_seqs = all_seqs[:, (their_input_n - input_n):, :] self.dim_used = np.array(range(3, all_seqs.shape[2])) all_seqs = all_seqs[:, (their_input_n - input_n):, 3:] n, seq_len, dim_len = all_seqs.shape all_seqs = all_seqs.transpose(0, 2, 1) all_seqs = all_seqs.reshape(-1, input_n + output_n) all_seqs = all_seqs.transpose() dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n) dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n) pad_idx = np.repeat([input_n - 1], output_n) i_idx = np.append(np.arange(0, input_n), pad_idx) input_dct_seq = np.matmul(dct_m_in[0:dct_n, :], all_seqs[i_idx, :]) input_dct_seq = input_dct_seq.transpose().reshape(-1, dim_len, dct_n) # input_dct_seq = input_dct_seq.reshape(-1, dim_len * dct_used) output_dct_seq = np.matmul(dct_m_out[0:dct_n, :], all_seqs) output_dct_seq = output_dct_seq.transpose().reshape(-1, dim_len, dct_n) # output_dct_seq = output_dct_seq.reshape(-1, dim_len * dct_used) self.input_dct_seq = input_dct_seq self.output_dct_seq = output_dct_seq
def main(opt): is_cuda = torch.cuda.is_available() # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n sample_rate = opt.sample_rate dct_n = opt.dct_n model = nnmodel.GCN(input_feature=dct_n, hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=66) if is_cuda: model.to('cuda' if torch.cuda.is_available else 'cpu') model_path_len = './checkpoint/pretrained/h36m3D_in10_out10_dctn15.pth.tar' print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') err_best = ckpt['err'] start_epoch = ckpt['epoch'] model.load_state_dict(ckpt['state_dict']) print(">>> ckpt len loaded (epoch: {} | err: {})".format( start_epoch, err_best)) # data loading print(">>> loading data") acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion3D(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate, dct_used=dct_n) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) dim_used = test_dataset.dim_used print(">>> data loaded !") model.eval() fig = plt.figure() ax = plt.gca(projection='3d') for act in acts: for i, (inputs, targets, all_seq) in enumerate(test_data[act]): inputs = Variable(inputs).float() all_seq = Variable(all_seq).float() if is_cuda: inputs = inputs.to( 'cuda' if torch.cuda.is_available else 'cpu') all_seq = all_seq.to( 'cuda' if torch.cuda.is_available else 'cpu') outputs = model(inputs) n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float() if is_cuda: idct_m = idct_m.to( 'cuda' if torch.cuda.is_available else 'cpu') outputs_t = outputs.view(-1, dct_n).transpose(0, 1) outputs_3d = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose( 0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2) pred_3d = all_seq.clone() dim_used = np.array(dim_used) # joints at same loc joint_to_ignore = np.array([16, 20, 23, 24, 28, 31]) index_to_ignore = np.concatenate( (joint_to_ignore * 3, joint_to_ignore * 3 + 1, joint_to_ignore * 3 + 2)) joint_equal = np.array([13, 19, 22, 13, 27, 30]) index_to_equal = np.concatenate( (joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2)) pred_3d[:, :, dim_used] = outputs_3d pred_3d[:, :, index_to_ignore] = pred_3d[:, :, index_to_equal] pred_p3d = pred_3d.contiguous().view( n, seq_len, -1, 3)[:, input_n:, :, :].cpu().data.numpy() targ_p3d = all_seq.contiguous().view( n, seq_len, -1, 3)[:, input_n:, :, :].cpu().data.numpy() for k in range(8): plt.cla() figure_title = "action:{}, seq:{},".format(act, (k + 1)) viz.plot_predictions_direct(targ_p3d[k], pred_p3d[k], fig, ax, figure_title) plt.pause(1)
def test(train_loader, model, stepsize, input_n=20, output_n=50, dct_n=20, is_cuda=False, dim_used=[]): N = 0 # t_l = 0 if output_n == 25: eval_frame = [1, 3, 7, 9, 13, 24] elif output_n == 50: eval_frame = [1, 3, 7, 9, 13, 24, 35] elif output_n == 100: eval_frame = [1, 3, 7, 9, 13, 24, 35, 49] elif output_n == 10: eval_frame = [1, 3, 7, 9] t_e = np.zeros(len(eval_frame)) t_3d = np.zeros(len(eval_frame)) model.eval() # calculate no of iterations in auto regression to perform iterations = int(output_n / stepsize) print('iterations: {}'.format(iterations)) st = time.time() bar = Bar('>>>', fill='>', max=len(train_loader)) for i, (inputs, targets, all_seq) in enumerate(train_loader): bt = time.time() all_seq = Variable(all_seq).float() dim_used_len = len(dim_used) if is_cuda: all_seq = all_seq.cuda() dct_m_in, _ = data_utils.get_dct_matrix(dct_n) dct_m_in = Variable(torch.from_numpy(dct_m_in)).float().cuda() _, idct_m = data_utils.get_dct_matrix(dct_n) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() n, seq_len, dim_full_len = all_seq.data.shape y_hat = None # Auto regression for idx in range(iterations): # start index of the input sequence start = input_n + idx * stepsize # end index of the input sequence stop = start + stepsize if y_hat is None: # slice the sequence of length = (input_n + output_n) in iteration 1 input_seq = all_seq[:, :dct_n, dim_used] else: # stack output from prev iteration and next frames to form the next input seq input_seq = torch.cat( (y_hat, all_seq[:, start:stop, dim_used]), 1) # calculate DCT of the input seq input_dct_seq = torch.matmul(dct_m_in, input_seq).transpose(1, 2) if is_cuda: input_dct_seq = input_dct_seq.cuda() y = model(input_dct_seq) y_t = y.view(-1, dct_n).transpose(0, 1) y_exp = torch.matmul(idct_m, y_t).transpose(0, 1).contiguous().view( -1, dim_used_len, dct_n).transpose(1, 2) y_hat = y_exp[:, stepsize:, :] # accumulate the output frames in a single tensor if idx == 0: outputs = y_exp else: outputs = torch.cat((outputs, y_exp[:, input_n:, :]), 1) pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs pred_expmap = pred_expmap[:, input_n:, :].contiguous().view( -1, dim_full_len) targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view( -1, dim_full_len) pred_expmap[:, 0:6] = 0 targ_expmap[:, 0:6] = 0 pred_expmap = pred_expmap.view(-1, 3) targ_expmap = targ_expmap.view(-1, 3) # get euler angles from expmap pred_eul = data_utils.rotmat2euler_torch( data_utils.expmap2rotmat_torch(pred_expmap)) pred_eul = pred_eul.view(-1, dim_full_len).view(-1, output_n, dim_full_len) targ_eul = data_utils.rotmat2euler_torch( data_utils.expmap2rotmat_torch(targ_expmap)) targ_eul = targ_eul.view(-1, dim_full_len).view(-1, output_n, dim_full_len) # get 3d coordinates targ_p3d = data_utils.expmap2xyz_torch( targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3) pred_p3d = data_utils.expmap2xyz_torch( pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3) # update loss and testing errors for k in np.arange(0, len(eval_frame)): j = eval_frame[k] ''' t_e[k] += torch.mean(torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2, 1)).cpu().data.numpy()[0] * n t_3d[k] += torch.mean(torch.norm( targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2, 1)).cpu().data.numpy()[0] * n ''' t_e[k] += torch.mean( torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2, 1)).item() * n t_3d[k] += torch.mean( torch.norm( targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2, 1)).item() * n # t_l += loss.cpu().data.numpy()[0] * n N += n bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format( i + 1, len(train_loader), time.time() - bt, time.time() - st) bar.next() bar.finish() return t_e / N, t_3d / N
def test(self, train_loader, dataset='h3.6m', input_n=20, output_n=50, dct_n=20, cartesian=False, dim_used=[]): N = 0 # t_l = 0 if output_n >= 25: eval_frame = [1, 3, 7, 9, 13, 24] elif output_n == 10: eval_frame = [1, 3, 7, 9] t_e = np.zeros(len(eval_frame)) t_3d = np.zeros(len(eval_frame)) self.model.eval() st = time.time() bar = Bar('>>>', fill='>', max=len(train_loader)) for i, (inputs, targets, all_seq) in enumerate(train_loader): bt = time.time() if self.is_cuda: inputs = Variable(inputs.cuda()).float() all_seq = Variable(all_seq.cuda(non_blocking=True)).float() outputs, reconstructions, log_var, z = self.model(inputs.float()) n = outputs.shape[0] n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) all_seq[:, :, 0:6] = 0 # inverse dct transformation _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, dct_n).transpose(0, 1) if cartesian == False: outputs_exp = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose( 1, 2) pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs_exp pred_expmap = pred_expmap[:, input_n:, :].contiguous().view(-1, dim_full_len) targ_expmap = all_seq[:, input_n:, :].clone().contiguous().view(-1, dim_full_len) pred_expmap[:, 0:6] = 0 targ_expmap[:, 0:6] = 0 pred_expmap = pred_expmap.view(-1, 3) targ_expmap = targ_expmap.view(-1, 3) # get euler angles from expmap pred_eul = data_utils.rotmat2euler_torch(data_utils.expmap2rotmat_torch(pred_expmap)) pred_eul = pred_eul.view(-1, dim_full_len).view(-1, output_n, dim_full_len) targ_eul = data_utils.rotmat2euler_torch(data_utils.expmap2rotmat_torch(targ_expmap)) targ_eul = targ_eul.view(-1, dim_full_len).view(-1, output_n, dim_full_len) if dataset == 'h3.6m': # get 3d coordinates targ_p3d = data_utils.expmap2xyz_torch(targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3) pred_p3d = data_utils.expmap2xyz_torch(pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3) elif dataset == 'cmu_mocap': # get 3d coordinates targ_p3d = data_utils.expmap2xyz_torch_cmu(targ_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3) pred_p3d = data_utils.expmap2xyz_torch_cmu(pred_expmap.view(-1, dim_full_len)).view(n, output_n, -1, 3) for k in np.arange(0, len(eval_frame)): j = eval_frame[k] t_e[k] += torch.mean(torch.norm(pred_eul[:, j, :] - targ_eul[:, j, :], 2, 1)).cpu().data.numpy() * n t_3d[k] += torch.mean(torch.norm( targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2, 1)).cpu().data.numpy() * n elif cartesian: outputs_3d = torch.matmul(idct_m[:, :dct_n], outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose( 1, 2) pred_3d = all_seq.clone() dim_used = np.array(dim_used) # deal with joints at same position joint_to_ignore = np.array([16, 20, 29, 24, 27, 33, 36]) index_to_ignore = np.concatenate( (joint_to_ignore * 3, joint_to_ignore * 3 + 1, joint_to_ignore * 3 + 2)) joint_equal = np.array([15, 15, 15, 23, 23, 32, 32]) index_to_equal = np.concatenate((joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2)) pred_3d[:, :, dim_used] = outputs_3d pred_3d[:, :, index_to_ignore] = pred_3d[:, :, index_to_equal] pred_p3d = pred_3d.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :] targ_p3d = all_seq.contiguous().view(n, seq_len, -1, 3)[:, input_n:, :, :] for k in np.arange(0, len(eval_frame)): j = eval_frame[k] t_e[k] += torch.mean(torch.norm( targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2, 1)).cpu().data.numpy()[0] * n t_3d[k] += torch.mean(torch.norm( targ_p3d[:, j, :, :].contiguous().view(-1, 3) - pred_p3d[:, j, :, :].contiguous().view(-1, 3), 2, 1)).cpu().data.numpy()[0] * n N += n bar.suffix = '{}/{}|batch time {:.4f}s|total time{:.2f}s'.format(i + 1, len(train_loader), time.time() - bt, time.time() - st) bar.next() bar.finish() return t_e / N, t_3d / N