def __init__(self, path_to_data, actions, input_n=20, output_n=10, split=0, sample_rate=2, autoencoder=IdentityAutoencoder(), subset=False, treat_subj5_differently=True): """ :param split: 0 train, 1 testing, 2 validation """ # Note: the default autoencoder is an indentity mapping which is what is used in the paper self.path_to_data = path_to_data self.split = split subs = np.array([[1, 6, 7, 8, 9], [5], [11]], dtype=object) acts = data_utils.define_actions(actions) if subset: subs = np.array([[1], [5], [11]], dtype=object) acts = ['walking'] subjs = subs[split] all_seqs, dim_ignore, dim_used = data_utils.load_data_3d(path_to_data, subjs, acts, sample_rate, input_n + output_n, treat_subj5_differently=treat_subj5_differently) self.all_seqs = all_seqs self.dim_used = dim_used # (nb_total_seq, len_seq, nb_joints) all_seqs = torch.from_numpy(all_seqs[:, :, dim_used]).float() # (nb_total_seq, nb_joints, hidden_dim) self.all_seqs_encoded = autoencoder(all_seqs.transpose(2, 1))[1]
def __init__(self, path_to_data, actions, input_n=10, output_n=10, split=0, sample_rate=2, data_mean=0, data_std=0): """ read h36m data to get the dct coefficients. :param path_to_data: :param actions: actions to read :param input_n: past frame length :param output_n: future frame length :param dct_n: number of dct coeff. used :param split: 0 train, 1 test, 2 validation :param sample_rate: 2 :param data_mean: mean of expmap :param data_std: standard deviation of expmap """ self.path_to_data = path_to_data self.split = split subs = np.array([[1, 6, 7, 8, 9], [5], [11]]) acts = data_utils.define_actions(actions) # subs = np.array([[1], [5], [11]]) # acts = ['walking'] subjs = subs[split] all_seqs, dim_ignore, dim_use, data_mean, data_std = data_utils.load_data(path_to_data, subjs, acts, sample_rate, input_n + output_n, data_mean=data_mean, data_std=data_std, input_n=input_n) self.data_mean = data_mean self.data_std = data_std # first 6 elements are global translation and global rotation dim_used = dim_use[6:] self.all_seqs = all_seqs self.dim_used = dim_used all_seqs = all_seqs[:, :, dim_used] all_seqs = all_seqs.transpose(0, 2, 1) all_seqs = all_seqs.reshape(-1, input_n + output_n) all_seqs = all_seqs.transpose() # padding the observed sequence so that it has the same length as observed + future sequence pad_idx = np.repeat([input_n - 1], output_n) i_idx = np.append(np.arange(0, input_n), pad_idx) input_chebyshev_coef = data_utils.get_chebyshev_coef(input_n + output_n, all_seqs[i_idx, :]) input_chebyshev_coef = input_chebyshev_coef.transpose().reshape([-1, len(dim_used), input_n + output_n]) target_chebyshev_coef = data_utils.get_chebyshev_coef(input_n + output_n, all_seqs) target_chebyshev_coef = target_chebyshev_coef.transpose().reshape([-1, len(dim_used), input_n + output_n]) self.input_chebyshev_coef = input_chebyshev_coef #self.output_chebyshev_coef = target_chebyshev_coef self.output_chebyshev_coef = target_chebyshev_coef-input_chebyshev_coef
def __init__(self, path_to_data, actions, input_n=10, output_n=10, dct_n=20, split=0, sample_rate=2, data_mean=0, data_std=0): """ read h36m data to get the dct coefficients. :param path_to_data: :param actions: actions to read :param input_n: past frame length :param output_n: future frame length :param dct_n: number of dct coeff. used :param split: 0 train, 1 test, 2 validation :param sample_rate: 2 :param data_mean: mean of expmap :param data_std: standard deviation of expmap """ self.path_to_data = path_to_data self.split = split subs = np.array([[1, 6, 7, 8, 9], [5], [11]]) acts = data_utils.define_actions(actions) # subs = np.array([[1], [5], [11]]) # acts = ['walking'] subjs = subs[split] all_seqs, dim_ignore, dim_use, data_mean, data_std = data_utils.load_data( path_to_data, subjs, acts, sample_rate, 1, data_mean=data_mean, data_std=data_std, input_n=1) self.data_mean = data_mean self.data_std = data_std self.max = np.max(all_seqs) self.min = np.min(all_seqs) # first 6 elements are global translation and global rotation dim_used = dim_use[6:] self.all_seqs = all_seqs self.dim_used = dim_used m = self.all_seqs.shape[0] n = self.all_seqs.shape[-1] self.all_seqs = self.all_seqs.reshape((m, n)) #print("all seq_shape ", self.all_seqs.shape) self.all_seqs = self.all_seqs[:, dim_used]
def __init__(self, path_to_data, actions, input_n=20, output_n=10, dct_used=15, split=0, sample_rate=2): """ :param path_to_data: :param actions: :param input_n: :param output_n: :param dct_used: :param split: 0 train, 1 testing, 2 validation :param sample_rate: """ self.path_to_data = path_to_data self.split = split self.dct_used = dct_used subs = np.array([[1, 6, 7, 8, 9], [5], [11]]) acts = data_utils.define_actions(actions) # subs = np.array([[1], [5], [11]]) # acts = ['walking'] subjs = subs[split] all_seqs, dim_ignore, dim_used = data_utils.load_data_3d( path_to_data, subjs, acts, sample_rate, input_n + output_n) self.all_seqs = all_seqs self.dim_used = dim_used all_seqs = all_seqs[:, :, dim_used] all_seqs = all_seqs.transpose(0, 2, 1) all_seqs = all_seqs.reshape(-1, input_n + output_n) all_seqs = all_seqs.transpose() dct_m_in, _ = data_utils.get_dct_matrix(input_n + output_n) dct_m_out, _ = data_utils.get_dct_matrix(input_n + output_n) pad_idx = np.repeat([input_n - 1], output_n) i_idx = np.append(np.arange(0, input_n), pad_idx) input_dct_seq = np.matmul(dct_m_in[0:dct_used, :], all_seqs[i_idx, :]) input_dct_seq = input_dct_seq.transpose().reshape( [-1, len(dim_used), dct_used]) # input_dct_seq = input_dct_seq.reshape(-1, len(dim_used) * dct_used) output_dct_seq = np.matmul(dct_m_out[0:dct_used, :], all_seqs) output_dct_seq = output_dct_seq.transpose().reshape( [-1, len(dim_used), dct_used]) # output_dct_seq = output_dct_seq.reshape(-1, len(dim_used) * dct_used) self.input_dct_seq = input_dct_seq self.output_dct_seq = output_dct_seq
def main(opt): model = nnmodel.InceptionGCN(opt.linear_size, opt.dropout, num_stage=opt.num_stage, node_n=66, opt=opt) # If you change the path of the model, change the configuration of opt accordingly e.g. if the model # was trained for long term prediction, change opt.output_n to 25 model_path_len = './checkpoint/pretrained/ckpt_main_3d_3D_in10_out10_best.pth.tar' ckpt = torch.load(model_path_len, map_location='cpu') model.load_state_dict(ckpt['state_dict']) # data loading acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion3D(path_to_data=opt.data_dir, actions=act, input_n=opt.input_n, output_n=opt.output_n, split=1, sample_rate=opt.sample_rate) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) dim_used = test_dataset.dim_used print(">>> data loaded !") model.eval() fig = plt.figure() ax = plt.gca(projection='3d') for act in acts: print(act) if opt.output_n == 25 and act not in [ 'smoking', 'eating', 'discussion', 'walking' ]: continue for i, (inputs, all_seq) in enumerate(test_data[act]): print(act) preds = model(inputs) pred_exmap = all_seq.clone() pred_exmap[:, :, dim_used] = preds.detach().transpose(1, 2) for k in range(0, 8): plt.cla() figure_title = "action:{}, seq:{},".format(act, (k + 1)) viz.plot_predictions(all_seq.numpy()[k, :, :], pred_exmap.numpy()[k, :, :], fig, ax, figure_title)
def __init__(self, path_to_data, actions, input_n=20, output_n=10, dct_used=15, split=0, sample_rate=2): """ :param path_to_data: :param actions: :param input_n: :param output_n: :param dct_used: :param split: 0 train, 1 testing, 2 validation :param sample_rate: """ self.path_to_data = path_to_data self.split = split self.dct_used = dct_used subs = np.array([[1, 6, 7, 8, 9], [5], [11]]) acts = data_utils.define_actions(actions) # subs = np.array([[1], [5], [11]]) # acts = ['walking'] subjs = subs[split] all_seqs, dim_ignore, dim_used = data_utils.load_data_3d( path_to_data, subjs, acts, sample_rate, 20) self.all_seqs = all_seqs self.dim_used = dim_used m = self.all_seqs.shape[0] n = self.all_seqs.shape[-1] #average pose over 20 contiguous timesteps self.all_seqs = np.mean(self.all_seqs, axis=1) #normalise to unit cube self.all_seqs = self.all_seqs.reshape((m, n)) max_per_pose = np.amax(self.all_seqs, -1) max_per_pose = np.repeat(max_per_pose.reshape(m, 1), 96, axis=1) min_per_pose = np.amin(self.all_seqs, -1) min_per_pose = np.repeat(min_per_pose.reshape(m, 1), 96, axis=1) self.all_seqs = (self.all_seqs - min_per_pose) / (max_per_pose - min_per_pose)
def main(opt): is_cuda = torch.cuda.is_available() # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n sample_rate = opt.sample_rate model = nnmodel.GCN(input_feature=(input_n + output_n), hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=48) if is_cuda: model.cuda() model_path_len = './checkpoint/pretrained/h36m_in10_out25.pth.tar' print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') err_best = ckpt['err'] start_epoch = ckpt['epoch'] model.load_state_dict(ckpt['state_dict']) print(">>> ckpt len loaded (epoch: {} | err: {})".format(start_epoch, err_best)) # data loading print(">>> loading data") acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate) test_data[act] = DataLoader( dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) dim_used = test_dataset.dim_used print(">>> data loaded !") model.eval() fig = plt.figure() ax = plt.gca(projection='3d') for act in acts: for i, (inputs, targets, all_seq) in enumerate(test_data[act]): inputs = Variable(inputs).float() all_seq = Variable(all_seq).float() if is_cuda: inputs = inputs.cuda() all_seq = all_seq.cuda() outputs = model(inputs) n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() outputs_t = outputs.view(-1, seq_len).transpose(0, 1) outputs_exp = torch.matmul(idct_m, outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2) pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs_exp targ_expmap = all_seq pred_expmap = pred_expmap.cpu().data.numpy() targ_expmap = targ_expmap.cpu().data.numpy() for k in range(8): plt.cla() figure_title = "action:{}, seq:{},".format(act, (k + 1)) viz.plot_predictions(targ_expmap[k, :, :], pred_expmap[k, :, :], fig, ax, figure_title) plt.pause(1)
def main(opt): is_cuda = torch.cuda.is_available() # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n sample_rate = opt.sample_rate dct_n = opt.dct_n model = nnmodel.GCN(input_feature=dct_n, hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=66) if is_cuda: model.to('cuda' if torch.cuda.is_available else 'cpu') model_path_len = './checkpoint/pretrained/h36m3D_in10_out10_dctn15.pth.tar' print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') err_best = ckpt['err'] start_epoch = ckpt['epoch'] model.load_state_dict(ckpt['state_dict']) print(">>> ckpt len loaded (epoch: {} | err: {})".format( start_epoch, err_best)) # data loading print(">>> loading data") acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion3D(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate, dct_used=dct_n) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) dim_used = test_dataset.dim_used print(">>> data loaded !") model.eval() fig = plt.figure() ax = plt.gca(projection='3d') for act in acts: for i, (inputs, targets, all_seq) in enumerate(test_data[act]): inputs = Variable(inputs).float() all_seq = Variable(all_seq).float() if is_cuda: inputs = inputs.to( 'cuda' if torch.cuda.is_available else 'cpu') all_seq = all_seq.to( 'cuda' if torch.cuda.is_available else 'cpu') outputs = model(inputs) n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) _, idct_m = data_utils.get_dct_matrix(seq_len) idct_m = Variable(torch.from_numpy(idct_m)).float() if is_cuda: idct_m = idct_m.to( 'cuda' if torch.cuda.is_available else 'cpu') outputs_t = outputs.view(-1, dct_n).transpose(0, 1) outputs_3d = torch.matmul(idct_m[:, 0:dct_n], outputs_t).transpose( 0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2) pred_3d = all_seq.clone() dim_used = np.array(dim_used) # joints at same loc joint_to_ignore = np.array([16, 20, 23, 24, 28, 31]) index_to_ignore = np.concatenate( (joint_to_ignore * 3, joint_to_ignore * 3 + 1, joint_to_ignore * 3 + 2)) joint_equal = np.array([13, 19, 22, 13, 27, 30]) index_to_equal = np.concatenate( (joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2)) pred_3d[:, :, dim_used] = outputs_3d pred_3d[:, :, index_to_ignore] = pred_3d[:, :, index_to_equal] pred_p3d = pred_3d.contiguous().view( n, seq_len, -1, 3)[:, input_n:, :, :].cpu().data.numpy() targ_p3d = all_seq.contiguous().view( n, seq_len, -1, 3)[:, input_n:, :, :].cpu().data.numpy() for k in range(8): plt.cla() figure_title = "action:{}, seq:{},".format(act, (k + 1)) viz.plot_predictions_direct(targ_p3d[k], pred_p3d[k], fig, ax, figure_title) plt.pause(1)
opt = parser.parse_args() is_cuda = torch.cuda.is_available() input_n = 10 output_n = 25 dct_n = 35 sample_rate = 2 cartesian = False if opt.dataset == 'h3.6m': node_n = 48 n_z = 384 actions = data_utils.define_actions('all', 'h3.6m', out_of_distribution=False) elif opt.dataset == 'cmu_mocap': node_n = 64 actions = data_utils.define_actions('all', 'cmu_mocap', out_of_distribution=False) n_z = 512 model = nnmodel.GCN(input_feature=dct_n, hidden_feature=256, p_dropout=0.3, num_stage=12, node_n=node_n, variational=True, n_z=32, num_decoder_stage=6) if is_cuda: model.cuda() print(">>> total params: {:.2f}M".format(sum(p.numel() for p in model.parameters()) / 1000000.0)) # optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) if opt.model_path == None: model_path_len = 'checkpoint/test/ckpt_main_cmu_mocap_in50_out25_dctn35_dropout_0.3_var_lambda_0.003_nz_8_lr_0.0005_n_layers_6_last.pth.tar' else:
def main(opt): start_epoch = 0 err_best = 10000 lr_now = opt.lr is_cuda = torch.cuda.is_available() opt.is_load = True # save option in log script_name = os.path.basename(__file__).split('.')[0] script_name = script_name + '_3D_in{:d}_out{:d}_dct_n_{:d}'.format( opt.input_n, opt.output_n, opt.dct_n) # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n dct_n = opt.dct_n sample_rate = opt.sample_rate model = nnmodel.GCN(input_feature=dct_n, hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=66) if is_cuda: model.cuda() print(">>> total params: {:.2f}M".format( sum(p.numel() for p in model.parameters()) / 1000000.0)) optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) if opt.is_load: model_path_len = 'checkpoint/pretrained/h36m3D_in10_out25_dctn30.pth.tar' print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') start_epoch = ckpt['epoch'] err_best = ckpt['err'] lr_now = ckpt['lr'] model.load_state_dict(ckpt['state_dict']) optimizer.load_state_dict(ckpt['optimizer']) print(">>> ckpt len loaded (epoch: {} | err: {})".format( start_epoch, err_best)) # data loading print(">>> loading data") # train_dataset = H36motion3D(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n, # split=0, dct_used=dct_n, sample_rate=sample_rate) acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion3D(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate, dct_used=dct_n) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) # val_dataset = H36motion3D(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n, # split=2, dct_used=dct_n, sample_rate=sample_rate) # load dadasets for training # train_loader = DataLoader( # dataset=train_dataset, # batch_size=opt.train_batch, # shuffle=True, # num_workers=opt.job, # pin_memory=True) # val_loader = DataLoader( # dataset=val_dataset, # batch_size=opt.test_batch, # shuffle=False, # num_workers=opt.job, # pin_memory=True) print(">>> data loaded !") # print(">>> train data {}".format(train_dataset.__len__())) # print(">>> test data {}".format(test_dataset.__len__())) # print(">>> validation data {}".format(val_dataset.__len__())) # for epoch in range(start_epoch, opt.epochs): # # if (epoch + 1) % opt.lr_decay == 0: # lr_now = utils.lr_decay(optimizer, lr_now, opt.lr_gamma) # # print('==========================') # print('>>> epoch: {} | lr: {:.5f}'.format(epoch + 1, lr_now)) ret_log = np.array([start_epoch]) head = np.array(['epoch']) # per epoch # lr_now, t_l = train(train_loader, model, optimizer, lr_now=lr_now, max_norm=opt.max_norm, is_cuda=is_cuda, # dim_used=train_dataset.dim_used, dct_n=dct_n) # ret_log = np.append(ret_log, [lr_now, t_l]) # head = np.append(head, ['lr', 't_l']) # # v_3d = val(val_loader, model, is_cuda=is_cuda, dim_used=train_dataset.dim_used, dct_n=dct_n) # # ret_log = np.append(ret_log, [v_3d]) # head = np.append(head, ['v_3d']) test_3d_temp = np.array([]) test_3d_head = np.array([]) for act in acts: test_l, test_3d = test(test_data[act], model, input_n=input_n, output_n=output_n, is_cuda=is_cuda, dim_used=test_dataset.dim_used, dct_n=dct_n) # ret_log = np.append(ret_log, test_l) ret_log = np.append(ret_log, test_3d) head = np.append( head, [act + '3d80', act + '3d160', act + '3d320', act + '3d400']) if output_n > 10: head = np.append(head, [act + '3d560', act + '3d1000']) ret_log = np.append(ret_log, test_3d_temp) head = np.append(head, test_3d_head) # update log file and save checkpoint df = pd.DataFrame(np.expand_dims(ret_log, axis=0)) # if epoch == start_epoch: df.to_csv(opt.ckpt + '/' + script_name + '.csv', header=head, index=False)
def __init__(self, path_to_data, actions, input_n=20, dct_used=15, split=0, sample_rate=2): """ :param path_to_data: :param actions: :param input_n: :param output_n: :param dct_used: :param split: 0 train, 1 testing, 2 validation :param sample_rate: """ self.path_to_data = path_to_data self.split = split self.dct_used = dct_used subs = np.array([[1, 6, 7, 8, 9], [5], [11]]) acts = data_utils.define_actions(actions) subjs = subs[split] labels = [] all_seqs = np.array([]) #print(acts) for act in acts: #print(act) action_seq, dim_ignore, dim_used = data_utils.load_data_3d( path_to_data, subjs, [act], sample_rate, input_n) #print(action_seq.shape) try: all_seqs = np.concatenate((all_seqs, action_seq), axis=0) except: all_seqs = action_seq label = [str(act)] * action_seq.shape[0] labels = labels + label #print(len(labels)) #print(all_seqs.shape) #print(labels[0], labels[-1]) #all_seqs, dim_ignore, dim_used = data_utils.load_data_3d(path_to_data, subjs, acts, sample_rate, input_n ) self.labels = labels self.all_seqs = all_seqs t_n = input_n b_n, f_n = self.all_seqs.shape[0], 96 self.all_seqs = self.all_seqs.reshape(b_n, f_n, t_n) #normalise to unit cube max_per_pose = np.amax(self.all_seqs, axis=(1, 2)) max_per_pose = np.repeat(max_per_pose.reshape(b_n, 1, 1), 96, axis=1) max_per_pose = np.repeat(max_per_pose.reshape(b_n, 96, 1), t_n, axis=2) min_per_pose = np.amin(self.all_seqs, axis=(1, 2)) min_per_pose = np.repeat(min_per_pose.reshape(b_n, 1, 1), 96, axis=1) min_per_pose = np.repeat(min_per_pose.reshape(b_n, 96, 1), t_n, axis=2) self.all_seqs = (self.all_seqs - min_per_pose) / (max_per_pose - min_per_pose)
def main(opt): start_epoch = 0 err_best = 10000 lr_now = opt.lr is_cuda = torch.cuda.is_available() # define log csv file script_name = os.path.basename(__file__).split('.')[0] script_name = script_name + "_in{:d}_out{:d}_dctn{:d}".format( opt.input_n, opt.output_n, opt.dct_n) # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n dct_n = opt.dct_n sample_rate = opt.sample_rate # 48 nodes for angle prediction model = nnmodel.GCN(input_feature=input_n + output_n, hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=48) if is_cuda: model.cuda() print(">>> total params: {:.2f}M".format( sum(p.numel() for p in model.parameters()) / 1000000.0)) optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) # continue from checkpoint if opt.is_load: model_path_len = 'checkpoint/test/ckpt_main_gcn_muti_att_best.pth.tar' print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') start_epoch = ckpt['epoch'] err_best = ckpt['err'] lr_now = ckpt['lr'] model.load_state_dict(ckpt['state_dict']) optimizer.load_state_dict(ckpt['optimizer']) print(">>> ckpt len loaded (epoch: {} | err: {})".format( start_epoch, err_best)) # data loading print(">>> loading data") train_dataset = H36motion(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n, split=0, sample_rate=sample_rate) data_std = train_dataset.data_std data_mean = train_dataset.data_mean val_dataset = H36motion(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n, split=2, sample_rate=sample_rate, data_mean=data_mean, data_std=data_std) # load dadasets for training train_loader = DataLoader(dataset=train_dataset, batch_size=opt.train_batch, shuffle=True, num_workers=opt.job, pin_memory=True) val_loader = DataLoader(dataset=val_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate, data_mean=data_mean, data_std=data_std) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) print(">>> data loaded !") print(">>> train data {}".format(train_dataset.__len__())) print(">>> validation data {}".format(val_dataset.__len__())) for epoch in range(start_epoch, opt.epochs): if (epoch + 1) % opt.lr_decay == 0: lr_now = utils.lr_decay(optimizer, lr_now, opt.lr_gamma) print('==========================') print('>>> epoch: {} | lr: {:.5f}'.format(epoch + 1, lr_now)) ret_log = np.array([epoch + 1]) head = np.array(['epoch']) # per epoch lr_now, t_l, t_e, t_3d = train(train_loader, model, optimizer, input_n=input_n, output_n=output_n, lr_now=lr_now, max_norm=opt.max_norm, is_cuda=is_cuda, dim_used=train_dataset.dim_used) ret_log = np.append(ret_log, [lr_now, t_l, t_e, t_3d]) head = np.append(head, ['lr', 't_l', 't_e', 't_3d']) v_e, v_3d = val(val_loader, model, input_n=input_n, output_n=output_n, is_cuda=is_cuda, dim_used=train_dataset.dim_used) ret_log = np.append(ret_log, [v_e, v_3d]) head = np.append(head, ['v_e', 'v_3d']) test_3d_temp = np.array([]) test_3d_head = np.array([]) for act in acts: test_e, test_3d = test(test_data[act], model, input_n=input_n, output_n=output_n, is_cuda=is_cuda, dim_used=train_dataset.dim_used) ret_log = np.append(ret_log, test_e) test_3d_temp = np.append(test_3d_temp, test_3d) test_3d_head = np.append( test_3d_head, [act + '3d80', act + '3d160', act + '3d320', act + '3d400']) head = np.append( head, [act + '80', act + '160', act + '320', act + '400']) if output_n > 10: head = np.append(head, [act + '560', act + '1000']) test_3d_head = np.append(test_3d_head, [act + '3d560', act + '3d1000']) ret_log = np.append(ret_log, test_3d_temp) head = np.append(head, test_3d_head) # update log file and save checkpoint df = pd.DataFrame(np.expand_dims(ret_log, axis=0)) if epoch == start_epoch: df.to_csv(opt.ckpt + '/' + script_name + '.csv', header=head, index=False) else: with open(opt.ckpt + '/' + script_name + '.csv', 'a') as f: df.to_csv(f, header=False, index=False) if not np.isnan(v_e): is_best = v_e < err_best err_best = min(v_e, err_best) else: is_best = False file_name = [ 'ckpt_' + script_name + '_best.pth.tar', 'ckpt_' + script_name + '_last.pth.tar' ] utils.save_ckpt( { 'epoch': epoch + 1, 'lr': lr_now, 'err': test_e[0], 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict() }, ckpt_path=opt.ckpt, is_best=is_best, file_name=file_name)
def get_poses(self, input_n, output_n, sample_rate, dct_n, out_of_distribution_action=None, val_categorise=False): if out_of_distribution_action != None: self.out_of_distribution = True self.acts_train = data_utils.define_actions( out_of_distribution_action, self.dataset, out_of_distribution=False) self.acts_OoD = data_utils.define_actions( out_of_distribution_action, self.dataset, out_of_distribution=True) self.acts_test = data_utils.define_actions( 'all', self.dataset, out_of_distribution=False) else: self.out_of_distribution = False self.acts_train = data_utils.define_actions( 'all', self.dataset, out_of_distribution=False) self.acts_OoD = None self.acts_test = data_utils.define_actions( 'all', self.dataset, out_of_distribution=False) if self.dataset == 'h3.6m': self.cartesian = False self.node_n = 48 self.train_dataset = H36motion_pose(path_to_data=self.data_dir, actions=self.acts_train, input_n=input_n, output_n=output_n, split=0, sample_rate=sample_rate, dct_n=dct_n) self.data_std = self.train_dataset.data_std self.data_mean = self.train_dataset.data_mean self.dim_used = self.train_dataset.dim_used self.val_dataset = H36motion_pose(path_to_data=self.data_dir, actions=self.acts_train, input_n=input_n, output_n=output_n, split=2, sample_rate=sample_rate, data_mean=self.data_mean, data_std=self.data_std, dct_n=dct_n) if self.out_of_distribution: self.OoD_val_dataset = H36motion_pose( path_to_data=self.data_dir, actions=self.acts_OoD, input_n=input_n, output_n=output_n, split=2, sample_rate=sample_rate, data_mean=self.data_mean, data_std=self.data_std, dct_n=dct_n) else: self.OoD_val_dataset = None self.test_dataset = dict() for act in self.acts_test: self.test_dataset[act] = H36motion_pose( path_to_data=self.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate, data_mean=self.data_mean, data_std=self.data_std, dct_n=dct_n) elif self.dataset == 'h3.6m_3d': self.cartesian = False self.node_n = 96 if val_categorise: self.train_dataset = dict() self.val_dataset = dict() for act in self.acts_train: self.train_dataset[act] = H36motion3D_pose( path_to_data=self.data_dir, actions=act, input_n=input_n, output_n=output_n, split=0, sample_rate=sample_rate, dct_used=dct_n) self.val_dataset[act] = H36motion3D_pose( path_to_data=self.data_dir, actions=act, input_n=input_n, output_n=output_n, split=2, sample_rate=sample_rate, dct_used=dct_n) else: self.train_dataset = H36motion3D_pose( path_to_data=self.data_dir, actions=self.acts_train, input_n=input_n, output_n=output_n, split=0, sample_rate=sample_rate, dct_used=dct_n) self.val_dataset = H36motion3D_pose(path_to_data=self.data_dir, actions=self.acts_train, input_n=input_n, output_n=output_n, split=2, sample_rate=sample_rate, dct_used=dct_n) if self.out_of_distribution: self.OoD_val_dataset = H36motion3D_pose( path_to_data=self.data_dir, actions=self.acts_OoD, input_n=input_n, output_n=output_n, split=2, sample_rate=sample_rate, dct_used=dct_n) else: self.OoD_val_dataset = None self.test_dataset = dict() for act in self.acts_test: self.test_dataset[act] = H36motion3D_pose( path_to_data=self.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate, dct_used=dct_n) else: raise Exception("Dataset name ({}) is not valid!".format(dataset)) return self.out_of_distribution
def get_dct_and_sequences(self, input_n, output_n, sample_rate, dct_n, out_of_distribution_action=None): if out_of_distribution_action != None: self.out_of_distribution = True acts_train = data_utils.define_actions(out_of_distribution_action, self.dataset, out_of_distribution=False) acts_OoD = data_utils.define_actions(out_of_distribution_action, self.dataset, out_of_distribution=True) acts_test = data_utils.define_actions('all', self.dataset, out_of_distribution=False) else: self.out_of_distribution = False acts_train = data_utils.define_actions('all', self.dataset, out_of_distribution=False) acts_OoD = None acts_test = data_utils.define_actions('all', self.dataset, out_of_distribution=False) self.acts_test = acts_test if self.dataset == 'h3.6m': self.cartesian = False self.node_n = 48 self.train_dataset = H36motion(path_to_data=self.data_dir, actions=acts_train, input_n=input_n, output_n=output_n, split=0, sample_rate=sample_rate, dct_n=dct_n) self.data_std = self.train_dataset.data_std self.data_mean = self.train_dataset.data_mean self.dim_used = self.train_dataset.dim_used self.val_dataset = H36motion(path_to_data=self.data_dir, actions=acts_train, input_n=input_n, output_n=output_n, split=2, sample_rate=sample_rate, data_mean=self.data_mean, data_std=self.data_std, dct_n=dct_n) if self.out_of_distribution: self.OoD_val_dataset = H36motion(path_to_data=self.data_dir, actions=acts_OoD, input_n=input_n, output_n=output_n, split=2, sample_rate=sample_rate, data_mean=self.data_mean, data_std=self.data_std, dct_n=dct_n) else: self.OoD_val_dataset = None self.test_dataset = dict() for act in acts_test: self.test_dataset[act] = H36motion(path_to_data=self.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate, data_mean=self.data_mean, data_std=self.data_std, dct_n=dct_n) elif self.dataset == 'cmu_mocap': self.cartesian = False self.node_n = 64 self.train_dataset = CMU_Motion(path_to_data=self.data_dir, actions=acts_train, input_n=input_n, output_n=output_n, split=0, dct_n=dct_n) self.data_std = self.train_dataset.data_std self.data_mean = self.train_dataset.data_mean self.dim_used = self.train_dataset.dim_used self.val_dataset = None self.OoD_val_dataset = None self.test_dataset = dict() for act in acts_test: self.test_dataset[act] = CMU_Motion(path_to_data=self.data_dir, actions=[act], input_n=input_n, output_n=output_n, split=1, data_mean=self.data_mean, data_std=self.data_std, dim_used=self.dim_used, dct_n=dct_n) elif self.dataset == 'cmu_mocap_3d': self.cartesian = True self.node_n = 75 self.train_dataset = CMU_Motion3D(path_to_data=self.data_dir, actions=acts_train, input_n=input_n, output_n=output_n, split=0, dct_n=dct_n) self.data_std = self.train_dataset.data_std self.data_mean = self.train_dataset.data_mean self.dim_used = self.train_dataset.dim_used self.val_dataset = None self.OoD_val_dataset = None self.test_dataset = dict() for act in acts_test: self.test_dataset[act] = CMU_Motion3D( path_to_data=self.data_dir, actions=[act], input_n=input_n, output_n=output_n, split=1, data_mean=self.data_mean, data_std=self.data_std, dim_used=self.dim_used, dct_n=dct_n) else: raise Exception("Dataset name ({}) is not valid!".format(dataset)) return self.out_of_distribution
n_z=1) print(">>> total params: {:.2f}M".format( sum(p.numel() for p in model.parameters()) / 1000000.0)) lr = 0.00003 optimizer = torch.optim.Adam(model.parameters(), lr=lr) if is_cuda: model.cuda() ckpt = torch.load(opt.ckpt) start_epoch = ckpt['epoch'] err_best = ckpt['err'] lr_now = ckpt['lr'] model.load_state_dict(ckpt['state_dict']) optimizer.load_state_dict(ckpt['optimizer']) print(">>> loading data") acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion(path_to_data='h3.6m/dataset/', actions=act, input_n=10, output_n=10, split=1, sample_rate=2) test_data[act] = DataLoader(dataset=test_dataset, batch_size=1, shuffle=False, num_workers=1, pin_memory=True) dim_used = test_dataset.dim_used print(">>> data loaded !")
action='store_true', help='toggle do degradation experiments') parser.add_argument('--n_z', type=int, default=2, help='Number of latent variables') parser.add_argument('--algorithm', type=str, default="PCA", help='choose PCA or UMAP') parser.set_defaults(multi_dim_experiment=False) parser.set_defaults(degradation_experiment=False) opt = parser.parse_args() acts = data_utils.define_actions("all") train_dataset = H36motion3D_pose(path_to_data="h3.6m/dataset/", actions=acts, input_n=1, output_n=1, split=0, sample_rate=2, dct_used=2) val_dataset = H36motion3D_pose(path_to_data="h3.6m/dataset/", actions=acts, input_n=1, output_n=1, split=2, sample_rate=2, dct_used=2)
def main(opt): start_epoch = 0 err_best = 10000 lr_now = opt.lr is_cuda = torch.cuda.is_available() # define log csv file script_name = os.path.basename(__file__).split('.')[0] script_name = script_name + "_in{:d}_out{:d}_dctn{:d}".format( opt.input_n, opt.output_n, opt.dct_n) # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n dct_n = opt.dct_n sample_rate = opt.sample_rate # 48 nodes for angle prediction model = nnmodel.GCN(input_feature=dct_n, hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=48) if is_cuda: model.cuda() print(">>> total params: {:.2f}M".format( sum(p.numel() for p in model.parameters()) / 1000000.0)) optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) # continue from checkpoint if opt.is_load: model_path_len = 'checkpoint/test/ckpt_main_gcn_muti_att_best.pth.tar' print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') start_epoch = ckpt['epoch'] err_best = ckpt['err'] lr_now = ckpt['lr'] model.load_state_dict(ckpt['state_dict']) optimizer.load_state_dict(ckpt['optimizer']) print(">>> ckpt len loaded (epoch: {} | err: {})".format( start_epoch, err_best)) # data loading print(">>> loading data") train_dataset = H36motion(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n, split=0, sample_rate=sample_rate, dct_n=dct_n) data_std = train_dataset.data_std data_mean = train_dataset.data_mean val_dataset = H36motion(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n, split=2, sample_rate=sample_rate, data_mean=data_mean, data_std=data_std, dct_n=dct_n) # load datasets for training train_loader = DataLoader(dataset=train_dataset, batch_size=opt.train_batch, shuffle=True, num_workers=opt.job, pin_memory=True) val_loader = DataLoader(dataset=val_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate, data_mean=data_mean, data_std=data_std, dct_n=dct_n) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) print(">>> data loaded !") print(">>> train data {}".format(train_dataset.__len__())) print(">>> validation data {}".format(val_dataset.__len__())) for epoch in range(start_epoch, opt.epochs): if (epoch + 1) % opt.lr_decay == 0: lr_now = utils.lr_decay(optimizer, lr_now, opt.lr_gamma) print('==========================') print('>>> epoch: {} | lr: {:.5f}'.format(epoch + 1, lr_now)) ret_log = np.array([epoch + 1]) head = np.array(['epoch']) # per epoch a = train_dataset.dim_used
def main(opt): is_cuda = torch.cuda.is_available() # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n itera = 50 #model = nnmodel.GCN(input_feature=(input_n + output_n), hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=48) model = AttModel.AttModelRef4(in_features=opt.in_features, kernel_size=opt.kernel_size, d_model=opt.d_model, num_stage=opt.num_stage, dct_n=opt.dct_n, device=opt.device) if is_cuda: model.cuda() model_path_len = '/home/costa/src/Transformer/checkpoint/trans_N6_last_pose_in50_out10_ks10_dctn20/ckpt_best.pth.tar' model_path_len = '/home/costa/Desktop/ckpt_best.pth.tar' print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') err_best = ckpt['err'] start_epoch = ckpt['epoch'] model.load_state_dict(ckpt['state_dict']) print(">>> ckpt len loaded (epoch: {} | err: {})".format( start_epoch, err_best)) # data loading print(">>> loading data") acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = datasets.Datasets(opt=opt, actions=[act], split=1, itera=itera) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch_size, shuffle=False, pin_memory=True) dim_used = test_dataset.dimensions_to_use print(">>> data loaded !") joint_to_ignore = np.array([16, 20, 23, 24, 28, 31]) index_to_ignore = np.concatenate( (joint_to_ignore * 3, joint_to_ignore * 3 + 1, joint_to_ignore * 3 + 2)) joint_equal = np.array([13, 19, 22, 13, 27, 30]) index_to_equal = np.concatenate( (joint_equal * 3, joint_equal * 3 + 1, joint_equal * 3 + 2)) model.eval() fig = plt.figure() ax = plt.gca(projection='3d') for act in acts: for i, all_seq in enumerate(test_data[act]): inputs = Variable(all_seq[:, :opt.input_n, dim_used]).float() all_seq = Variable(all_seq).float() if is_cuda: inputs = inputs.cuda() all_seq = all_seq.cuda() outputs = model(inputs, opt.output_n, opt.input_n, itera=itera) n, seq_len, dim_full_len = all_seq.data.shape dim_used_len = len(dim_used) ''' _, idct_m = data_utils.get_dct_matrix(seq_len) if is_cuda: idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() else: idct_m = Variable(torch.from_numpy(idct_m)).float() outputs_t = outputs.view(-1, seq_len).transpose(0, 1) outputs_exp = torch.matmul(idct_m, outputs_t).transpose(0, 1).contiguous().view(-1, dim_used_len, seq_len).transpose(1, 2) p3d_out = p3d_h36.clone()[:, in_n:in_n + out_n] p3d_out[:, :, dim_used] = p3d_out_all[:, seq_in:, 0] p3d_out[:, :, index_to_ignore] = p3d_out[:, :, index_to_equal] pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs_exp ''' pred = all_seq.clone() pred[:, opt.input_n:opt.input_n + opt.output_n * itera, dim_used] = outputs[:, opt.kernel_size:, 0] pred[:, opt.input_n:opt.input_n + opt.output_n * itera, index_to_ignore] = pred[:, opt.input_n:opt.input_n + opt.output_n * itera, index_to_equal] targ = all_seq pred = pred.cpu().data.numpy() targ = targ.cpu().data.numpy() for k in range(8): plt.cla() figure_title = "action:{}, seq:{},".format(act, (k + 1)) viz.plot_predictions_from_3d( targ[k, opt.input_n:opt.input_n + opt.output_n * itera, :], pred[k, opt.input_n:opt.input_n + opt.output_n * itera, :], fig, ax, figure_title) plt.pause(1)
def main(opt): is_cuda = torch.cuda.is_available() desired_acts = ['eating', 'posing', 'sitting', 'posing', 'walkingdog'] # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n dct_n = opt.dct_n #calculate stepsize for auto regression based on input fames and DCT coefficients stepsize = dct_n - input_n sample_rate = opt.sample_rate model = nnmodel.GCN(input_feature=(input_n + stepsize), hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=48) if is_cuda: model.cuda() model_path_len = "checkpoint/pretrained/h36m_in{}_out{}_dctn{}.pth.tar".format( input_n, stepsize, dct_n) print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') err_best = ckpt['err'] start_epoch = ckpt['epoch'] model.load_state_dict(ckpt['state_dict']) print(">>> ckpt len loaded (epoch: {} | err: {})".format( start_epoch, err_best)) # data loading print(">>> loading data") acts = data_utils.define_actions('all') test_data = dict() for act in acts: test_dataset = H36motion(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, dct_n=dct_n, split=1, sample_rate=sample_rate) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) dim_used = test_dataset.dim_used print(">>> data loaded !") model.eval() fig = plt.figure() ax = plt.gca(projection='3d') #calculate no of iterations in auto regression to perform iterations = int(output_n / stepsize) print('iterations: {}'.format(iterations)) for act in acts: for i, (_, targets, all_seq) in enumerate(test_data[act]): all_seq = Variable(all_seq).float() dim_used_len = len(dim_used) if is_cuda: all_seq = all_seq.cuda() dct_m_in, _ = data_utils.get_dct_matrix(dct_n) dct_m_in = Variable(torch.from_numpy(dct_m_in)).float().cuda() _, idct_m = data_utils.get_dct_matrix(dct_n) idct_m = Variable(torch.from_numpy(idct_m)).float().cuda() targ_expmap = all_seq.cpu().data.numpy() y_hat = None #Auto regression for idx in range(iterations): #start index of the input sequence start = input_n + idx * stepsize #end index of the input sequence stop = start + stepsize if y_hat is None: #slice the sequence of length = (input_n + output_n) in iteration 1 input_seq = all_seq[:, :dct_n, dim_used] else: #stack output from prev iteration and next frames to form the next input seq input_seq = torch.cat( (y_hat, all_seq[:, start:stop, dim_used]), 1) #calculate DCT of the input seq input_dct_seq = torch.matmul(dct_m_in, input_seq).transpose(1, 2) if is_cuda: input_dct_seq = input_dct_seq.cuda() y = model(input_dct_seq) y_t = y.view(-1, dct_n).transpose(0, 1) y_exp = torch.matmul(idct_m, y_t).transpose(0, 1).contiguous().view( -1, dim_used_len, dct_n).transpose(1, 2) y_hat = y_exp[:, stepsize:, :] #accumulate the output frames in a single tensor if idx == 0: outputs = y_exp else: outputs = torch.cat((outputs, y_exp[:, input_n:, :]), 1) pred_expmap = all_seq.clone() dim_used = np.array(dim_used) pred_expmap[:, :, dim_used] = outputs pred_expmap = pred_expmap.cpu().data.numpy() #calculate loss and save to a file for later use #save_loss_file(act, pred_expmap, targ_expmap, input_n, output_n) if act in desired_acts: for k in range(8): plt.cla() figure_title = "action:{}, seq:{},".format(act, (k + 1)) viz.plot_predictions(targ_expmap[k, :, :], pred_expmap[k, :, :], fig, ax, figure_title) plt.pause(1)
try: os.makedirs(opt.save_dir) except OSError: pass logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%Y/%m/%d %H:%M:%S', filename=os.path.join(opt.save_dir, 'train_test.log'), level=logging.INFO) logging.info('======================================================') # load model model['st_gcn'] = Model(opt).cuda() model['post_refine'] = post_refine(opt).cuda() dataset = Human36mDataset('data/data_3d_h36m.npz', opt) actions = define_actions(opt.actions) if opt.pro_train: train_data = Fusion(opt=opt, train=True, dataset=dataset) train_dataloader = DataLoader(train_data, batch_size=256, shuffle=True, num_workers=8, pin_memory=False) if opt.pro_test: test_data = Fusion(opt=opt, train=False, dataset=dataset) test_dataloader = DataLoader(test_data, batch_size=256, shuffle=False, num_workers=8, pin_memory=False) # 3. set optimizer all_param = [] for i_model in model: all_param += list(model[i_model].parameters()) if opt.optimizer == 'SGD': optimizer_all = optim.SGD(all_param, lr=opt.learning_rate, momentum=0.9, nesterov=True, weight_decay=opt.weight_decay)
def main(opt): start_epoch = 0 err_best = 10000 lr_now = opt.lr is_cuda = torch.cuda.is_available() opt.is_load = True # define log csv file script_name = os.path.basename(__file__).split('.')[0] script_name = script_name + "_in{:d}_out{:d}_dctn{:d}".format( opt.input_n, opt.output_n, opt.dct_n) desired_acts = ['eating', 'walkingdog'] # create model print(">>> creating model") input_n = opt.input_n output_n = opt.output_n dct_n = opt.dct_n # calculate stepsize for auto regression based on input fames and DCT coefficients stepsize = dct_n - input_n sample_rate = opt.sample_rate # 48 nodes for angle prediction model = nnmodel.GCN(input_feature=dct_n, hidden_feature=opt.linear_size, p_dropout=opt.dropout, num_stage=opt.num_stage, node_n=48) if is_cuda: model.cuda() print(">>> total params: {:.2f}M".format( sum(p.numel() for p in model.parameters()) / 1000000.0)) optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) # continue from checkpoint if opt.is_load: model_path_len = "checkpoint/logs/ckpt_main_in{}_out{}_dctn{}_best.pth.tar".format( input_n, stepsize, dct_n) print(">>> loading ckpt len from '{}'".format(model_path_len)) if is_cuda: ckpt = torch.load(model_path_len) else: ckpt = torch.load(model_path_len, map_location='cpu') start_epoch = ckpt['epoch'] err_best = ckpt['err'] lr_now = ckpt['lr'] model.load_state_dict(ckpt['state_dict']) optimizer.load_state_dict(ckpt['optimizer']) print(">>> ckpt len loaded (epoch: {} | err: {})".format( start_epoch, err_best)) # data loading print(">>> loading data") # train_dataset = H36motion(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n, # split=0, sample_rate=sample_rate, dct_n=dct_n) # data_std = train_dataset.data_std # data_mean = train_dataset.data_mean dim_used = [ 6, 7, 8, 9, 12, 13, 14, 15, 21, 22, 23, 24, 27, 28, 29, 30, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 51, 52, 53, 54, 55, 56, 57, 60, 61, 62, 75, 76, 77, 78, 79, 80, 81, 84, 85, 86 ] # val_dataset = H36motion(path_to_data=opt.data_dir, actions='all', input_n=input_n, output_n=output_n, # split=2, sample_rate=sample_rate, data_mean=data_mean, data_std=data_std, dct_n=dct_n) # # load dadasets for training # train_loader = DataLoader( # dataset=train_dataset, # batch_size=opt.train_batch, # shuffle=True, # num_workers=opt.job, # pin_memory=True) # val_loader = DataLoader( # dataset=val_dataset, # batch_size=opt.test_batch, # shuffle=False, # num_workers=opt.job, # pin_memory=True) acts = data_utils.define_actions('all') print(acts) test_data = dict() for act in acts: test_dataset = H36motion(path_to_data=opt.data_dir, actions=act, input_n=input_n, output_n=output_n, split=1, sample_rate=sample_rate, dct_n=dct_n) test_data[act] = DataLoader(dataset=test_dataset, batch_size=opt.test_batch, shuffle=False, num_workers=opt.job, pin_memory=True) print(">>> data loaded !") # print(">>> train data {}".format(train_dataset.__len__())) # print(">>> validation data {}".format(val_dataset.__len__())) # for epoch in range(start_epoch, opt.epochs): # # if (epoch + 1) % opt.lr_decay == 0: # lr_now = utils.lr_decay(optimizer, lr_now, opt.lr_gamma) # print('==========================') # print('>>> epoch: {} | lr: {:.5f}'.format(epoch + 1, lr_now)) ret_log = np.array([start_epoch]) head = np.array(['epoch']) # per epoch # lr_now, t_l, t_e, t_3d = train(train_loader, model, optimizer, input_n=input_n, # lr_now=lr_now, max_norm=opt.max_norm, is_cuda=is_cuda, # dim_used=train_dataset.dim_used, dct_n=dct_n) # ret_log = np.append(ret_log, [lr_now, t_l, t_e, t_3d]) # head = np.append(head, ['lr', 't_l', 't_e', 't_3d']) # # v_e, v_3d = val(val_loader, model, input_n=input_n, is_cuda=is_cuda, dim_used=train_dataset.dim_used, # dct_n=dct_n) # # ret_log = np.append(ret_log, [v_e, v_3d]) # head = np.append(head, ['v_e', 'v_3d']) test_3d_temp = np.array([]) test_3d_head = np.array([]) for act in acts: test_e, test_3d = test(test_data[act], model, stepsize, input_n=input_n, output_n=output_n, is_cuda=is_cuda, dim_used=dim_used, dct_n=dct_n) ret_log = np.append(ret_log, test_e) test_3d_temp = np.append(test_3d_temp, test_3d) test_3d_head = np.append( test_3d_head, [act + '3d80', act + '3d160', act + '3d320', act + '3d400']) head = np.append(head, [act + '80', act + '160', act + '320', act + '400']) if output_n > 10: if output_n == 25: head = np.append(head, [act + '560', act + '1000']) test_3d_head = np.append(test_3d_head, [act + '3d560', act + '3d1000']) if output_n == 50: head = np.append(head, [act + '560', act + '1000', act + '2000']) test_3d_head = np.append( test_3d_head, [act + '3d560', act + '3d1000', act + '3d2000']) if output_n == 100: head = np.append( head, [act + '560', act + '1000', act + '2000', act + '4000']) test_3d_head = np.append(test_3d_head, [ act + '3d560', act + '3d1000', act + '3d2000', act + '3d4000' ]) ret_log = np.append(ret_log, test_3d_temp) head = np.append(head, test_3d_head) # update log file and save checkpoint df = pd.DataFrame(np.expand_dims(ret_log, axis=0)) # if epoch == start_epoch: df.to_csv(opt.ckpt + '/' + script_name + '.csv', header=head, index=False)