def main(args): print('==> Using settings {}'.format(args)) convm = torch.zeros(3, 17, 17, dtype=torch.float) print('==> Loading dataset...') dataset_path = path.join('data', 'data_3d_' + args.dataset + '.npz') if args.dataset == 'h36m': from common.h36m_dataset import Human36mDataset dataset = Human36mDataset(dataset_path) else: raise KeyError('Invalid dataset') print('==> Preparing data...') dataset = read_3d_data(dataset) print('==> Loading 2D detections...') keypoints = create_2d_data( path.join('data', 'data_2d_' + args.dataset + '_' + args.keypoints + '.npz'), dataset) cudnn.benchmark = True device = torch.device("cuda") # Create model print("==> Creating model...") if args.architecture == 'linear': from models.linear_model import LinearModel, init_weights num_joints = dataset.skeleton().num_joints() model_pos = LinearModel(num_joints * 2, (num_joints - 1) * 3).to(device) model_pos.apply(init_weights) elif args.architecture == 'gcn': from models.sem_gcn import SemGCN from common.graph_utils import adj_mx_from_skeleton p_dropout = (None if args.dropout == 0.0 else args.dropout) adj = adj_mx_from_skeleton(dataset.skeleton()) model_pos = SemGCN(convm, adj, args.hid_dim, num_layers=args.num_layers, p_dropout=p_dropout, nodes_group=dataset.skeleton().joints_group() if args.non_local else None).to(device) else: raise KeyError('Invalid model architecture') print("==> Total parameters: {:.2f}M".format( sum(p.numel() for p in model_pos.parameters()) / 1000000.0)) # Resume from a checkpoint ckpt_path = args.evaluate if path.isfile(ckpt_path): print("==> Loading checkpoint '{}'".format(ckpt_path)) ckpt = torch.load(ckpt_path) start_epoch = ckpt['epoch'] error_best = ckpt['error'] model_pos.load_state_dict(ckpt['state_dict']) print("==> Loaded checkpoint (Epoch: {} | Error: {})".format( start_epoch, error_best)) else: raise RuntimeError("==> No checkpoint found at '{}'".format(ckpt_path)) print('==> Rendering...') poses_2d = keypoints[args.viz_subject][args.viz_action] out_poses_2d = poses_2d[args.viz_camera] out_actions = [args.viz_camera] * out_poses_2d.shape[0] poses_3d = dataset[args.viz_subject][args.viz_action]['positions_3d'] assert len(poses_3d) == len(poses_2d), 'Camera count mismatch' out_poses_3d = poses_3d[args.viz_camera] ground_truth = dataset[args.viz_subject][args.viz_action]['positions_3d'][ args.viz_camera].copy() input_keypoints = out_poses_2d.copy() render_loader = DataLoader(PoseGenerator([out_poses_3d], [out_poses_2d], [out_actions]), batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=True) prediction = evaluate(render_loader, model_pos, device, args.architecture)[0] # Invert camera transformation cam = dataset.cameras()[args.viz_subject][args.viz_camera] prediction = camera_to_world(prediction, R=cam['orientation'], t=0) prediction[:, :, 2] -= np.min(prediction[:, :, 2]) ground_truth = camera_to_world(ground_truth, R=cam['orientation'], t=0) ground_truth[:, :, 2] -= np.min(ground_truth[:, :, 2]) anim_output = {'Regression': prediction, 'Ground truth': ground_truth} input_keypoints = image_coordinates(input_keypoints[..., :2], w=cam['res_w'], h=cam['res_h']) render_animation(input_keypoints, anim_output, dataset.skeleton(), dataset.fps(), args.viz_bitrate, cam['azimuth'], args.viz_output, limit=args.viz_limit, downsample=args.viz_downsample, size=args.viz_size, input_video_path=args.viz_video, viewport=(cam['res_w'], cam['res_h']), input_video_skip=args.viz_skip)
def main(args): print('==> Using settings {}'.format(args)) print('==> Loading dataset...') dataset_path = path.join('data', 'data_3d_' + args.dataset + '.npz') if args.dataset == 'h36m': from common.h36m_dataset import Human36mDataset, TRAIN_SUBJECTS, TEST_SUBJECTS dataset = Human36mDataset(dataset_path) subjects_train = TRAIN_SUBJECTS subjects_test = TEST_SUBJECTS else: raise KeyError('Invalid dataset') print('==> Preparing data...') dataset = read_3d_data(dataset) print('==> Loading 2D detections...') keypoints = create_2d_data(path.join('data', 'data_2d_' + args.dataset + '_' + args.keypoints + '.npz'), dataset) action_filter = None if args.actions == '*' else args.actions.split(',') if action_filter is not None: action_filter = map(lambda x: dataset.define_actions(x)[0], action_filter) print('==> Selected actions: {}'.format(action_filter)) stride = args.downsample cudnn.benchmark = True device = torch.device("cpu") ############################# cuda!!!!!!!!!!!!!!!!!!!!!!!!!!!! # Create model print("==> Creating model...") num_joints = dataset.skeleton().num_joints() model_pos = LinearModel(num_joints * 2, (num_joints - 1) * 3).to(device) model_pos.apply(init_weights) print("==> Total parameters: {:.2f}M".format(sum(p.numel() for p in model_pos.parameters()) / 1000000.0)) criterion = nn.MSELoss(reduction='mean').to(device) optimizer = torch.optim.Adam(model_pos.parameters(), lr=args.lr) # Optionally resume from a checkpoint if args.resume or args.evaluate: ckpt_path = (args.resume if args.resume else args.evaluate) if path.isfile(ckpt_path): print("==> Loading checkpoint '{}'".format(ckpt_path)) ckpt = torch.load(ckpt_path, map_location='cpu') start_epoch = ckpt['epoch'] error_best = ckpt['error'] glob_step = ckpt['step'] lr_now = ckpt['lr'] model_pos.load_state_dict(ckpt['state_dict']) optimizer.load_state_dict(ckpt['optimizer']) print("==> Loaded checkpoint (Epoch: {} | Error: {})".format(start_epoch, error_best)) if args.resume: ckpt_dir_path = path.dirname(ckpt_path) logger = Logger(path.join(ckpt_dir_path, 'log.txt'), resume=True) else: raise RuntimeError("==> No checkpoint found at '{}'".format(ckpt_path)) else: start_epoch = 0 error_best = None glob_step = 0 lr_now = args.lr ckpt_dir_path = path.join(args.checkpoint, datetime.datetime.now().isoformat()) if not path.exists(ckpt_dir_path): os.makedirs(ckpt_dir_path) print('==> Making checkpoint dir: {}'.format(ckpt_dir_path)) logger = Logger(os.path.join(ckpt_dir_path, 'log.txt')) logger.set_names(['epoch', 'lr', 'loss_train', 'error_eval_p1', 'error_eval_p2']) if args.evaluate: print('==> Evaluating...') if action_filter is None: action_filter = dataset.define_actions() errors_p1 = np.zeros(len(action_filter)) errors_p2 = np.zeros(len(action_filter)) for i, action in enumerate(action_filter): poses_valid, poses_valid_2d, actions_valid = fetch(subjects_test, dataset, keypoints, [action], stride) valid_loader = DataLoader(PoseGenerator(poses_valid, poses_valid_2d, actions_valid), batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=True) errors_p1[i], errors_p2[i] = evaluate(valid_loader, model_pos, device) print('Protocol #1 (MPJPE) action-wise average: {:.2f} (mm)'.format(np.mean(errors_p1).item())) print('Protocol #2 (P-MPJPE) action-wise average: {:.2f} (mm)'.format(np.mean(errors_p2).item())) exit(0) poses_train, poses_train_2d, actions_train = fetch(subjects_train, dataset, keypoints, action_filter, stride) train_loader = DataLoader(PoseGenerator(poses_train, poses_train_2d, actions_train), batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True) poses_valid, poses_valid_2d, actions_valid = fetch(subjects_test, dataset, keypoints, action_filter, stride) valid_loader = DataLoader(PoseGenerator(poses_valid, poses_valid_2d, actions_valid), batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=True) for epoch in range(start_epoch, args.epochs): print('\nEpoch: %d | LR: %.8f' % (epoch + 1, lr_now)) # Train for one epoch epoch_loss, lr_now, glob_step = train(train_loader, model_pos, criterion, optimizer, device, args.lr, lr_now, glob_step, args.lr_decay, args.lr_gamma, max_norm=args.max_norm) # Evaluate error_eval_p1, error_eval_p2 = evaluate(valid_loader, model_pos, device) # Update log file logger.append([epoch + 1, lr_now, epoch_loss, error_eval_p1, error_eval_p2]) # Save checkpoint if error_best is None or error_best > error_eval_p1: error_best = error_eval_p1 save_ckpt({'epoch': epoch + 1, 'lr': lr_now, 'step': glob_step, 'state_dict': model_pos.state_dict(), 'optimizer': optimizer.state_dict(), 'error': error_eval_p1}, ckpt_dir_path, suffix='best') if (epoch + 1) % args.snapshot == 0: save_ckpt({'epoch': epoch + 1, 'lr': lr_now, 'step': glob_step, 'state_dict': model_pos.state_dict(), 'optimizer': optimizer.state_dict(), 'error': error_eval_p1}, ckpt_dir_path) logger.close() logger.plot(['loss_train', 'error_eval_p1']) savefig(path.join(ckpt_dir_path, 'log.eps')) return