def main(**args):
input_media = args.pop('input_media')
config = easy_configuration.configure(**args)
# supongo que es una imagen
img_np = cv2.imread(input_media)
img = get_img(input_media)
keypoints = get_keypoints(img_np)
mesh_viewer_background.start_render(img.shape[1],img.shape[0])
for person_id in range(keypoints.shape[0]):
# assumes neutral gender.
body_model = config['neutral_model']
out_img_fn = 'temp.png'
fit_single_frame(img, keypoints[[person_id]],
body_model=body_model,
camera=config['camera'],
joint_weights=config['joint_weights'],
shape_prior=config['shape_prior'],
expr_prior=config['expr_prior'],
body_pose_prior=config['body_pose_prior'],
left_hand_prior=config['left_hand_prior'],
right_hand_prior=config['right_hand_prior'],
jaw_prior=config['jaw_prior'],
angle_prior=config['angle_prior'],
out_img_fn= out_img_fn,
**args)
# fit_single_frame saves partial detection result to 'temp.png',
# here we load that image again so as to render the next detection on top of that image again
img = cv2.imread(out_img_fn).astype(np.float32)[:, :, ::-1] / 255.0
# show the frame to our screen
cv2.imshow("Frame", img)
# cv2.imshow("Frame", mask)
key = cv2.waitKey(0)
mesh_viewer_background.finish_render()
def main(**args):
data_folder = args.get('recording_dir')
recording_name = osp.basename(args.get('recording_dir'))
scene_name = recording_name.split("_")[0]
base_dir = os.path.abspath(
osp.join(args.get('recording_dir'), os.pardir, os.pardir))
keyp_dir = osp.join(base_dir, 'keypoints')
keyp_folder = osp.join(keyp_dir, recording_name)
cam2world_dir = osp.join(base_dir, 'cam2world')
scene_dir = osp.join(base_dir, 'scenes')
calib_dir = osp.join(base_dir, 'calibration')
sdf_dir = osp.join(base_dir, 'sdf')
body_segments_dir = osp.join(base_dir, 'body_segments')
output_folder = args.get('output_folder')
output_folder = osp.expandvars(output_folder)
output_folder = osp.join(output_folder, recording_name)
if not osp.exists(output_folder):
os.makedirs(output_folder)
# Store the arguments for the current experiment
conf_fn = osp.join(output_folder, 'conf.yaml')
with open(conf_fn, 'w') as conf_file:
yaml.dump(args, conf_file)
#remove 'output_folder' from args list
args.pop('output_folder')
result_folder = args.pop('result_folder', 'results')
result_folder = osp.join(output_folder, result_folder)
if not osp.exists(result_folder):
os.makedirs(result_folder)
mesh_folder = args.pop('mesh_folder', 'meshes')
mesh_folder = osp.join(output_folder, mesh_folder)
if not osp.exists(mesh_folder):
os.makedirs(mesh_folder)
out_img_folder = osp.join(output_folder, 'images')
if not osp.exists(out_img_folder):
os.makedirs(out_img_folder)
body_scene_rendering_dir = os.path.join(output_folder, 'renderings')
if not osp.exists(body_scene_rendering_dir):
os.mkdir(body_scene_rendering_dir)
float_dtype = args['float_dtype']
if float_dtype == 'float64':
dtype = torch.float64
elif float_dtype == 'float32':
dtype = torch.float64
else:
print('Unknown float type {}, exiting!'.format(float_dtype))
sys.exit(-1)
use_cuda = args.get('use_cuda', True)
if use_cuda and not torch.cuda.is_available():
print('CUDA is not available, exiting!')
sys.exit(-1)
img_folder = args.pop('img_folder', 'Color')
dataset_obj = create_dataset(img_folder=img_folder,
data_folder=data_folder,
keyp_folder=keyp_folder,
calib_dir=calib_dir,
**args)
start = time.time()
input_gender = args.pop('gender', 'neutral')
gender_lbl_type = args.pop('gender_lbl_type', 'none')
max_persons = args.pop('max_persons', -1)
float_dtype = args.get('float_dtype', 'float32')
if float_dtype == 'float64':
dtype = torch.float64
elif float_dtype == 'float32':
dtype = torch.float32
else:
raise ValueError('Unknown float type {}, exiting!'.format(float_dtype))
joint_mapper = JointMapper(dataset_obj.get_model2data())
model_params = dict(model_path=args.get('model_folder'),
joint_mapper=joint_mapper,
create_global_orient=True,
create_body_pose=not args.get('use_vposer'),
create_betas=True,
create_left_hand_pose=True,
create_right_hand_pose=True,
create_expression=True,
create_jaw_pose=True,
create_leye_pose=True,
create_reye_pose=True,
create_transl=True,
dtype=dtype,
**args)
male_model = smplx.create(gender='male', **model_params)
# SMPL-H has no gender-neutral model
if args.get('model_type') != 'smplh':
neutral_model = smplx.create(gender='neutral', **model_params)
female_model = smplx.create(gender='female', **model_params)
# Create the camera object
camera_center = None \
if args.get('camera_center_x') is None or args.get('camera_center_y') is None \
else torch.tensor([args.get('camera_center_x'), args.get('camera_center_y')], dtype=dtype).view(-1, 2)
camera = create_camera(focal_length_x=args.get('focal_length_x'),
focal_length_y=args.get('focal_length_y'),
center=camera_center,
batch_size=args.get('batch_size'),
dtype=dtype)
if hasattr(camera, 'rotation'):
camera.rotation.requires_grad = False
use_hands = args.get('use_hands', True)
use_face = args.get('use_face', True)
body_pose_prior = create_prior(prior_type=args.get('body_prior_type'),
dtype=dtype,
**args)
jaw_prior, expr_prior = None, None
if use_face:
jaw_prior = create_prior(prior_type=args.get('jaw_prior_type'),
dtype=dtype,
**args)
expr_prior = create_prior(prior_type=args.get('expr_prior_type', 'l2'),
dtype=dtype,
**args)
left_hand_prior, right_hand_prior = None, None
if use_hands:
lhand_args = args.copy()
lhand_args['num_gaussians'] = args.get('num_pca_comps')
left_hand_prior = create_prior(
prior_type=args.get('left_hand_prior_type'),
dtype=dtype,
use_left_hand=True,
**lhand_args)
rhand_args = args.copy()
rhand_args['num_gaussians'] = args.get('num_pca_comps')
right_hand_prior = create_prior(
prior_type=args.get('right_hand_prior_type'),
dtype=dtype,
use_right_hand=True,
**rhand_args)
shape_prior = create_prior(prior_type=args.get('shape_prior_type', 'l2'),
dtype=dtype,
**args)
angle_prior = create_prior(prior_type='angle', dtype=dtype)
if use_cuda and torch.cuda.is_available():
device = torch.device('cuda')
camera = camera.to(device=device)
female_model = female_model.to(device=device)
male_model = male_model.to(device=device)
if args.get('model_type') != 'smplh':
neutral_model = neutral_model.to(device=device)
body_pose_prior = body_pose_prior.to(device=device)
angle_prior = angle_prior.to(device=device)
shape_prior = shape_prior.to(device=device)
if use_face:
expr_prior = expr_prior.to(device=device)
jaw_prior = jaw_prior.to(device=device)
if use_hands:
left_hand_prior = left_hand_prior.to(device=device)
right_hand_prior = right_hand_prior.to(device=device)
else:
device = torch.device('cpu')
# A weight for every joint of the model
joint_weights = dataset_obj.get_joint_weights().to(device=device,
dtype=dtype)
# Add a fake batch dimension for broadcasting
joint_weights.unsqueeze_(dim=0)
for idx, data in enumerate(dataset_obj):
img = data['img']
fn = data['fn']
keypoints = data['keypoints']
depth_im = data['depth_im']
mask = data['mask']
init_trans = None if data['init_trans'] is None else torch.tensor(
data['init_trans'], dtype=dtype).view(-1, 3)
scan = data['scan_dict']
print('Processing: {}'.format(data['img_path']))
curr_result_folder = osp.join(result_folder, fn)
if not osp.exists(curr_result_folder):
os.makedirs(curr_result_folder)
curr_mesh_folder = osp.join(mesh_folder, fn)
if not osp.exists(curr_mesh_folder):
os.makedirs(curr_mesh_folder)
#TODO: SMPLifyD and PROX won't work for multiple persons
for person_id in range(keypoints.shape[0]):
if person_id >= max_persons and max_persons > 0:
continue
curr_result_fn = osp.join(curr_result_folder,
'{:03d}.pkl'.format(person_id))
curr_mesh_fn = osp.join(curr_mesh_folder,
'{:03d}.ply'.format(person_id))
curr_body_scene_rendering_fn = osp.join(body_scene_rendering_dir,
fn + '.png')
curr_img_folder = osp.join(output_folder, 'images', fn,
'{:03d}'.format(person_id))
if not osp.exists(curr_img_folder):
os.makedirs(curr_img_folder)
if gender_lbl_type != 'none':
if gender_lbl_type == 'pd' and 'gender_pd' in data:
gender = data['gender_pd'][person_id]
if gender_lbl_type == 'gt' and 'gender_gt' in data:
gender = data['gender_gt'][person_id]
else:
gender = input_gender
if gender == 'neutral':
body_model = neutral_model
elif gender == 'female':
body_model = female_model
elif gender == 'male':
body_model = male_model
out_img_fn = osp.join(curr_img_folder, 'output.png')
fit_single_frame(
img,
keypoints[[person_id]],
init_trans,
scan,
cam2world_dir=cam2world_dir,
scene_dir=scene_dir,
sdf_dir=sdf_dir,
body_segments_dir=body_segments_dir,
scene_name=scene_name,
body_model=body_model,
camera=camera,
joint_weights=joint_weights,
dtype=dtype,
output_folder=output_folder,
result_folder=curr_result_folder,
out_img_fn=out_img_fn,
result_fn=curr_result_fn,
mesh_fn=curr_mesh_fn,
body_scene_rendering_fn=curr_body_scene_rendering_fn,
shape_prior=shape_prior,
expr_prior=expr_prior,
body_pose_prior=body_pose_prior,
left_hand_prior=left_hand_prior,
right_hand_prior=right_hand_prior,
jaw_prior=jaw_prior,
angle_prior=angle_prior,
**args)
elapsed = time.time() - start
time_msg = time.strftime('%H hours, %M minutes, %S seconds',
time.gmtime(elapsed))
print('Processing the data took: {}'.format(time_msg))
def main(**args):
output_folder = args.pop('output_folder')
output_folder = osp.expandvars(output_folder)
if not osp.exists(output_folder):
os.makedirs(output_folder)
# Store the arguments for the current experiment
conf_fn = osp.join(output_folder, 'conf.yaml')
with open(conf_fn, 'w') as conf_file:
yaml.dump(args, conf_file)
result_folder = args.pop('result_folder', 'results')
result_folder = osp.join(output_folder, result_folder)
if not osp.exists(result_folder):
os.makedirs(result_folder)
mesh_folder = args.pop('mesh_folder', 'meshes')
mesh_folder = osp.join(output_folder, mesh_folder)
if not osp.exists(mesh_folder):
os.makedirs(mesh_folder)
out_img_folder = osp.join(output_folder, 'images')
if not osp.exists(out_img_folder):
os.makedirs(out_img_folder)
float_dtype = args['float_dtype']
if float_dtype == 'float64':
dtype = torch.float64
elif float_dtype == 'float32':
dtype = torch.float64
else:
print('Unknown float type {}, exiting!'.format(float_dtype))
sys.exit(-1)
use_cuda = args.get('use_cuda', True)
if use_cuda and not torch.cuda.is_available():
print('CUDA is not available, exiting!')
sys.exit(-1)
img_folder = args.pop('img_folder', 'images')
dataset_obj = create_dataset(img_folder=img_folder, **args)
start = time.time()
input_gender = args.pop('gender', 'neutral')
gender_lbl_type = args.pop('gender_lbl_type', 'none')
max_persons = args.pop('max_persons', -1)
float_dtype = args.get('float_dtype', 'float32')
if float_dtype == 'float64':
dtype = torch.float64
elif float_dtype == 'float32':
dtype = torch.float32
else:
raise ValueError('Unknown float type {}, exiting!'.format(float_dtype))
joint_mapper = JointMapper(dataset_obj.get_model2data())
model_params = dict(model_path=args.get('model_folder'),
joint_mapper=joint_mapper,
create_global_orient=True,
create_body_pose=not args.get('use_vposer'),
create_betas=True,
create_left_hand_pose=True,
create_right_hand_pose=True,
create_expression=True,
create_jaw_pose=True,
create_leye_pose=True,
create_reye_pose=True,
create_transl=False,
dtype=dtype,
**args)
male_model = smplx.create(gender='male', **model_params)
# SMPL-H has no gender-neutral model
if args.get('model_type') != 'smplh':
neutral_model = smplx.create(gender='neutral', **model_params)
female_model = smplx.create(gender='female', **model_params)
# Create the camera object
focal_length = args.get('focal_length')
camera = create_camera(focal_length_x=focal_length,
focal_length_y=focal_length,
dtype=dtype,
**args)
if hasattr(camera, 'rotation'):
camera.rotation.requires_grad = False
use_hands = args.get('use_hands', True)
use_face = args.get('use_face', True)
body_pose_prior = create_prior(prior_type=args.get('body_prior_type'),
dtype=dtype,
**args)
jaw_prior, expr_prior = None, None
if use_face:
jaw_prior = create_prior(prior_type=args.get('jaw_prior_type'),
dtype=dtype,
**args)
expr_prior = create_prior(prior_type=args.get('expr_prior_type', 'l2'),
dtype=dtype,
**args)
left_hand_prior, right_hand_prior = None, None
if use_hands:
lhand_args = args.copy()
lhand_args['num_gaussians'] = args.get('num_pca_comps')
left_hand_prior = create_prior(
prior_type=args.get('left_hand_prior_type'),
dtype=dtype,
use_left_hand=True,
**lhand_args)
rhand_args = args.copy()
rhand_args['num_gaussians'] = args.get('num_pca_comps')
right_hand_prior = create_prior(
prior_type=args.get('right_hand_prior_type'),
dtype=dtype,
use_right_hand=True,
**rhand_args)
shape_prior = create_prior(prior_type=args.get('shape_prior_type', 'l2'),
dtype=dtype,
**args)
angle_prior = create_prior(prior_type='angle', dtype=dtype)
if use_cuda and torch.cuda.is_available():
device = torch.device('cuda')
camera = camera.to(device=device)
female_model = female_model.to(device=device)
male_model = male_model.to(device=device)
if args.get('model_type') != 'smplh':
neutral_model = neutral_model.to(device=device)
body_pose_prior = body_pose_prior.to(device=device)
angle_prior = angle_prior.to(device=device)
shape_prior = shape_prior.to(device=device)
if use_face:
expr_prior = expr_prior.to(device=device)
jaw_prior = jaw_prior.to(device=device)
if use_hands:
left_hand_prior = left_hand_prior.to(device=device)
right_hand_prior = right_hand_prior.to(device=device)
else:
device = torch.device('cpu')
# A weight for every joint of the model
joint_weights = dataset_obj.get_joint_weights().to(device=device,
dtype=dtype)
# Add a fake batch dimension for broadcasting
joint_weights.unsqueeze_(dim=0)
for idx, data in enumerate(dataset_obj):
img = data['img']
fn = data['fn']
keypoints = data['keypoints']
print('Processing: {}'.format(data['img_path']))
curr_result_folder = osp.join(result_folder, fn)
if not osp.exists(curr_result_folder):
os.makedirs(curr_result_folder)
curr_mesh_folder = osp.join(mesh_folder, fn)
if not osp.exists(curr_mesh_folder):
os.makedirs(curr_mesh_folder)
for person_id in range(keypoints.shape[0]):
if person_id >= max_persons and max_persons > 0:
continue
curr_result_fn = osp.join(curr_result_folder,
'{:03d}.pkl'.format(person_id))
curr_mesh_fn = osp.join(curr_mesh_folder,
'{:03d}.obj'.format(person_id))
curr_img_folder = osp.join(output_folder, 'images', fn,
'{:03d}'.format(person_id))
if not osp.exists(curr_img_folder):
os.makedirs(curr_img_folder)
if gender_lbl_type != 'none':
if gender_lbl_type == 'pd' and 'gender_pd' in data:
gender = data['gender_pd'][person_id]
if gender_lbl_type == 'gt' and 'gender_gt' in data:
gender = data['gender_gt'][person_id]
else:
gender = input_gender
if gender == 'neutral':
body_model = neutral_model
elif gender == 'female':
body_model = female_model
elif gender == 'male':
body_model = male_model
try:
out_img_fn = osp.join(curr_img_folder,
'{:06d}.png'.format(int(fn)))
except:
out_img_fn = osp.join(curr_img_folder, 'output.png')
fit_single_frame(img,
keypoints[[person_id]],
body_model=body_model,
camera=camera,
joint_weights=joint_weights,
dtype=dtype,
output_folder=output_folder,
result_folder=curr_result_folder,
out_img_fn=out_img_fn,
result_fn=curr_result_fn,
mesh_fn=curr_mesh_fn,
shape_prior=shape_prior,
expr_prior=expr_prior,
body_pose_prior=body_pose_prior,
left_hand_prior=left_hand_prior,
right_hand_prior=right_hand_prior,
jaw_prior=jaw_prior,
angle_prior=angle_prior,
**args)
elapsed = time.time() - start
time_msg = time.strftime('%H hours, %M minutes, %S seconds',
time.gmtime(elapsed))
print('Processing the data took: {}'.format(time_msg))
def main(**args):
output_folder = args.pop('output_folder')
output_folder = osp.expandvars(output_folder)
if not osp.exists(output_folder):
os.makedirs(output_folder)
# Store the arguments for the current experiment
conf_fn = osp.join(output_folder, 'conf.yaml')
with open(conf_fn, 'w') as conf_file:
yaml.dump(args, conf_file)
result_folder = args.pop('result_folder', 'results')
result_folder = osp.join(output_folder, result_folder)
if not osp.exists(result_folder):
os.makedirs(result_folder)
mesh_folder = args.pop('mesh_folder', 'meshes')
mesh_folder = osp.join(output_folder, mesh_folder)
if not osp.exists(mesh_folder):
os.makedirs(mesh_folder)
float_dtype = args['float_dtype']
if float_dtype == 'float64':
dtype = torch.float64
elif float_dtype == 'float32':
dtype = torch.float64
else:
print('Unknown float type {}, exiting!'.format(float_dtype))
sys.exit(-1)
use_cuda = args.get('use_cuda', True)
if use_cuda and not torch.cuda.is_available():
print('CUDA is not available, exiting!')
sys.exit(-1)
dataset_obj = create_dataset(**args)
start = time.time()
input_gender = args.pop('gender', 'neutral')
gender_lbl_type = args.pop('gender_lbl_type', 'none')
max_persons = args.pop('max_persons', -1)
float_dtype = args.get('float_dtype', 'float32')
if float_dtype == 'float64':
dtype = torch.float64
elif float_dtype == 'float32':
dtype = torch.float32
else:
raise ValueError('Unknown float type {}, exiting!'.format(float_dtype))
body_model = SMPL(args.get('model_folder'), dtype=dtype, joint_regressor_extra=args.get('j_regressor_extra'))
use_hands = args.get('use_hands', True)
use_face = args.get('use_face', True)
body_pose_prior = create_prior(
prior_type=args.get('body_prior_type'),
dtype=dtype,
**args)
jaw_prior, expr_prior = None, None
if use_face:
jaw_prior = create_prior(
prior_type=args.get('jaw_prior_type'),
dtype=dtype,
**args)
expr_prior = create_prior(
prior_type=args.get('expr_prior_type', 'l2'),
dtype=dtype, **args)
left_hand_prior, right_hand_prior = None, None
if use_hands:
lhand_args = args.copy()
lhand_args['num_gaussians'] = args.get('num_pca_comps')
left_hand_prior = create_prior(
prior_type=args.get('left_hand_prior_type'),
dtype=dtype,
use_left_hand=True,
**lhand_args)
rhand_args = args.copy()
rhand_args['num_gaussians'] = args.get('num_pca_comps')
right_hand_prior = create_prior(
prior_type=args.get('right_hand_prior_type'),
dtype=dtype,
use_right_hand=True,
**rhand_args)
shape_prior = create_prior(
prior_type=args.get('shape_prior_type', 'l2'),
dtype=dtype, **args)
angle_prior = create_prior(prior_type='angle', dtype=dtype)
if use_cuda and torch.cuda.is_available():
device = torch.device('cuda')
body_model = body_model.to(device=device)
body_pose_prior = body_pose_prior.to(device=device)
angle_prior = angle_prior.to(device=device)
shape_prior = shape_prior.to(device=device)
if use_face:
expr_prior = expr_prior.to(device=device)
jaw_prior = jaw_prior.to(device=device)
if use_hands:
left_hand_prior = left_hand_prior.to(device=device)
right_hand_prior = right_hand_prior.to(device=device)
else:
device = torch.device('cpu')
# A weight for every joint of the model
joint_weights = dataset_obj.get_joint_weights().to(device=device,
dtype=dtype)
# Add a fake batch dimension for broadcasting
joint_weights.unsqueeze_(dim=0)
for idx, keypoints_3d in enumerate(dataset_obj):
curr_result_folder = os.path.join(result_folder, str(idx).zfill(7))
if not osp.exists(curr_result_folder):
os.makedirs(curr_result_folder)
curr_mesh_folder = os.path.join(mesh_folder, str(idx).zfill(7))
if not osp.exists(curr_mesh_folder):
os.makedirs(curr_mesh_folder)
for person_idx in range(len(keypoints_3d)):
print(f'Processing: Image {idx} Person {person_idx}')
curr_result_fn = osp.join(curr_result_folder, f'{person_idx:02d}.pkl')
curr_mesh_fn = osp.join(curr_mesh_folder, f'{person_idx:02d}.obj')
try:
fit_single_frame(keypoints_3d[[person_idx]],
body_model=body_model,
joint_weights=joint_weights,
dtype=dtype,
result_fn=curr_result_fn,
mesh_fn=curr_mesh_fn,
shape_prior=shape_prior,
expr_prior=expr_prior,
body_pose_prior=body_pose_prior,
left_hand_prior=left_hand_prior,
right_hand_prior=right_hand_prior,
jaw_prior=jaw_prior,
angle_prior=angle_prior,
**args)
elapsed = time.time() - start
time_msg = time.strftime('%H hours, %M minutes, %S seconds',
time.gmtime(elapsed))
print('Processing the data took: {}'.format(time_msg))
except:
print('Some error occured during fitting')