# rotations = [cv2.Rodrigues(aa)[0] for aa in pose.reshape(-1, 3)]
# print('ROTATIONS:', rotations)
visualize(img_path, img, proc_param, joints[0], verts[0], cams[0])
def join_csv():
path = 'hmr/output/csv/'
all_files = glob.glob(os.path.join(path, "*.csv"))
all_files.sort(key=lambda x: int(x.split('/')[-1].split('.')[0]))
df_from_each_file = (pd.read_csv(f) for f in all_files)
concatenated_df = pd.concat(df_from_each_file, ignore_index=True)
concatenated_df['frame'] = concatenated_df.index+1
concatenated_df.to_csv("hmr/output/csv_joined/csv_joined.csv", index=False)
if __name__ == '__main__':
config = flags.FLAGS
config(sys.argv)
# Using pre-trained model, change this to use your own.
config.load_path = src.config.PRETRAINED_MODEL
config.batch_size = 1
renderer = vis_util.SMPLRenderer(face_path=config.smpl_face_path)
main(config.img_path, config.json_path)
join_csv()
print('\nResult is in hmr/output (you can open images in Colaboratory by double-clicking them)')
def main(config):
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
try:
# Currently, memory growth needs to be the same across GPUs
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
logical_gpus = tf.config.experimental.list_logical_devices('GPU')
print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
except RuntimeError as e:
# Memory growth must be set before GPUs have been initialized
print(e)
renderer = vis_util.SMPLRenderer(
img_size=config.img_size,
face_path=config.smpl_face_path)
config.checkpoint_dir = "training_checkpoints_125_epochs_lspe"
predictor = Predictor(config)
cv2.namedWindow("preview", cv2.WINDOW_NORMAL)
cv2.setWindowProperty("preview", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
vc = cv2.VideoCapture(0)
if vc.isOpened(): # try to get the first frame
rval, frame = vc.read()
else:
rval = False
draw_skel = False
draw_mesh = True
rotate_img = False
show_both = False
while rval:
frame = cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
input_img, proc_param, img = preprocess_image(frame, config)
verts, cam, joints = predictor.do_prediction(input_img)
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, np.squeeze(verts), np.squeeze(cam), np.squeeze(joints)[:,:2], img_size=frame.shape[:2])
if tf.math.is_nan(joints_orig).numpy().any():
print("nothing found")
rend_img = frame
else:
if draw_skel:
rend_img = vis_util.draw_skeleton(frame, joints_orig)
if draw_mesh:
if rotate_img:
rend_img = renderer.rotated(vert_shifted, 60, cam=cam_for_render, img_size=frame.shape[:2])
else:
rend_img = renderer(vert_shifted, cam_for_render, frame, True)
if show_both:
img2 = renderer.rotated(vert_shifted, 60, cam=cam_for_render, img_size=frame.shape[:2])
rend_img = np.concatenate((rend_img, img2), axis=1)
cv2.imshow("preview", rend_img)
for i in range(5):
rval, frame = vc.read()
key = cv2.waitKey(20)
if key == 27: # exit on ESC
break
if key == ord('s'):
draw_skel = True
draw_mesh = False
rotate_img = False
show_both = False
if key == ord('m'):
draw_skel = False
draw_mesh = True
rotate_img = False
show_both = False
if key == ord('r'):
draw_skel = False
draw_mesh = True
rotate_img = True
show_both = False
if key == ord('b'):
draw_skel = False
draw_mesh = True
rotate_img = False
show_both = True
cv2.destroyWindow("preview")
# Theta is the 85D vector holding [camera, pose, shape]
# where camera is 3D [s, tx, ty]
# pose is 72D vector holding the rotation of 24 joints of SMPL in axis angle format
# shape is 10D shape coefficients of SMPL
joints, verts, cams, joints3d, theta = model.predict(input_img,
get_theta=True)
visualize(img, proc_param, joints[0], verts[0], cams[0])
if __name__ == '__main__':
config = flags.FLAGS
config(sys.argv)
i = 0
for file in sorted(os.listdir(config.img_path)):
ext = os.path.splitext(file)[-1].lower()
if ext in extensions:
# Using pre-trained model, change this to use your own.
tf.reset_default_graph()
config.load_path = src.config.PRETRAINED_MODEL
config.batch_size = 1
renderer = vis_util.SMPLRenderer(glb_path=os.path.join(
config.glb_path, file),
face_path=config.smpl_face_path)
main(os.path.join(config.img_path, file), config.json_path)
renderer.trimesh.export(
os.path.join(config.glb_path,
str(i) + ".glb"))
i += 1
def predict(image, weight, height):
global config, renderer
config = flags.FLAGS
config(sys.argv)
# Using pre-trained model, change this to use your own.
config.load_path = src.config.PRETRAINED_MODEL
config.batch_size = 1
renderer = vis_util.SMPLRenderer(face_path=config.smpl_face_path)
tf.reset_default_graph()
sess = tf.Session()
model = RunModel(config, sess=sess)
input_img, proc_param, img = preprocess_image_V2(image)
# Add batch dimension: 1 x D x D x 3
input_img = np.expand_dims(input_img, 0)
joints, verts, cams, joints3d, theta = model.predict(input_img,
get_theta=True)
sess.close()
cams = theta[:, :model.num_cam]
poses = theta[:, model.num_cam:(model.num_cam + model.num_theta)]
shapes = theta[:, (model.num_cam + model.num_theta):]
viz_result = visualize(img, proc_param, joints[0], verts[0], cams[0])
'''
Start adjusting the shape
'''
shape_adjuster = torch.load("./trained/model_release_1.5961363467")
smpl = SMPL("./models/neutral_smpl_with_cocoplus_reg.pkl")
beta = torch.from_numpy(shapes).float().cuda()
theta = torch.zeros((1, 72)).float().cuda()
heights = torch.from_numpy(np.asarray([height]))
volume = torch.from_numpy(np.asarray([weight]))
verts, joints3d, Rs = smpl.forward(beta, theta, True)
flatten_joints3d = joints3d.view(1, -1)
heights = torch.unsqueeze(heights, -1).float().cuda()
volumes = torch.unsqueeze(volume, -1).float().cuda()
input_to_net = torch.cat((flatten_joints3d, heights, volumes), 1)
adjusted_betas = shape_adjuster.forward(input_to_net)
adjusted_verts, adjusted_joints3d, Rs = smpl.forward(
adjusted_betas, theta, True)
adjusted_heights = measure.compute_height(adjusted_verts)
adjusted_volumes = measure.compute_volume(adjusted_verts, smpl.f)
print(adjusted_heights, adjusted_volumes)
# debug_display_cloud(verts[0], joints3d[0], adjusted_verts[0], adjusted_joints3d[0])
# Change the posture for measurement
from measurement import POSE1
theta = torch.from_numpy(np.expand_dims(POSE1, 0)).float().cuda()
m_adjusted_verts, adjusted_joints3d, Rs = smpl.forward(
adjusted_betas, theta, True)
return viz_result, \
torch.squeeze(verts).detach().cpu().numpy(), \
torch.squeeze(adjusted_verts).detach().cpu().numpy(), \
torch.squeeze(m_adjusted_verts).detach().cpu().numpy(), \
torch.squeeze(adjusted_volumes).detach().cpu().numpy(),\
torch.squeeze(adjusted_heights).detach().cpu().numpy(),
def train(self):
# For rendering!
self.renderer = vis_util.SMPLRenderer(
img_size=self.img_size, face_path=self.config.smpl_face_path)
step = 0
with self.sv.managed_session(config=self.sess_config) as sess:
while not self.sv.should_stop():
fetch_dict = {
"summary": self.summary_op_always,
"step": self.global_step,
"e_loss": self.e_loss,
# The meat
"e_opt": self.e_opt,
"loss_kp": self.e_loss_kp
}
if not self.encoder_only:
fetch_dict.update({
# For D:
"d_opt": self.d_opt,
"d_loss": self.d_loss,
"loss_disc": self.e_loss_disc,
})
if self.use_3d_label:
fetch_dict.update({
"loss_3d_params": self.e_loss_3d,
"loss_3d_joints": self.e_loss_3d_joints
})
if step % self.log_img_step == 0:
fetch_dict.update({
"input_img": self.show_imgs,
"gt_kp": self.show_kps,
"e_verts": self.all_verts,
"joints": self.all_pred_kps,
"cam": self.all_pred_cams,
})
if not self.encoder_only:
fetch_dict.update(
{"summary_occasional": self.summary_op_occ})
t0 = time()
result = sess.run(fetch_dict)
t1 = time()
self.summary_writer.add_summary(result['summary'],
global_step=result['step'])
e_loss = result['e_loss']
step = result['step']
epoch = float(step) / self.num_itr_per_epoch
if self.encoder_only:
print("itr %d/(epoch %.1f): time %g, Enc_loss: %.4f" %
(step, epoch, t1 - t0, e_loss))
else:
d_loss = result['d_loss']
print(
"itr %d/(epoch %.1f): time %g, Enc_loss: %.4f, Disc_loss: %.4f"
% (step, epoch, t1 - t0, e_loss, d_loss))
if step % self.log_img_step == 0:
if not self.encoder_only:
self.summary_writer.add_summary(
result['summary_occasional'],
global_step=result['step'])
self.draw_results(result)
self.summary_writer.flush()
if epoch > self.max_epoch:
self.sv.request_stop()
step += 1
print('Finish training on %s' % self.model_dir)
plt.imshow(rend_img)
return fig
# define parser
parser = argparse.ArgumentParser()
parser.add_argument('num', type=int)
args = parser.parse_args()
# define some paths
smpl_model_path = '/gpfs/milgram/project/yildirim/hakan/hmr/models/neutral_smpl_with_cocoplus_reg.pkl'
smpl_face_path = '/gpfs/milgram/project/yildirim/hakan/hmr/src/tf_smpl/smpl_faces.npy'
# define SMPL model and renderer
smpl = SMPL(smpl_model_path)
renderer = vis_util.SMPLRenderer(face_path=smpl_face_path)
# define camera, position and shape parameters
# cam = np.load('latents/cam'+str(args.num)+'.npy').flatten()
cam = np.array([1., 0, 0], np.float32)
pose = np.load('latents/pose' + str(args.num) + '.npy')
shape = np.load('latents/shape' + str(args.num) + '.npy')
cam_tf = tf.Variable(cam)
pose = tf.Variable(pose)
shape = tf.Variable(shape)
# define pre-processing dict, no clue why they require it
proc_param = {
'end_pt': np.array([339, 339]),
'img_size': 230,
'scale': 0.9739130434782609,
smpl = SMPL('../../smpl/model_cocoplus.pkl', obj_saveable=True).to(device)
pose = get_theta('00001')
beta = get_beta('00001')
vbeta = torch.tensor(np.array([beta])).float().to(device)
vpose = torch.tensor(np.array([pose])).float().to(device)
vcam = torch.tensor([0.9, 0, 0]).expand((1, 3)).float().to(device)
verts, joints, _ = smpl(vbeta, vpose, get_skin=True)
pred_kp = batch_orth_proj_idrot(joints.cpu(), vcam.cpu())
r = torch.ones((1, 3))
verts, joints, r = verts.cpu().numpy(), pred_kp.cpu().numpy(), vcam.cpu(
).numpy()
print(img.shape)
print(type(joints), type(verts), type(r))
print(joints[0].shape, verts[0].shape, r[0].shape)
visualize(img, proc_param, joints[0], verts[0], r[0])
if __name__ == '__main__':
renderer = vis_util.SMPLRenderer(face_path='../../smpl/smpl_faces.npy')
main(img_path='../../up3d/00001_image.png')
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = rnd.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
rend_img_overlay = renderer(
vert_shifted, cam=cam_for_render, img=img, do_alpha=True)
return rend_img_overlay
if __name__ == '__main__':
config = flags.FLAGS
config(sys.argv)
# Using pre-trained model, change this to use your own.
config.load_path = src.config.PRETRAINED_MODEL
config.batch_size = 1
renderer = rnd.SMPLRenderer(face_path=config.smpl_face_path)
cam = cv2.VideoCapture(0)
yolo_sess = tf.Session()
yolo = YOLO(yolo_sess=yolo_sess)
hmr_graph = tf.Graph()
hmr_sess = tf.Session(graph=hmr_graph)
with hmr_graph.as_default():
model = RunModel(config, sess=hmr_sess)
while True:
_, frame = cam.read()
with yolo_sess.as_default():
