# rotations = [cv2.Rodrigues(aa)[0] for aa in pose.reshape(-1, 3)] # print('ROTATIONS:', rotations) visualize(img_path, img, proc_param, joints[0], verts[0], cams[0]) def join_csv(): path = 'hmr/output/csv/' all_files = glob.glob(os.path.join(path, "*.csv")) all_files.sort(key=lambda x: int(x.split('/')[-1].split('.')[0])) df_from_each_file = (pd.read_csv(f) for f in all_files) concatenated_df = pd.concat(df_from_each_file, ignore_index=True) concatenated_df['frame'] = concatenated_df.index+1 concatenated_df.to_csv("hmr/output/csv_joined/csv_joined.csv", index=False) if __name__ == '__main__': config = flags.FLAGS config(sys.argv) # Using pre-trained model, change this to use your own. config.load_path = src.config.PRETRAINED_MODEL config.batch_size = 1 renderer = vis_util.SMPLRenderer(face_path=config.smpl_face_path) main(config.img_path, config.json_path) join_csv() print('\nResult is in hmr/output (you can open images in Colaboratory by double-clicking them)')
def main(config): gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: try: # Currently, memory growth needs to be the same across GPUs for gpu in gpus: tf.config.experimental.set_memory_growth(gpu, True) logical_gpus = tf.config.experimental.list_logical_devices('GPU') print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs") except RuntimeError as e: # Memory growth must be set before GPUs have been initialized print(e) renderer = vis_util.SMPLRenderer( img_size=config.img_size, face_path=config.smpl_face_path) config.checkpoint_dir = "training_checkpoints_125_epochs_lspe" predictor = Predictor(config) cv2.namedWindow("preview", cv2.WINDOW_NORMAL) cv2.setWindowProperty("preview", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) vc = cv2.VideoCapture(0) if vc.isOpened(): # try to get the first frame rval, frame = vc.read() else: rval = False draw_skel = False draw_mesh = True rotate_img = False show_both = False while rval: frame = cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE) input_img, proc_param, img = preprocess_image(frame, config) verts, cam, joints = predictor.do_prediction(input_img) cam_for_render, vert_shifted, joints_orig = vis_util.get_original( proc_param, np.squeeze(verts), np.squeeze(cam), np.squeeze(joints)[:,:2], img_size=frame.shape[:2]) if tf.math.is_nan(joints_orig).numpy().any(): print("nothing found") rend_img = frame else: if draw_skel: rend_img = vis_util.draw_skeleton(frame, joints_orig) if draw_mesh: if rotate_img: rend_img = renderer.rotated(vert_shifted, 60, cam=cam_for_render, img_size=frame.shape[:2]) else: rend_img = renderer(vert_shifted, cam_for_render, frame, True) if show_both: img2 = renderer.rotated(vert_shifted, 60, cam=cam_for_render, img_size=frame.shape[:2]) rend_img = np.concatenate((rend_img, img2), axis=1) cv2.imshow("preview", rend_img) for i in range(5): rval, frame = vc.read() key = cv2.waitKey(20) if key == 27: # exit on ESC break if key == ord('s'): draw_skel = True draw_mesh = False rotate_img = False show_both = False if key == ord('m'): draw_skel = False draw_mesh = True rotate_img = False show_both = False if key == ord('r'): draw_skel = False draw_mesh = True rotate_img = True show_both = False if key == ord('b'): draw_skel = False draw_mesh = True rotate_img = False show_both = True cv2.destroyWindow("preview")
# Theta is the 85D vector holding [camera, pose, shape] # where camera is 3D [s, tx, ty] # pose is 72D vector holding the rotation of 24 joints of SMPL in axis angle format # shape is 10D shape coefficients of SMPL joints, verts, cams, joints3d, theta = model.predict(input_img, get_theta=True) visualize(img, proc_param, joints[0], verts[0], cams[0]) if __name__ == '__main__': config = flags.FLAGS config(sys.argv) i = 0 for file in sorted(os.listdir(config.img_path)): ext = os.path.splitext(file)[-1].lower() if ext in extensions: # Using pre-trained model, change this to use your own. tf.reset_default_graph() config.load_path = src.config.PRETRAINED_MODEL config.batch_size = 1 renderer = vis_util.SMPLRenderer(glb_path=os.path.join( config.glb_path, file), face_path=config.smpl_face_path) main(os.path.join(config.img_path, file), config.json_path) renderer.trimesh.export( os.path.join(config.glb_path, str(i) + ".glb")) i += 1
def predict(image, weight, height): global config, renderer config = flags.FLAGS config(sys.argv) # Using pre-trained model, change this to use your own. config.load_path = src.config.PRETRAINED_MODEL config.batch_size = 1 renderer = vis_util.SMPLRenderer(face_path=config.smpl_face_path) tf.reset_default_graph() sess = tf.Session() model = RunModel(config, sess=sess) input_img, proc_param, img = preprocess_image_V2(image) # Add batch dimension: 1 x D x D x 3 input_img = np.expand_dims(input_img, 0) joints, verts, cams, joints3d, theta = model.predict(input_img, get_theta=True) sess.close() cams = theta[:, :model.num_cam] poses = theta[:, model.num_cam:(model.num_cam + model.num_theta)] shapes = theta[:, (model.num_cam + model.num_theta):] viz_result = visualize(img, proc_param, joints[0], verts[0], cams[0]) ''' Start adjusting the shape ''' shape_adjuster = torch.load("./trained/model_release_1.5961363467") smpl = SMPL("./models/neutral_smpl_with_cocoplus_reg.pkl") beta = torch.from_numpy(shapes).float().cuda() theta = torch.zeros((1, 72)).float().cuda() heights = torch.from_numpy(np.asarray([height])) volume = torch.from_numpy(np.asarray([weight])) verts, joints3d, Rs = smpl.forward(beta, theta, True) flatten_joints3d = joints3d.view(1, -1) heights = torch.unsqueeze(heights, -1).float().cuda() volumes = torch.unsqueeze(volume, -1).float().cuda() input_to_net = torch.cat((flatten_joints3d, heights, volumes), 1) adjusted_betas = shape_adjuster.forward(input_to_net) adjusted_verts, adjusted_joints3d, Rs = smpl.forward( adjusted_betas, theta, True) adjusted_heights = measure.compute_height(adjusted_verts) adjusted_volumes = measure.compute_volume(adjusted_verts, smpl.f) print(adjusted_heights, adjusted_volumes) # debug_display_cloud(verts[0], joints3d[0], adjusted_verts[0], adjusted_joints3d[0]) # Change the posture for measurement from measurement import POSE1 theta = torch.from_numpy(np.expand_dims(POSE1, 0)).float().cuda() m_adjusted_verts, adjusted_joints3d, Rs = smpl.forward( adjusted_betas, theta, True) return viz_result, \ torch.squeeze(verts).detach().cpu().numpy(), \ torch.squeeze(adjusted_verts).detach().cpu().numpy(), \ torch.squeeze(m_adjusted_verts).detach().cpu().numpy(), \ torch.squeeze(adjusted_volumes).detach().cpu().numpy(),\ torch.squeeze(adjusted_heights).detach().cpu().numpy(),
def train(self): # For rendering! self.renderer = vis_util.SMPLRenderer( img_size=self.img_size, face_path=self.config.smpl_face_path) step = 0 with self.sv.managed_session(config=self.sess_config) as sess: while not self.sv.should_stop(): fetch_dict = { "summary": self.summary_op_always, "step": self.global_step, "e_loss": self.e_loss, # The meat "e_opt": self.e_opt, "loss_kp": self.e_loss_kp } if not self.encoder_only: fetch_dict.update({ # For D: "d_opt": self.d_opt, "d_loss": self.d_loss, "loss_disc": self.e_loss_disc, }) if self.use_3d_label: fetch_dict.update({ "loss_3d_params": self.e_loss_3d, "loss_3d_joints": self.e_loss_3d_joints }) if step % self.log_img_step == 0: fetch_dict.update({ "input_img": self.show_imgs, "gt_kp": self.show_kps, "e_verts": self.all_verts, "joints": self.all_pred_kps, "cam": self.all_pred_cams, }) if not self.encoder_only: fetch_dict.update( {"summary_occasional": self.summary_op_occ}) t0 = time() result = sess.run(fetch_dict) t1 = time() self.summary_writer.add_summary(result['summary'], global_step=result['step']) e_loss = result['e_loss'] step = result['step'] epoch = float(step) / self.num_itr_per_epoch if self.encoder_only: print("itr %d/(epoch %.1f): time %g, Enc_loss: %.4f" % (step, epoch, t1 - t0, e_loss)) else: d_loss = result['d_loss'] print( "itr %d/(epoch %.1f): time %g, Enc_loss: %.4f, Disc_loss: %.4f" % (step, epoch, t1 - t0, e_loss, d_loss)) if step % self.log_img_step == 0: if not self.encoder_only: self.summary_writer.add_summary( result['summary_occasional'], global_step=result['step']) self.draw_results(result) self.summary_writer.flush() if epoch > self.max_epoch: self.sv.request_stop() step += 1 print('Finish training on %s' % self.model_dir)
plt.imshow(rend_img) return fig # define parser parser = argparse.ArgumentParser() parser.add_argument('num', type=int) args = parser.parse_args() # define some paths smpl_model_path = '/gpfs/milgram/project/yildirim/hakan/hmr/models/neutral_smpl_with_cocoplus_reg.pkl' smpl_face_path = '/gpfs/milgram/project/yildirim/hakan/hmr/src/tf_smpl/smpl_faces.npy' # define SMPL model and renderer smpl = SMPL(smpl_model_path) renderer = vis_util.SMPLRenderer(face_path=smpl_face_path) # define camera, position and shape parameters # cam = np.load('latents/cam'+str(args.num)+'.npy').flatten() cam = np.array([1., 0, 0], np.float32) pose = np.load('latents/pose' + str(args.num) + '.npy') shape = np.load('latents/shape' + str(args.num) + '.npy') cam_tf = tf.Variable(cam) pose = tf.Variable(pose) shape = tf.Variable(shape) # define pre-processing dict, no clue why they require it proc_param = { 'end_pt': np.array([339, 339]), 'img_size': 230, 'scale': 0.9739130434782609,
smpl = SMPL('../../smpl/model_cocoplus.pkl', obj_saveable=True).to(device) pose = get_theta('00001') beta = get_beta('00001') vbeta = torch.tensor(np.array([beta])).float().to(device) vpose = torch.tensor(np.array([pose])).float().to(device) vcam = torch.tensor([0.9, 0, 0]).expand((1, 3)).float().to(device) verts, joints, _ = smpl(vbeta, vpose, get_skin=True) pred_kp = batch_orth_proj_idrot(joints.cpu(), vcam.cpu()) r = torch.ones((1, 3)) verts, joints, r = verts.cpu().numpy(), pred_kp.cpu().numpy(), vcam.cpu( ).numpy() print(img.shape) print(type(joints), type(verts), type(r)) print(joints[0].shape, verts[0].shape, r[0].shape) visualize(img, proc_param, joints[0], verts[0], r[0]) if __name__ == '__main__': renderer = vis_util.SMPLRenderer(face_path='../../smpl/smpl_faces.npy') main(img_path='../../up3d/00001_image.png')
Renders the result in original image coordinate frame. """ cam_for_render, vert_shifted, joints_orig = rnd.get_original( proc_param, verts, cam, joints, img_size=img.shape[:2]) rend_img_overlay = renderer( vert_shifted, cam=cam_for_render, img=img, do_alpha=True) return rend_img_overlay if __name__ == '__main__': config = flags.FLAGS config(sys.argv) # Using pre-trained model, change this to use your own. config.load_path = src.config.PRETRAINED_MODEL config.batch_size = 1 renderer = rnd.SMPLRenderer(face_path=config.smpl_face_path) cam = cv2.VideoCapture(0) yolo_sess = tf.Session() yolo = YOLO(yolo_sess=yolo_sess) hmr_graph = tf.Graph() hmr_sess = tf.Session(graph=hmr_graph) with hmr_graph.as_default(): model = RunModel(config, sess=hmr_sess) while True: _, frame = cam.read() with yolo_sess.as_default():