def compute_ortho_projection_derivatives_warp_parameters( s_uv, w_uv, rho, r_phi, r_theta, r_varphi): # Precomputations n_parameters = len(rho) n_points = np.size(s_uv, 0) dp_dgamma = np.zeros([2, n_parameters, n_points]) const_term = np.vstack((8 * rho[0] / 2, 8 * rho[0] / 2)) # Compute the derivative of the perspective projection wrt focal length dp_dgamma[:, 0, :] = np.vstack(([0.5 * w_uv[0, :].T, 0.5 * w_uv[1, :].T])) # Compute the derivative of the phi rotation matrix dr_phi_dphi = np.eye(4, 4) dr_phi_dphi[1:3, 1:3] = np.array([[-np.sin(rho[1]), -np.cos(rho[1])], [np.cos(rho[1]), -np.sin(rho[1])]]) dr_phi_dphi = Homogeneous(dr_phi_dphi) # Compute the derivative of the warp wrt phi dW_dphi_uv = dr_phi_dphi.apply(r_theta.apply(r_varphi.apply(s_uv))).T dp_dgamma[:, 1, :] = dW_dphi_uv[:2, :] * const_term # Compute the derivative of the theta rotation matrix dr_theta_dtheta = np.eye(4, 4) dr_theta_dtheta[:3, :3] = np.array([[-np.sin(rho[2]), 0, -np.cos(rho[2])], [0, 0, 0], [np.cos(rho[2]), 0, -np.sin(rho[2])]]) dr_theta_dtheta = Homogeneous(dr_theta_dtheta) # Compute the derivative of the warp wrt theta dW_dtheta_uv = r_phi.apply(dr_theta_dtheta.apply(r_varphi.apply(s_uv))).T dp_dgamma[:, 2, :] = dW_dtheta_uv[:2, :] * const_term # Compute the derivative of the varphi rotation matrix dr_varphi_dvarphi = np.eye(4, 4) dr_varphi_dvarphi[:2, :2] = np.array([[-np.sin(rho[3]), -np.cos(rho[3])], [np.cos(rho[3]), -np.sin(rho[3])]]) dr_varphi_dvarphi = Homogeneous(dr_varphi_dvarphi) # Compute the derivative of the warp wrt varphi dW_dvarphi_uv = r_phi.apply(r_theta.apply(dr_varphi_dvarphi.apply(s_uv))).T dp_dgamma[:, 3, :] = dW_dvarphi_uv[:2, :] * const_term # Compute the derivative of the projection function wrt tx dp_dtx_uv = np.vstack((np.ones([1, n_points]), np.zeros([1, n_points]))) dp_dgamma[:, 4, :] = dp_dtx_uv * const_term # Define the derivative of the projection function wrt ty dp_dty_uv = np.vstack((np.zeros([1, n_points]), np.ones([1, n_points]))) dp_dgamma[:, 5, :] = dp_dty_uv * const_term return dp_dgamma
def render_cmesh_cloud(cmesh, figsize=[14, 14], oritation=[0, 0, 0], normalise=True, ax=None): if isinstance(cmesh, ColouredTriMesh): points = cmesh.points colours = cmesh.colours else: points = cmesh[..., :3] colours = cmesh[..., 3:] colours = np.clip(colours, 0, 1) # normalise to unit sphere if normalise: points -= points.mean(axis=0) points /= points.max() zori = Homogeneous(rotation_matrix(np.deg2rad(oritation[2]), [0, 0, 1])) yori = Homogeneous(rotation_matrix(np.deg2rad(oritation[1]), [0, 1, 0])) xori = Homogeneous(rotation_matrix(np.deg2rad(oritation[0]), [1, 0, 0])) points = zori.apply(points) points = yori.apply(points) points = xori.apply(points) if not ax: plt.close() f = plt.figure(figsize=figsize) ax = f.add_subplot(111, projection='3d') ax.view_init(0, 0) # make the panes transparent ax.xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0)) ax.yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0)) ax.zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0)) # make the grid lines transparent ax.xaxis._axinfo["grid"]['color'] = (1, 1, 1, 0) ax.yaxis._axinfo["grid"]['color'] = (1, 1, 1, 0) ax.zaxis._axinfo["grid"]['color'] = (1, 1, 1, 0) ax.set_xlim3d(-1, 1) ax.set_ylim3d(-1, 1) ax.set_zlim3d(-1, 1) ax.scatter3D(points[:, 0], points[:, 1], points[:, 2], c=colours, s=1) ax.axis('off')
def test_homogeneous_apply_batched(): e = np.eye(3) * 2 p = np.random.rand(10, 2) e[2, 2] = 1 e[:2, -1] = [2, 3] h = Homogeneous(e) p_applied = h.apply(p, batch_size=2) p_manual = p * 2 + np.array([2, 3]) assert_allclose(p_applied, p_manual)
def test_homogeneous_apply(): e = np.eye(3) * 2 p = np.random.rand(10, 2) e[2, 2] = 1 e[:2, -1] = [2, 3] print e h = Homogeneous(e) p_applied = h.apply(p) p_manual = p * 2 + np.array([2, 3]) assert_allclose(p_applied, p_manual)
def test_homogeneous_apply(): e = np.eye(3) * 2 p = np.random.rand(10, 2) e[2, 2] = 1 e[:2, -1] = [2, 3] print(e) h = Homogeneous(e) p_applied = h.apply(p) p_manual = p * 2 + np.array([2, 3]) assert_allclose(p_applied, p_manual)
def compute_pers_projection_derivatives_warp_parameters( s_uv, w_uv, rho, r_phi, r_theta, r_varphi): # Precomputations n_parameters = len(rho) n_points = np.size(s_uv, 0) dp_dgamma = np.zeros([2, n_parameters, n_points]) w = w_uv[:, 2] const = 4 * rho[0] / (w**2) # Compute the derivative of the perspective projection wrt focal length dp_dgamma[:, 0, :] = np.vstack( ([0.5 * w_uv[:, 0] / w, 0.5 * w_uv[:, 1] / w])) # Compute the derivative of the phi rotation matrix dr_phi_dphi = np.eye(4, 4) dr_phi_dphi[1:3, 1:3] = np.array([[-np.sin(rho[1]), -np.cos(rho[1])], [np.cos(rho[1]), -np.sin(rho[1])]]) dr_phi_dphi = Homogeneous(dr_phi_dphi) # Compute the derivative of the warp wrt phi dW_dphi_uv = dr_phi_dphi.apply(r_theta.apply(r_varphi.apply(s_uv))).T # Compute the derivative of the projection wrt phi dp_dphi_uv = np.vstack( (dW_dphi_uv[0, :] * w - w_uv[:, 0] * dW_dphi_uv[2, :], dW_dphi_uv[1, :] * w - w_uv[:, 1] * dW_dphi_uv[2, :])) dp_dgamma[:, 1, :] = const * dp_dphi_uv # Compute the derivative of the theta rotation matrix dr_theta_dtheta = np.eye(4, 4) dr_theta_dtheta[:3, :3] = np.array([[-np.sin(rho[2]), 0, -np.cos(rho[2])], [0, 0, 0], [np.cos(rho[2]), 0, -np.sin(rho[2])]]) dr_theta_dtheta = Homogeneous(dr_theta_dtheta) # Compute the derivative of the warp wrt theta dW_dtheta_uv = r_phi.apply(dr_theta_dtheta.apply(r_varphi.apply(s_uv))).T # Compute the derivative of the projection wrt theta dp_dtheta_uv = np.vstack( (dW_dtheta_uv[0, :] * w - w_uv[:, 0] * dW_dtheta_uv[2, :], dW_dtheta_uv[1, :] * w - w_uv[:, 1] * dW_dtheta_uv[2, :])) dp_dgamma[:, 2, :] = const * dp_dtheta_uv # Compute the derivative of the varphi rotation matrix dr_varphi_dvarphi = np.eye(4, 4) dr_varphi_dvarphi[:2, :2] = np.array([[-np.sin(rho[3]), -np.cos(rho[3])], [np.cos(rho[3]), -np.sin(rho[3])]]) dr_varphi_dvarphi = Homogeneous(dr_varphi_dvarphi) # Compute the derivative of the warp wrt varphi dW_dvarphi_uv = r_phi.apply(r_theta.apply(dr_varphi_dvarphi.apply(s_uv))).T # Compute the derivative of the projection wrt varphi dp_dvarphi_uv = np.vstack( (dW_dvarphi_uv[0, :] * w - w_uv[:, 0] * dW_dvarphi_uv[2, :], dW_dvarphi_uv[1, :] * w - w_uv[:, 1] * dW_dvarphi_uv[2, :])) dp_dgamma[:, 3, :] = const * dp_dvarphi_uv # Compute the derivative of the projection function wrt tx dp_dtx_uv = np.vstack((w_uv[:, 2], np.zeros(n_points))) dp_dgamma[:, 4, :] = const * dp_dtx_uv # Compute the derivative of the projection function wrt ty dp_dty_uv = np.vstack((np.zeros(n_points), w)) dp_dgamma[:, 5, :] = const * dp_dty_uv # Compute the derivative of the projection function wrt tz #dp_dtz_uv = np.vstack((-w_uv[:, 0], -w_uv[:, 1])) #dp_dgamma[:, 6, :] = const*dp_dtz_uv return dp_dgamma
def train_op(models, data, i_epoch, i_batch, epoch_end, training_history=None, **kwargs): model_ae_mesh, model_3dmm, _ = models [input_mesh, input_image], [mesh_gt, _] = dm.engine.training.tf_dataset_adapter(data) sess = dm.K.get_session() # ---------------------- # Train AutoEncoder # ---------------------- loss_ae = model_ae_mesh.train_on_batch(input_mesh, mesh_gt) # ------------------ # Train 3DMM # ------------------ if i_epoch > STARTING_3DMM: loss_3dmm = model_3dmm.train_on_batch(input_image, mesh_gt) else: loss_3dmm = 0 logs = dm.utils.Summary({ "losses/AE": loss_ae, "losses/3dmm": loss_3dmm, "learning_rate/ae": model_ae_mesh.optimizer.lr.eval(sess), "learning_rate/3dmm": model_3dmm.optimizer.lr.eval(sess), }) if epoch_end: t_rot30l = Homogeneous( dm.utils.rotation_matrix(np.deg2rad(30), [1, 0, 0])) t_rot30r = Homogeneous( dm.utils.rotation_matrix(np.deg2rad(-30), [1, 0, 0])) ae_mesh = model_ae_mesh.predict(input_mesh) pred_mesh = model_3dmm.predict(input_image) logs.update_images({ 'inputs/image': input_image, 'pred/mesh/AE': dm.utils.mesh.render_meshes(ae_mesh[:4], trilist), 'pred/mesh/3dmm/r0': dm.utils.mesh.render_meshes(pred_mesh[:4], trilist), 'pred/mesh/3dmm/r+30': dm.utils.mesh.render_meshes([ np.concatenate([t_rot30l.apply(m[:, :3]), m[:, 3:]], axis=-1) for m in pred_mesh[:4] ], trilist), 'pred/mesh/3dmm/r-30': dm.utils.mesh.render_meshes([ np.concatenate([t_rot30r.apply(m[:, :3]), m[:, 3:]], axis=-1) for m in pred_mesh[:4] ], trilist), 'gt/mesh': dm.utils.mesh.render_meshes(input_mesh[:4], trilist), }) return logs