def compute_ortho_projection_derivatives_warp_parameters(
s_uv, w_uv, rho, r_phi, r_theta, r_varphi):
# Precomputations
n_parameters = len(rho)
n_points = np.size(s_uv, 0)
dp_dgamma = np.zeros([2, n_parameters, n_points])
const_term = np.vstack((8 * rho[0] / 2, 8 * rho[0] / 2))
# Compute the derivative of the perspective projection wrt focal length
dp_dgamma[:, 0, :] = np.vstack(([0.5 * w_uv[0, :].T, 0.5 * w_uv[1, :].T]))
# Compute the derivative of the phi rotation matrix
dr_phi_dphi = np.eye(4, 4)
dr_phi_dphi[1:3, 1:3] = np.array([[-np.sin(rho[1]), -np.cos(rho[1])],
[np.cos(rho[1]), -np.sin(rho[1])]])
dr_phi_dphi = Homogeneous(dr_phi_dphi)
# Compute the derivative of the warp wrt phi
dW_dphi_uv = dr_phi_dphi.apply(r_theta.apply(r_varphi.apply(s_uv))).T
dp_dgamma[:, 1, :] = dW_dphi_uv[:2, :] * const_term
# Compute the derivative of the theta rotation matrix
dr_theta_dtheta = np.eye(4, 4)
dr_theta_dtheta[:3, :3] = np.array([[-np.sin(rho[2]), 0, -np.cos(rho[2])],
[0, 0, 0],
[np.cos(rho[2]), 0, -np.sin(rho[2])]])
dr_theta_dtheta = Homogeneous(dr_theta_dtheta)
# Compute the derivative of the warp wrt theta
dW_dtheta_uv = r_phi.apply(dr_theta_dtheta.apply(r_varphi.apply(s_uv))).T
dp_dgamma[:, 2, :] = dW_dtheta_uv[:2, :] * const_term
# Compute the derivative of the varphi rotation matrix
dr_varphi_dvarphi = np.eye(4, 4)
dr_varphi_dvarphi[:2, :2] = np.array([[-np.sin(rho[3]), -np.cos(rho[3])],
[np.cos(rho[3]), -np.sin(rho[3])]])
dr_varphi_dvarphi = Homogeneous(dr_varphi_dvarphi)
# Compute the derivative of the warp wrt varphi
dW_dvarphi_uv = r_phi.apply(r_theta.apply(dr_varphi_dvarphi.apply(s_uv))).T
dp_dgamma[:, 3, :] = dW_dvarphi_uv[:2, :] * const_term
# Compute the derivative of the projection function wrt tx
dp_dtx_uv = np.vstack((np.ones([1, n_points]), np.zeros([1, n_points])))
dp_dgamma[:, 4, :] = dp_dtx_uv * const_term
# Define the derivative of the projection function wrt ty
dp_dty_uv = np.vstack((np.zeros([1, n_points]), np.ones([1, n_points])))
dp_dgamma[:, 5, :] = dp_dty_uv * const_term
return dp_dgamma
def render_cmesh_cloud(cmesh,
figsize=[14, 14],
oritation=[0, 0, 0],
normalise=True,
ax=None):
if isinstance(cmesh, ColouredTriMesh):
points = cmesh.points
colours = cmesh.colours
else:
points = cmesh[..., :3]
colours = cmesh[..., 3:]
colours = np.clip(colours, 0, 1)
# normalise to unit sphere
if normalise:
points -= points.mean(axis=0)
points /= points.max()
zori = Homogeneous(rotation_matrix(np.deg2rad(oritation[2]), [0, 0, 1]))
yori = Homogeneous(rotation_matrix(np.deg2rad(oritation[1]), [0, 1, 0]))
xori = Homogeneous(rotation_matrix(np.deg2rad(oritation[0]), [1, 0, 0]))
points = zori.apply(points)
points = yori.apply(points)
points = xori.apply(points)
if not ax:
plt.close()
f = plt.figure(figsize=figsize)
ax = f.add_subplot(111, projection='3d')
ax.view_init(0, 0)
# make the panes transparent
ax.xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
ax.yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
ax.zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
# make the grid lines transparent
ax.xaxis._axinfo["grid"]['color'] = (1, 1, 1, 0)
ax.yaxis._axinfo["grid"]['color'] = (1, 1, 1, 0)
ax.zaxis._axinfo["grid"]['color'] = (1, 1, 1, 0)
ax.set_xlim3d(-1, 1)
ax.set_ylim3d(-1, 1)
ax.set_zlim3d(-1, 1)
ax.scatter3D(points[:, 0], points[:, 1], points[:, 2], c=colours, s=1)
ax.axis('off')
def test_homogeneous_apply_batched():
e = np.eye(3) * 2
p = np.random.rand(10, 2)
e[2, 2] = 1
e[:2, -1] = [2, 3]
h = Homogeneous(e)
p_applied = h.apply(p, batch_size=2)
p_manual = p * 2 + np.array([2, 3])
assert_allclose(p_applied, p_manual)
def test_homogeneous_apply():
e = np.eye(3) * 2
p = np.random.rand(10, 2)
e[2, 2] = 1
e[:2, -1] = [2, 3]
print e
h = Homogeneous(e)
p_applied = h.apply(p)
p_manual = p * 2 + np.array([2, 3])
assert_allclose(p_applied, p_manual)
def test_homogeneous_apply():
e = np.eye(3) * 2
p = np.random.rand(10, 2)
e[2, 2] = 1
e[:2, -1] = [2, 3]
print(e)
h = Homogeneous(e)
p_applied = h.apply(p)
p_manual = p * 2 + np.array([2, 3])
assert_allclose(p_applied, p_manual)
def compute_pers_projection_derivatives_warp_parameters(
s_uv, w_uv, rho, r_phi, r_theta, r_varphi):
# Precomputations
n_parameters = len(rho)
n_points = np.size(s_uv, 0)
dp_dgamma = np.zeros([2, n_parameters, n_points])
w = w_uv[:, 2]
const = 4 * rho[0] / (w**2)
# Compute the derivative of the perspective projection wrt focal length
dp_dgamma[:, 0, :] = np.vstack(
([0.5 * w_uv[:, 0] / w, 0.5 * w_uv[:, 1] / w]))
# Compute the derivative of the phi rotation matrix
dr_phi_dphi = np.eye(4, 4)
dr_phi_dphi[1:3, 1:3] = np.array([[-np.sin(rho[1]), -np.cos(rho[1])],
[np.cos(rho[1]), -np.sin(rho[1])]])
dr_phi_dphi = Homogeneous(dr_phi_dphi)
# Compute the derivative of the warp wrt phi
dW_dphi_uv = dr_phi_dphi.apply(r_theta.apply(r_varphi.apply(s_uv))).T
# Compute the derivative of the projection wrt phi
dp_dphi_uv = np.vstack(
(dW_dphi_uv[0, :] * w - w_uv[:, 0] * dW_dphi_uv[2, :],
dW_dphi_uv[1, :] * w - w_uv[:, 1] * dW_dphi_uv[2, :]))
dp_dgamma[:, 1, :] = const * dp_dphi_uv
# Compute the derivative of the theta rotation matrix
dr_theta_dtheta = np.eye(4, 4)
dr_theta_dtheta[:3, :3] = np.array([[-np.sin(rho[2]), 0, -np.cos(rho[2])],
[0, 0, 0],
[np.cos(rho[2]), 0, -np.sin(rho[2])]])
dr_theta_dtheta = Homogeneous(dr_theta_dtheta)
# Compute the derivative of the warp wrt theta
dW_dtheta_uv = r_phi.apply(dr_theta_dtheta.apply(r_varphi.apply(s_uv))).T
# Compute the derivative of the projection wrt theta
dp_dtheta_uv = np.vstack(
(dW_dtheta_uv[0, :] * w - w_uv[:, 0] * dW_dtheta_uv[2, :],
dW_dtheta_uv[1, :] * w - w_uv[:, 1] * dW_dtheta_uv[2, :]))
dp_dgamma[:, 2, :] = const * dp_dtheta_uv
# Compute the derivative of the varphi rotation matrix
dr_varphi_dvarphi = np.eye(4, 4)
dr_varphi_dvarphi[:2, :2] = np.array([[-np.sin(rho[3]), -np.cos(rho[3])],
[np.cos(rho[3]), -np.sin(rho[3])]])
dr_varphi_dvarphi = Homogeneous(dr_varphi_dvarphi)
# Compute the derivative of the warp wrt varphi
dW_dvarphi_uv = r_phi.apply(r_theta.apply(dr_varphi_dvarphi.apply(s_uv))).T
# Compute the derivative of the projection wrt varphi
dp_dvarphi_uv = np.vstack(
(dW_dvarphi_uv[0, :] * w - w_uv[:, 0] * dW_dvarphi_uv[2, :],
dW_dvarphi_uv[1, :] * w - w_uv[:, 1] * dW_dvarphi_uv[2, :]))
dp_dgamma[:, 3, :] = const * dp_dvarphi_uv
# Compute the derivative of the projection function wrt tx
dp_dtx_uv = np.vstack((w_uv[:, 2], np.zeros(n_points)))
dp_dgamma[:, 4, :] = const * dp_dtx_uv
# Compute the derivative of the projection function wrt ty
dp_dty_uv = np.vstack((np.zeros(n_points), w))
dp_dgamma[:, 5, :] = const * dp_dty_uv
# Compute the derivative of the projection function wrt tz
#dp_dtz_uv = np.vstack((-w_uv[:, 0], -w_uv[:, 1]))
#dp_dgamma[:, 6, :] = const*dp_dtz_uv
return dp_dgamma
def train_op(models,
data,
i_epoch,
i_batch,
epoch_end,
training_history=None,
**kwargs):
model_ae_mesh, model_3dmm, _ = models
[input_mesh,
input_image], [mesh_gt,
_] = dm.engine.training.tf_dataset_adapter(data)
sess = dm.K.get_session()
# ----------------------
# Train AutoEncoder
# ----------------------
loss_ae = model_ae_mesh.train_on_batch(input_mesh, mesh_gt)
# ------------------
# Train 3DMM
# ------------------
if i_epoch > STARTING_3DMM:
loss_3dmm = model_3dmm.train_on_batch(input_image, mesh_gt)
else:
loss_3dmm = 0
logs = dm.utils.Summary({
"losses/AE":
loss_ae,
"losses/3dmm":
loss_3dmm,
"learning_rate/ae":
model_ae_mesh.optimizer.lr.eval(sess),
"learning_rate/3dmm":
model_3dmm.optimizer.lr.eval(sess),
})
if epoch_end:
t_rot30l = Homogeneous(
dm.utils.rotation_matrix(np.deg2rad(30), [1, 0, 0]))
t_rot30r = Homogeneous(
dm.utils.rotation_matrix(np.deg2rad(-30), [1, 0, 0]))
ae_mesh = model_ae_mesh.predict(input_mesh)
pred_mesh = model_3dmm.predict(input_image)
logs.update_images({
'inputs/image':
input_image,
'pred/mesh/AE':
dm.utils.mesh.render_meshes(ae_mesh[:4], trilist),
'pred/mesh/3dmm/r0':
dm.utils.mesh.render_meshes(pred_mesh[:4], trilist),
'pred/mesh/3dmm/r+30':
dm.utils.mesh.render_meshes([
np.concatenate([t_rot30l.apply(m[:, :3]), m[:, 3:]],
axis=-1) for m in pred_mesh[:4]
], trilist),
'pred/mesh/3dmm/r-30':
dm.utils.mesh.render_meshes([
np.concatenate([t_rot30r.apply(m[:, :3]), m[:, 3:]],
axis=-1) for m in pred_mesh[:4]
], trilist),
'gt/mesh':
dm.utils.mesh.render_meshes(input_mesh[:4], trilist),
})
return logs
