def homog_compose_after_inplace_scale_test():
# this should be fine
homog = Homogeneous(np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]]))
s = Scale([3, 4])
homog.compose_after_inplace(s)
assert_allclose(homog.h_matrix, np.array([[0, 4, 0], [3, 0, 0], [0, 0,
1]]))
def retrieve_view_projection_transforms(image, mesh, group=None):
rows = image.shape[0]
cols = image.shape[1]
max_d = max(rows, cols)
camera_matrix = np.array([[max_d, 0, cols / 2.0], [0, max_d, rows / 2.0],
[0, 0, 1.0]])
distortion_coeffs = np.zeros(4)
converged, r_vec, t_vec = cv2.solvePnP(
mesh.landmarks[group].lms.points,
image.landmarks[group].lms.points[:, ::-1], camera_matrix,
distortion_coeffs)
rotation_matrix = cv2.Rodrigues(r_vec)[0]
h_camera_matrix = np.eye(4)
h_camera_matrix[:3, :3] = camera_matrix
c = Homogeneous(h_camera_matrix)
t = Translation(t_vec.ravel())
r = Rotation(rotation_matrix)
view_t_flipped = r.compose_before(t)
view_t = view_t_flipped.compose_before(axes_flip_t)
proj_t = Homogeneous(
weak_projection_matrix(image.width, image.height,
view_t_flipped.apply(mesh)))
return view_t, proj_t, r
def test_homogeneous_inverse():
e = np.eye(3) * 2
e[2, 2] = 1
e_inv = np.eye(3) * 0.5
e_inv[2, 2] = 1
h = Homogeneous(e)
assert_allclose(h.pseudoinverse().h_matrix, e_inv)
def __init__(self, image_shape):
# flip axis 0 and axis 1 so indexing is as expected
flip_xy = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
# scale to get the units correct
scale = Scale(image_shape)
self.flip_and_scale = flip_xy.compose_before(scale)
def homog_compose_before_alignment_nonuniformscale_test():
homog = Homogeneous(np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]]))
scale = UniformScale(2.5, 2)
source = PointCloud(np.array([[0, 1], [1, 1], [-1, -5], [3, -5]]))
target = scale.apply(source)
# estimate the transform from source and target
s = AlignmentUniformScale(source, target)
res = homog.compose_before(s)
assert (type(res) == Homogeneous)
def test_homogeneous_apply_batched():
e = np.eye(3) * 2
p = np.random.rand(10, 2)
e[2, 2] = 1
e[:2, -1] = [2, 3]
h = Homogeneous(e)
p_applied = h.apply(p, batch_size=2)
p_manual = p * 2 + np.array([2, 3])
assert_allclose(p_applied, p_manual)
def test_homog_compose_before_nonuniformscale():
homog = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
s = NonUniformScale([3, 4])
res = homog.compose_before(s)
assert(type(res) == Homogeneous)
assert_allclose(res.h_matrix, np.array([[0, 3, 0],
[4, 0, 0],
[0, 0, 1]]))
def test_homogeneous_apply():
e = np.eye(3) * 2
p = np.random.rand(10, 2)
e[2, 2] = 1
e[:2, -1] = [2, 3]
print(e)
h = Homogeneous(e)
p_applied = h.apply(p)
p_manual = p * 2 + np.array([2, 3])
assert_allclose(p_applied, p_manual)
def test_homog_compose_after_uniformscale():
homog = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
s = UniformScale(3, 2)
res = homog.compose_after(s)
assert(type(res) == Homogeneous)
assert_allclose(res.h_matrix, np.array([[0, 3, 0],
[3, 0, 0],
[0, 0, 1]]))
def test_homogeneous_apply():
e = np.eye(3) * 2
p = np.random.rand(10, 2)
e[2, 2] = 1
e[:2, -1] = [2, 3]
print e
h = Homogeneous(e)
p_applied = h.apply(p)
p_manual = p * 2 + np.array([2, 3])
assert_allclose(p_applied, p_manual)
def test_homog_compose_after_inplace_scale():
# this should be fine
homog = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
s = Scale([3, 4])
homog.compose_after_inplace(s)
assert_allclose(homog.h_matrix, np.array([[0, 4, 0],
[3, 0, 0],
[0, 0, 1]]))
def compute_view_matrix(rho):
view_t = np.eye(4)
view_t[0, 3] = -rho[4] # tw x
view_t[1, 3] = -rho[5] # tw y
r_phi, r_theta, r_varphi = compute_rotation_matrices(rho)
rotation = np.dot(np.dot(r_varphi.h_matrix, r_theta.h_matrix),
r_phi.h_matrix)
view_t[:3, :3] = rotation[:3, :3]
return Homogeneous(view_t), Homogeneous(rotation)
def test_homog_compose_after_alignment_rotation():
homog = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
source = PointCloud(np.array([[0, 1],
[1, 1],
[-1, -5],
[3, -5]]))
r = AlignmentRotation(source, source)
res = homog.compose_after(r)
assert(type(res) == Homogeneous)
def compute_ortho_projection_derivatives_warp_parameters(
s_uv, w_uv, rho, r_phi, r_theta, r_varphi):
# Precomputations
n_parameters = len(rho)
n_points = np.size(s_uv, 0)
dp_dgamma = np.zeros([2, n_parameters, n_points])
const_term = np.vstack((8 * rho[0] / 2, 8 * rho[0] / 2))
# Compute the derivative of the perspective projection wrt focal length
dp_dgamma[:, 0, :] = np.vstack(([0.5 * w_uv[0, :].T, 0.5 * w_uv[1, :].T]))
# Compute the derivative of the phi rotation matrix
dr_phi_dphi = np.eye(4, 4)
dr_phi_dphi[1:3, 1:3] = np.array([[-np.sin(rho[1]), -np.cos(rho[1])],
[np.cos(rho[1]), -np.sin(rho[1])]])
dr_phi_dphi = Homogeneous(dr_phi_dphi)
# Compute the derivative of the warp wrt phi
dW_dphi_uv = dr_phi_dphi.apply(r_theta.apply(r_varphi.apply(s_uv))).T
dp_dgamma[:, 1, :] = dW_dphi_uv[:2, :] * const_term
# Compute the derivative of the theta rotation matrix
dr_theta_dtheta = np.eye(4, 4)
dr_theta_dtheta[:3, :3] = np.array([[-np.sin(rho[2]), 0, -np.cos(rho[2])],
[0, 0, 0],
[np.cos(rho[2]), 0, -np.sin(rho[2])]])
dr_theta_dtheta = Homogeneous(dr_theta_dtheta)
# Compute the derivative of the warp wrt theta
dW_dtheta_uv = r_phi.apply(dr_theta_dtheta.apply(r_varphi.apply(s_uv))).T
dp_dgamma[:, 2, :] = dW_dtheta_uv[:2, :] * const_term
# Compute the derivative of the varphi rotation matrix
dr_varphi_dvarphi = np.eye(4, 4)
dr_varphi_dvarphi[:2, :2] = np.array([[-np.sin(rho[3]), -np.cos(rho[3])],
[np.cos(rho[3]), -np.sin(rho[3])]])
dr_varphi_dvarphi = Homogeneous(dr_varphi_dvarphi)
# Compute the derivative of the warp wrt varphi
dW_dvarphi_uv = r_phi.apply(r_theta.apply(dr_varphi_dvarphi.apply(s_uv))).T
dp_dgamma[:, 3, :] = dW_dvarphi_uv[:2, :] * const_term
# Compute the derivative of the projection function wrt tx
dp_dtx_uv = np.vstack((np.ones([1, n_points]), np.zeros([1, n_points])))
dp_dgamma[:, 4, :] = dp_dtx_uv * const_term
# Define the derivative of the projection function wrt ty
dp_dty_uv = np.vstack((np.zeros([1, n_points]), np.ones([1, n_points])))
dp_dgamma[:, 5, :] = dp_dty_uv * const_term
return dp_dgamma
def test_homog_compose_before_alignment_nonuniformscale():
homog = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
scale = UniformScale(2.5, 2)
source = PointCloud(np.array([[0, 1],
[1, 1],
[-1, -5],
[3, -5]]))
target = scale.apply(source)
# estimate the transform from source and target
s = AlignmentUniformScale(source, target)
res = homog.compose_before(s)
assert(type(res) == Homogeneous)
def render_mesh(sample_mesh, res=256, scale=1):
if sample_mesh.shape[-1] != 3:
sample_colours = sample_mesh[..., 3:]
else:
sample_colours = np.ones_like(sample_mesh) * [0, 0, 1]
sample_mesh = sample_mesh[..., :3]
sample_mesh = Homogeneous(
dm.utils.rotation_matrix(np.deg2rad(90),
[0, 0, -1])).apply(sample_mesh)
sample_mesh = ColouredTriMesh(sample_mesh * scale * res / 2 +
res / 2,
trilist=trilist,
colours=sample_colours)
sample_mesh = lambertian_shading(sample_mesh, ambient_colour=0)
store_path = Path(LOGDIR) / str(epoch)
if not store_path.exists():
store_path.mkdir()
m3io.export_mesh(sample_mesh,
store_path / '{}.obj'.format(time.time()))
mesh_img = rasterize_mesh(sample_mesh, [res, res])
mesh_img = mesh_img.rotate_ccw_about_centre(180)
return mesh_img.pixels_with_channels_at_back()
def scale_compose_after_inplace_homog_test():
# can't do this inplace - so should just give transform chain
homog = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
s = Scale([3, 4])
s.compose_after_inplace(homog)
def model_to_clip_space(im, mesh):
view_t, c_t, proj_t = retrieve_camera_matrix(im, mesh, initialize=True)
proj_t = weak_projection_matrix(im.shape[1], im.shape[0],
view_t.apply(mesh))
view_t = view_t.compose_before(axes_flip_t)
transform = Homogeneous(proj_t.dot(view_t.h_matrix))
return transform
def clip_to_image_transform(width, height):
r"""
Affine transform that converts 3D clip space coordinates into 2D image
space coordinates. Note that the z axis of the clip space coordinates is
ignored.
Parameters
----------
width: int
The width of the image
height: int
The height of the image
Returns
-------
:map`Homogeneous`
A homogeneous transform that moves clip space coordinates into image
space.
"""
# 1. Remove the z axis from the clip space
rem_z = dims_3to2()
# 2. invert the y direction (up becomes down)
invert_y = Scale([1, -1])
# 3. [-1, 1] [-1, 1] -> [0, 2] [0, 2]
t = Translation([1, 1])
# 4. [0, 2] [0, 2] -> [0, 1] [0, 1]
unit_scale = Scale(0.5, n_dims=2)
# 5. [0, 1] [0, 1] -> [0, w - 1] [0, h - 1]
im_scale = Scale([width - 1, height - 1])
# 6. [0, w] [0, h] -> [0, h] [0, w]
xy_yx = Homogeneous(
np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=np.float))
# reduce the full transform chain to a single affine matrix
transforms = [rem_z, invert_y, t, unit_scale, im_scale, xy_yx]
return reduce(lambda a, b: a.compose_before(b), transforms)
def test_translation_compose_after_homog():
# can't do this inplace - so should just give transform chain
homog = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
t = Translation([3, 4])
res = t.compose_after(homog)
assert(type(res) == Homogeneous)
def test_scale_compose_after_inplace_homog():
# can't do this inplace - so should just give transform chain
homog = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
s = Scale([3, 4])
with raises(ValueError):
s.compose_after_inplace(homog)
def test_rotation_compose_before_homog():
# can't do this inplace - so should just give transform chain
rotation = Rotation(np.array([[1, 0],
[0, 1]]))
homog = Homogeneous(np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, 1]]))
res = rotation.compose_before(homog)
assert(type(res) == Homogeneous)
def retrieve_camera_matrix(image, mesh, group=None, initialize=True):
import cv2
drop_h = Homogeneous(np.eye(4)[:3])
flip_xy_yx = Homogeneous(np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]]))
rows = image.shape[0]
cols = image.shape[1]
max_d = max(rows, cols)
camera_matrix = np.array([[max_d, 0, cols / 2.0], [0, max_d, rows / 2.0],
[0, 0, 1.0]])
distortion_coeffs = np.zeros(4)
# Initial guess for rotation/translation.
if initialize:
r_vec = np.array([[-2.7193267], [-0.14545351], [-0.34661788]])
t_vec = np.array([[0.], [0.], [280.]])
converged, r_vec, t_vec = cv2.solvePnP(
mesh.landmarks[group].lms.points,
image.landmarks[group].lms.points[:, ::-1], camera_matrix,
distortion_coeffs, r_vec, t_vec, 1)
else:
converged, r_vec, t_vec = cv2.solvePnP(
mesh.landmarks[group].lms.points,
image.landmarks[group].lms.points[:, ::-1], camera_matrix,
distortion_coeffs)
rotation_matrix = cv2.Rodrigues(r_vec)[0]
h_camera_matrix = np.eye(4)
h_camera_matrix[:3, :3] = camera_matrix
t_vec = t_vec.ravel()
if t_vec[2] < 0:
print('Position has a negative value in z-axis')
c = Homogeneous(h_camera_matrix)
t = Translation(t_vec)
r = Rotation(rotation_matrix)
view_t = r.compose_before(t)
proj_t = c.compose_before(drop_h).compose_before(flip_xy_yx)
return view_t, c, proj_t
def render_cmesh_cloud(cmesh,
figsize=[14, 14],
oritation=[0, 0, 0],
normalise=True,
ax=None):
if isinstance(cmesh, ColouredTriMesh):
points = cmesh.points
colours = cmesh.colours
else:
points = cmesh[..., :3]
colours = cmesh[..., 3:]
colours = np.clip(colours, 0, 1)
# normalise to unit sphere
if normalise:
points -= points.mean(axis=0)
points /= points.max()
zori = Homogeneous(rotation_matrix(np.deg2rad(oritation[2]), [0, 0, 1]))
yori = Homogeneous(rotation_matrix(np.deg2rad(oritation[1]), [0, 1, 0]))
xori = Homogeneous(rotation_matrix(np.deg2rad(oritation[0]), [1, 0, 0]))
points = zori.apply(points)
points = yori.apply(points)
points = xori.apply(points)
if not ax:
plt.close()
f = plt.figure(figsize=figsize)
ax = f.add_subplot(111, projection='3d')
ax.view_init(0, 0)
# make the panes transparent
ax.xaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
ax.yaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
ax.zaxis.set_pane_color((1.0, 1.0, 1.0, 0.0))
# make the grid lines transparent
ax.xaxis._axinfo["grid"]['color'] = (1, 1, 1, 0)
ax.yaxis._axinfo["grid"]['color'] = (1, 1, 1, 0)
ax.zaxis._axinfo["grid"]['color'] = (1, 1, 1, 0)
ax.set_xlim3d(-1, 1)
ax.set_ylim3d(-1, 1)
ax.set_zlim3d(-1, 1)
ax.scatter3D(points[:, 0], points[:, 1], points[:, 2], c=colours, s=1)
ax.axis('off')
def flip_xy_yx():
r"""
Return a :map`Homogeneous` that flips the x and y axes.
Returns
-------
:map`Homogeneous`
The matrix for applying the transformation
"""
return Homogeneous(
np.array([[0, 1, 0, 0], [1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]))
def dims_3to2():
r"""
Returns
------
:map`Homogeneous`
:map`Homogeneous` that strips off the 3D axis of a 3D shape
leaving just the first two axes.
"""
return Homogeneous(np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1]]))
def rotate_mesh(mesh, angle, axis=0):
rot_v = [0, 0, 0]
rot_v[axis] = 1
mesh = mesh.copy()
mesh_t = mesh.centre_of_bounds()
mesh.points -= mesh_t
mesh = Homogeneous(rotation_matrix(np.deg2rad(angle), rot_v)).apply(mesh)
mesh.points += mesh_t
return mesh
def dims_3to2():
r"""
Return a :map`Homogeneous` that strips off the 3D axis of a 3D shape
leaving just the first two axes.
Returns
------
:map`Homogeneous`
The matrix for applying the transformation
"""
return Homogeneous(np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 0, 1]]))
def compute_rotation_matrices(rho):
rot_phi = np.eye(4)
rot_phi[1:3, 1:3] = np.array([[np.cos(rho[1]), -np.sin(rho[1])],
[np.sin(rho[1]),
np.cos(rho[1])]])
rot_theta = np.eye(4)
rot_theta[0:3, 0:3] = np.array([[np.cos(rho[2]), 0,
np.sin(rho[2])], [0, 1, 0],
[-np.sin(rho[2]), 0,
np.cos(rho[2])]])
rot_varphi = np.eye(4)
rot_varphi[0:2, 0:2] = np.array([[np.cos(rho[3]), -np.sin(rho[3])],
[np.sin(rho[3]),
np.cos(rho[3])]])
r_phi = Homogeneous(rot_phi)
r_theta = Homogeneous(rot_theta)
r_varphi = Homogeneous(rot_varphi)
return r_phi, r_theta, r_varphi
def tcoords_to_image_coords(image_shape):
r"""
Returns a :map:`Homogeneous` transform that converts [0,1]
texture coordinates (tcoords) used on :map:`TexturedTriMesh`
instances to image coordinates, which behave just like image landmarks
do.
The operations that are performed are:
- Flipping the origin from bottom-left to top-left
- Permuting the axis so that st (or uv) -> yx
- Scaling the tcoords by the image shape (denormalising them). Note that
(1, 1) has to map to the highest pixel value, which is actually
(h - 1, w - 1) due to Menpo being 0-based with image operations.
Parameters
----------
image_shape : `tuple`
The shape of the texture that the tcoords index in to.
Returns
-------
:map:`Homogeneous`
A transform that, when applied to texture coordinates, converts them
to image coordinates.
"""
# flip the 'y' st 1 -> 0 and 0 -> 1, moving the axis to upper left
invert_unit_y = Homogeneous(
np.array([[1.0, 0.0, 0.0], [0.0, -1.0, 1.0], [0.0, 0.0, 1.0]])
)
# flip axis 0 and axis 1 so indexing is as expected
flip_xy_yx = Homogeneous(
np.array([[0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]])
)
return invert_unit_y.compose_before(flip_xy_yx).compose_before(
Scale(np.array(image_shape) - 1)
)
def clip_to_image(height, width):
# 2. invert the y direction (up becomes down)
invert_y = Scale([1, -1])
# 3. [-1, 1] [-1, 1] -> [0, 2] [0, 2]
t = Translation([1, 1])
# 4. [0, 2] [0, 2] -> [0, 1] [0, 1]
unit_scale = Scale(0.5, n_dims=2)
# 5. [0, 1] [0, 1] -> [0, w] [0, h]
im_scale = Scale([width, height])
# 6. [0, w] [0, h] -> [0, h] [0, w]
xy_yx = Homogeneous(
np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], dtype=np.float))
# reduce the full transform chain to a single affine matrix
transforms = [invert_y, t, unit_scale, im_scale, xy_yx]
return reduce(lambda a, b: a.compose_before(b), transforms)
def dims_2to3(x=0):
r"""
Return a :map`Homogeneous` that adds on a 3rd axis to a 2D shape.
Parameters
----------
x : `float`, optional
The value that will be assigned to the new third dimension
Returns
------
:map`Homogeneous`
The matrix for applying the transformation
"""
return Homogeneous(np.array([[1, 0, 0], [0, 1, 0], [0, 0, x], [0, 0, 1]]))
def rotate_y_180_rotate_z_180():
"""
Equivalent to rotating y by 180, then rotating z by 180.
This simulates calling gluLookAt to lookat the origin
gluLookAt(0,0,0,0,0,1,0,-1,0)
Returns
-------
:map`Homogeneous`
The matrix for applying the transformation
"""
axes_flip_matrix = np.eye(4)
axes_flip_matrix[1, 1] = -1
axes_flip_matrix[2, 2] = -1
return Homogeneous(axes_flip_matrix)
def pinhole_intrinsic_matrix(image_height, image_width, focal_length=None):
r"""
Create a basic "pinhole" type camera intrinsic matrix. Focal length is in pixels
and principal point is in the image center. Note this follows OpenCV image
conventions and thus the "first" axis is the x-axis rather than the typical
menpo convention of the "first" axis being the y-axis.
[fx, 0, cx, 0]
[ 0, fy, cy, 0]
[ 0, 0, 1, 0]
[ 0, 0, 0, 1]
Parameters
----------
image_height : int
Image height
image_width : int
Image width
focal_length : float, optional
If given, the focal length (fx=fy) in pixels. If not given, the max
of the width and height is used.
Returns
-------
:map`Homogeneous`
3D camera intrinsics matrix as a Homogeneous matrix
"""
if focal_length is None:
focal_length = max(image_height, image_width)
return Homogeneous(
np.array(
[
[focal_length, 0, image_width / 2, 0],
[0, focal_length, image_height / 2, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
]
)
)
def test_homogeneous_from_vector_inplace():
h = Homogeneous(np.eye(3))
e = np.eye(3) * 2
e[2, 2] = 1
h._from_vector_inplace(e.ravel())
assert_allclose(h.h_matrix, e)
def test_homogeneous_as_vector():
e = np.eye(3) * 2
e[2, 2] = 1
h = Homogeneous(e)
assert_allclose(h.as_vector(), e.flatten())
def test_homogeneous_has_true_inverse():
h = Homogeneous.init_identity(2)
assert h.has_true_inverse
def test_homogeneous_eye():
e = np.eye(3)
h = Homogeneous.init_identity(2)
assert_allclose(e, h.h_matrix)
def test_homogenous_set_h_matrix_raises_notimplementederror():
s = Homogeneous(np.eye(4))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
with raises(NotImplementedError):
s.set_h_matrix(s.h_matrix)
