def test_forward(self):
a_cpu = np.random.normal(size=(10, 3)).astype('float32')
b_cpu = np.random.normal(size=(10, 3)).astype('float32')
c_ref = np.cross(a_cpu, b_cpu)
a_gpu = chainer.cuda.to_gpu(a_cpu)
b_gpu = chainer.cuda.to_gpu(b_cpu)
c_cpu = neural_renderer.cross(a_cpu, b_cpu).data
c_gpu = neural_renderer.cross(a_gpu, b_gpu).data
chainer.testing.assert_allclose(c_ref, c_cpu)
chainer.testing.assert_allclose(c_ref, chainer.cuda.to_cpu(c_gpu))
def render_normal(self, vertices, faces):
# fill back
if self.fill_back:
faces = cf.concat((faces, faces[:, :, ::-1]), axis=1).data
# normal
faces_normal = nr.vertices_to_faces(vertices, faces)
(bs, nf) = faces_normal.shape[:2]
faces_normal = faces_normal.reshape((bs * nf, 3, 3))
v10 = faces_normal[:, 0] - faces_normal[:, 1]
v12 = faces_normal[:, 2] - faces_normal[:, 1]
normals = cf.normalize(nr.cross(v10, v12))
normals = normals.reshape((bs, nf, 3))
textures = normals[:, :, None, None, None, :]
textures = cf.tile(textures, (1, 1, 2, 2, 2, 1))
# viewpoint transformation
if self.camera_mode == 'look_at':
vertices = nr.look_at(vertices, self.eye)
elif self.camera_mode == 'look':
vertices = nr.look(vertices, self.eye, self.camera_direction, self.up)
# perspective transformation
if self.perspective:
vertices = nr.perspective(vertices, angle=self.viewing_angle)
# rasterization
faces = nr.vertices_to_faces(vertices, faces)
images = nr.rasterize(
faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps,
self.background_color)
return images
def look_at(vertices, eye, at=None, up=None):
"""
"Look at" transformation of vertices.
"""
assert (vertices.ndim == 3)
xp = chainer.cuda.get_array_module(vertices)
batch_size = vertices.shape[0]
if at is None:
at = xp.array([0, 0, 0], 'float32')
if up is None:
up = xp.array([0, 1, 0], 'float32')
if isinstance(eye, list) or isinstance(eye, tuple):
eye = xp.array(eye, 'float32')
if eye.ndim == 1:
eye = cf.tile(eye[None, :], (batch_size, 1))
if at.ndim == 1:
at = cf.tile(at[None, :], (batch_size, 1))
if up.ndim == 1:
up = cf.tile(up[None, :], (batch_size, 1))
# create new axes
z_axis = cf.normalize(at - eye)
x_axis = cf.normalize(neural_renderer.cross(up, z_axis))
y_axis = cf.normalize(neural_renderer.cross(z_axis, x_axis))
# create rotation matrix: [bs, 3, 3]
r = cf.concat((x_axis[:, None, :], y_axis[:, None, :], z_axis[:, None, :]),
axis=1)
if r.shape[0] != vertices.shape[0]:
r = cf.broadcast_to(r, (vertices.shape[0], 3, 3))
# apply
# [bs, nv, 3] -> [bs, nv, 3] -> [bs, nv, 3]
if vertices.shape != eye.shape:
eye = cf.broadcast_to(eye[:, None, :], vertices.shape)
vertices = vertices - eye
vertices = cf.matmul(vertices, r, transb=True)
return vertices
def look(vertices, viewpoints, direction=None, up=None):
"""
"Look at" transformation of vertices.
"""
assert (vertices.ndim == 3)
xp = chainer.cuda.get_array_module(vertices)
if direction is None:
direction = xp.array([0, 0, 1], 'float32')
if up is None:
up = xp.array([0, 1, 0], 'float32')
if isinstance(viewpoints, list) or isinstance(viewpoints, tuple):
viewpoints = xp.array(viewpoints, 'float32')
if viewpoints.ndim == 1:
viewpoints = viewpoints[None, :]
if direction.ndim == 1:
direction = direction[None, :]
if up.ndim == 1:
up = up[None, :]
# create new axes
z_axis = cf.normalize(direction)
x_axis = cf.normalize(neural_renderer.cross(up, z_axis))
y_axis = cf.normalize(neural_renderer.cross(z_axis, x_axis))
# create rotation matrix: [bs, 3, 3]
r = cf.concat((x_axis[:, None, :], y_axis[:, None, :], z_axis[:, None, :]),
axis=1)
if r.shape[0] != vertices.shape[0]:
r = cf.broadcast_to(r, vertices.shape)
# apply
# [bs, nv, 3] -> [bs, nv, 3] -> [bs, nv, 3]
if vertices.shape != viewpoints.shape:
viewpoints = cf.broadcast_to(viewpoints[:, None, :], vertices.shape)
vertices = vertices - viewpoints
vertices = cf.matmul(vertices, r, transb=True)
return vertices
def look_at(vertices, eye, at=None, up=None):
"""
"Look at" transformation of vertices.
"""
assert (vertices.ndim == 3)
xp = chainer.cuda.get_array_module(vertices)
batch_size = vertices.shape[0]
if at is None:
at = xp.array([0, 0, 0], 'float32')
if up is None:
up = xp.array([0, 1, 0], 'float32')
if isinstance(eye, list) or isinstance(eye, tuple):
eye = xp.array(eye, 'float32')
if eye.ndim == 1:
eye = cf.tile(eye[None, :], (batch_size, 1))
if at.ndim == 1:
at = cf.tile(at[None, :], (batch_size, 1))
if up.ndim == 1:
up = cf.tile(up[None, :], (batch_size, 1))
# create new axes
z_axis = cf.normalize(at - eye)
x_axis = cf.normalize(neural_renderer.cross(up, z_axis))
y_axis = cf.normalize(neural_renderer.cross(z_axis, x_axis))
# create rotation matrix: [bs, 3, 3]
r = cf.concat((x_axis[:, None, :], y_axis[:, None, :], z_axis[:, None, :]), axis=1)
if r.shape[0] != vertices.shape[0]:
r = cf.broadcast_to(r, (vertices.shape[0], 3, 3))
# apply
# [bs, nv, 3] -> [bs, nv, 3] -> [bs, nv, 3]
if vertices.shape != eye.shape:
eye = cf.broadcast_to(eye[:, None, :], vertices.shape)
vertices = vertices - eye
vertices = cf.matmul(vertices, r, transb=True)
return vertices
def lighting(faces,
textures,
intensity_ambient=0.5,
intensity_directional=0.5,
color_ambient=(1, 1, 1),
color_directional=(1, 1, 1),
direction=(0, 1, 0)):
xp = chainer.cuda.get_array_module(faces)
bs, nf = faces.shape[:2]
# arguments
if isinstance(color_ambient, tuple) or isinstance(color_ambient, list):
color_ambient = xp.array(color_ambient, 'float32')
if isinstance(color_directional, tuple) or isinstance(
color_directional, list):
color_directional = xp.array(color_directional, 'float32')
if isinstance(direction, tuple) or isinstance(direction, list):
direction = xp.array(direction, 'float32')
if color_ambient.ndim == 1:
color_ambient = cf.broadcast_to(color_ambient[None, :], (bs, 3))
if color_directional.ndim == 1:
color_directional = cf.broadcast_to(color_directional[None, :],
(bs, 3))
if direction.ndim == 1:
direction = cf.broadcast_to(direction[None, :], (bs, 3))
# create light
light = xp.zeros((bs, nf, 3), 'float32')
# ambient light
if intensity_ambient != 0:
light = light + intensity_ambient * cf.broadcast_to(
color_ambient[:, None, :], light.shape)
# directional light
if intensity_directional != 0:
faces = faces.reshape((bs * nf, 3, 3))
v10 = faces[:, 0] - faces[:, 1]
v12 = faces[:, 2] - faces[:, 1]
normals = cf.normalize(neural_renderer.cross(v10, v12))
normals = normals.reshape((bs, nf, 3))
if direction.ndim == 2:
direction = cf.broadcast_to(direction[:, None, :], normals.shape)
cos = cf.relu(cf.sum(normals * direction, axis=2))
light = (light + intensity_directional * cfmath.mul(
*cf.broadcast(color_directional[:, None, :], cos[:, :, None])))
# apply
light = cf.broadcast_to(light[:, :, None, None, None, :], textures.shape)
textures = textures * light
return textures
def lighting(
faces, textures, intensity_ambient=0.5, intensity_directional=0.5, color_ambient=(1, 1, 1),
color_directional=(1, 1, 1), direction=(0, 1, 0)):
xp = chainer.cuda.get_array_module(faces)
bs, nf = faces.shape[:2]
# arguments
if isinstance(color_ambient, tuple) or isinstance(color_ambient, list):
color_ambient = xp.array(color_ambient, 'float32')
if isinstance(color_directional, tuple) or isinstance(color_directional, list):
color_directional = xp.array(color_directional, 'float32')
if isinstance(direction, tuple) or isinstance(direction, list):
direction = xp.array(direction, 'float32')
if color_ambient.ndim == 1:
color_ambient = cf.broadcast_to(color_ambient[None, :], (bs, 3))
if color_directional.ndim == 1:
color_directional = cf.broadcast_to(color_directional[None, :], (bs, 3))
if direction.ndim == 1:
direction = cf.broadcast_to(direction[None, :], (bs, 3))
# create light
light = xp.zeros((bs, nf, 3), 'float32')
# ambient light
if intensity_ambient != 0:
light = light + intensity_ambient * cf.broadcast_to(color_ambient[:, None, :], light.shape)
# directional light
if intensity_directional != 0:
faces = faces.reshape((bs * nf, 3, 3))
v10 = faces[:, 0] - faces[:, 1]
v12 = faces[:, 2] - faces[:, 1]
normals = cf.normalize(neural_renderer.cross(v10, v12))
normals = normals.reshape((bs, nf, 3))
if direction.ndim == 2:
direction = cf.broadcast_to(direction[:, None, :], normals.shape)
cos = cf.relu(cf.sum(normals * direction, axis=2))
light = (
light + intensity_directional * cfmath.mul(*cf.broadcast(color_directional[:, None, :], cos[:, :, None])))
# apply
light = cf.broadcast_to(light[:, :, None, None, None, :], textures.shape)
textures = textures * light
return textures
def inflation_loss(vertices, faces, degrees, eps=1e-5):
assert vertices.ndim == 3
assert faces.ndim == 2
v0 = vertices[:, faces[:, 0], :]
v1 = vertices[:, faces[:, 1], :]
v2 = vertices[:, faces[:, 2], :]
batch_size, num_faces = v0.shape[:2]
v0 = cf.reshape(v0, (batch_size * num_faces, 3))
v1 = cf.reshape(v1, (batch_size * num_faces, 3))
v2 = cf.reshape(v2, (batch_size * num_faces, 3))
norms = neural_renderer.cross(v1 - v0, v2 - v0) # [bs * nf, 3]
norms = cf.normalize(norms)
v0_t = (v0 + norms).data
v1_t = (v1 + norms).data
v2_t = (v2 + norms).data
loss_v0 = cf.sum(cf.sqrt(cf.sum(cf.square(v0_t - v0), 1) + eps))
loss_v1 = cf.sum(cf.sqrt(cf.sum(cf.square(v1_t - v1), 1) + eps))
loss_v2 = cf.sum(cf.sqrt(cf.sum(cf.square(v2_t - v2), 1) + eps))
loss = loss_v0 + loss_v1 + loss_v2
loss /= batch_size
return loss
def sample_points(faces, num_points=16384):
if faces.ndim == 3:
faces = faces[None, :, :]
batch_size = faces.shape[0]
v1 = faces[:, :, 1] - faces[:, :, 0]
v2 = faces[:, :, 2] - faces[:, :, 0]
s = (neural_renderer.cross(v1.reshape((-1, 3)), v2.reshape(
(-1, 3))).data**2).sum(-1)**0.5
s = s.reshape((batch_size, -1))
s = s / s.sum()
c = s.cumsum(1)
p = cp.tile(np.arange(0, 1, 1. / num_points)[None, :], (batch_size, 1))
i = (p[:, :, None] <= c[:, None, :]).argmax(2)
vs = cp.zeros((batch_size, num_points, 3), 'float32')
for bn in range(batch_size):
v0 = faces[bn, i[bn], 0]
v1 = faces[bn, i[bn], 1]
v2 = faces[bn, i[bn], 2]
r1 = cp.tile(cp.random.uniform(0, 1, v1.shape[0])[:, None], (1, 3))
r2 = cp.tile(cp.random.uniform(0, 1, v1.shape[0])[:, None], (1, 3))
v = (v0 * (1 - (r1**0.5)) + v1 * (r1**0.5) * (1 - r2) + v2 *
(r1**0.5) * r2)
vs[bn] = v
return vs