def test_forward(self): a_cpu = np.random.normal(size=(10, 3)).astype('float32') b_cpu = np.random.normal(size=(10, 3)).astype('float32') c_ref = np.cross(a_cpu, b_cpu) a_gpu = chainer.cuda.to_gpu(a_cpu) b_gpu = chainer.cuda.to_gpu(b_cpu) c_cpu = neural_renderer.cross(a_cpu, b_cpu).data c_gpu = neural_renderer.cross(a_gpu, b_gpu).data chainer.testing.assert_allclose(c_ref, c_cpu) chainer.testing.assert_allclose(c_ref, chainer.cuda.to_cpu(c_gpu))
def render_normal(self, vertices, faces): # fill back if self.fill_back: faces = cf.concat((faces, faces[:, :, ::-1]), axis=1).data # normal faces_normal = nr.vertices_to_faces(vertices, faces) (bs, nf) = faces_normal.shape[:2] faces_normal = faces_normal.reshape((bs * nf, 3, 3)) v10 = faces_normal[:, 0] - faces_normal[:, 1] v12 = faces_normal[:, 2] - faces_normal[:, 1] normals = cf.normalize(nr.cross(v10, v12)) normals = normals.reshape((bs, nf, 3)) textures = normals[:, :, None, None, None, :] textures = cf.tile(textures, (1, 1, 2, 2, 2, 1)) # viewpoint transformation if self.camera_mode == 'look_at': vertices = nr.look_at(vertices, self.eye) elif self.camera_mode == 'look': vertices = nr.look(vertices, self.eye, self.camera_direction, self.up) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) # rasterization faces = nr.vertices_to_faces(vertices, faces) images = nr.rasterize( faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) return images
def look_at(vertices, eye, at=None, up=None): """ "Look at" transformation of vertices. """ assert (vertices.ndim == 3) xp = chainer.cuda.get_array_module(vertices) batch_size = vertices.shape[0] if at is None: at = xp.array([0, 0, 0], 'float32') if up is None: up = xp.array([0, 1, 0], 'float32') if isinstance(eye, list) or isinstance(eye, tuple): eye = xp.array(eye, 'float32') if eye.ndim == 1: eye = cf.tile(eye[None, :], (batch_size, 1)) if at.ndim == 1: at = cf.tile(at[None, :], (batch_size, 1)) if up.ndim == 1: up = cf.tile(up[None, :], (batch_size, 1)) # create new axes z_axis = cf.normalize(at - eye) x_axis = cf.normalize(neural_renderer.cross(up, z_axis)) y_axis = cf.normalize(neural_renderer.cross(z_axis, x_axis)) # create rotation matrix: [bs, 3, 3] r = cf.concat((x_axis[:, None, :], y_axis[:, None, :], z_axis[:, None, :]), axis=1) if r.shape[0] != vertices.shape[0]: r = cf.broadcast_to(r, (vertices.shape[0], 3, 3)) # apply # [bs, nv, 3] -> [bs, nv, 3] -> [bs, nv, 3] if vertices.shape != eye.shape: eye = cf.broadcast_to(eye[:, None, :], vertices.shape) vertices = vertices - eye vertices = cf.matmul(vertices, r, transb=True) return vertices
def look(vertices, viewpoints, direction=None, up=None): """ "Look at" transformation of vertices. """ assert (vertices.ndim == 3) xp = chainer.cuda.get_array_module(vertices) if direction is None: direction = xp.array([0, 0, 1], 'float32') if up is None: up = xp.array([0, 1, 0], 'float32') if isinstance(viewpoints, list) or isinstance(viewpoints, tuple): viewpoints = xp.array(viewpoints, 'float32') if viewpoints.ndim == 1: viewpoints = viewpoints[None, :] if direction.ndim == 1: direction = direction[None, :] if up.ndim == 1: up = up[None, :] # create new axes z_axis = cf.normalize(direction) x_axis = cf.normalize(neural_renderer.cross(up, z_axis)) y_axis = cf.normalize(neural_renderer.cross(z_axis, x_axis)) # create rotation matrix: [bs, 3, 3] r = cf.concat((x_axis[:, None, :], y_axis[:, None, :], z_axis[:, None, :]), axis=1) if r.shape[0] != vertices.shape[0]: r = cf.broadcast_to(r, vertices.shape) # apply # [bs, nv, 3] -> [bs, nv, 3] -> [bs, nv, 3] if vertices.shape != viewpoints.shape: viewpoints = cf.broadcast_to(viewpoints[:, None, :], vertices.shape) vertices = vertices - viewpoints vertices = cf.matmul(vertices, r, transb=True) return vertices
def look_at(vertices, eye, at=None, up=None): """ "Look at" transformation of vertices. """ assert (vertices.ndim == 3) xp = chainer.cuda.get_array_module(vertices) batch_size = vertices.shape[0] if at is None: at = xp.array([0, 0, 0], 'float32') if up is None: up = xp.array([0, 1, 0], 'float32') if isinstance(eye, list) or isinstance(eye, tuple): eye = xp.array(eye, 'float32') if eye.ndim == 1: eye = cf.tile(eye[None, :], (batch_size, 1)) if at.ndim == 1: at = cf.tile(at[None, :], (batch_size, 1)) if up.ndim == 1: up = cf.tile(up[None, :], (batch_size, 1)) # create new axes z_axis = cf.normalize(at - eye) x_axis = cf.normalize(neural_renderer.cross(up, z_axis)) y_axis = cf.normalize(neural_renderer.cross(z_axis, x_axis)) # create rotation matrix: [bs, 3, 3] r = cf.concat((x_axis[:, None, :], y_axis[:, None, :], z_axis[:, None, :]), axis=1) if r.shape[0] != vertices.shape[0]: r = cf.broadcast_to(r, (vertices.shape[0], 3, 3)) # apply # [bs, nv, 3] -> [bs, nv, 3] -> [bs, nv, 3] if vertices.shape != eye.shape: eye = cf.broadcast_to(eye[:, None, :], vertices.shape) vertices = vertices - eye vertices = cf.matmul(vertices, r, transb=True) return vertices
def lighting(faces, textures, intensity_ambient=0.5, intensity_directional=0.5, color_ambient=(1, 1, 1), color_directional=(1, 1, 1), direction=(0, 1, 0)): xp = chainer.cuda.get_array_module(faces) bs, nf = faces.shape[:2] # arguments if isinstance(color_ambient, tuple) or isinstance(color_ambient, list): color_ambient = xp.array(color_ambient, 'float32') if isinstance(color_directional, tuple) or isinstance( color_directional, list): color_directional = xp.array(color_directional, 'float32') if isinstance(direction, tuple) or isinstance(direction, list): direction = xp.array(direction, 'float32') if color_ambient.ndim == 1: color_ambient = cf.broadcast_to(color_ambient[None, :], (bs, 3)) if color_directional.ndim == 1: color_directional = cf.broadcast_to(color_directional[None, :], (bs, 3)) if direction.ndim == 1: direction = cf.broadcast_to(direction[None, :], (bs, 3)) # create light light = xp.zeros((bs, nf, 3), 'float32') # ambient light if intensity_ambient != 0: light = light + intensity_ambient * cf.broadcast_to( color_ambient[:, None, :], light.shape) # directional light if intensity_directional != 0: faces = faces.reshape((bs * nf, 3, 3)) v10 = faces[:, 0] - faces[:, 1] v12 = faces[:, 2] - faces[:, 1] normals = cf.normalize(neural_renderer.cross(v10, v12)) normals = normals.reshape((bs, nf, 3)) if direction.ndim == 2: direction = cf.broadcast_to(direction[:, None, :], normals.shape) cos = cf.relu(cf.sum(normals * direction, axis=2)) light = (light + intensity_directional * cfmath.mul( *cf.broadcast(color_directional[:, None, :], cos[:, :, None]))) # apply light = cf.broadcast_to(light[:, :, None, None, None, :], textures.shape) textures = textures * light return textures
def lighting( faces, textures, intensity_ambient=0.5, intensity_directional=0.5, color_ambient=(1, 1, 1), color_directional=(1, 1, 1), direction=(0, 1, 0)): xp = chainer.cuda.get_array_module(faces) bs, nf = faces.shape[:2] # arguments if isinstance(color_ambient, tuple) or isinstance(color_ambient, list): color_ambient = xp.array(color_ambient, 'float32') if isinstance(color_directional, tuple) or isinstance(color_directional, list): color_directional = xp.array(color_directional, 'float32') if isinstance(direction, tuple) or isinstance(direction, list): direction = xp.array(direction, 'float32') if color_ambient.ndim == 1: color_ambient = cf.broadcast_to(color_ambient[None, :], (bs, 3)) if color_directional.ndim == 1: color_directional = cf.broadcast_to(color_directional[None, :], (bs, 3)) if direction.ndim == 1: direction = cf.broadcast_to(direction[None, :], (bs, 3)) # create light light = xp.zeros((bs, nf, 3), 'float32') # ambient light if intensity_ambient != 0: light = light + intensity_ambient * cf.broadcast_to(color_ambient[:, None, :], light.shape) # directional light if intensity_directional != 0: faces = faces.reshape((bs * nf, 3, 3)) v10 = faces[:, 0] - faces[:, 1] v12 = faces[:, 2] - faces[:, 1] normals = cf.normalize(neural_renderer.cross(v10, v12)) normals = normals.reshape((bs, nf, 3)) if direction.ndim == 2: direction = cf.broadcast_to(direction[:, None, :], normals.shape) cos = cf.relu(cf.sum(normals * direction, axis=2)) light = ( light + intensity_directional * cfmath.mul(*cf.broadcast(color_directional[:, None, :], cos[:, :, None]))) # apply light = cf.broadcast_to(light[:, :, None, None, None, :], textures.shape) textures = textures * light return textures
def inflation_loss(vertices, faces, degrees, eps=1e-5): assert vertices.ndim == 3 assert faces.ndim == 2 v0 = vertices[:, faces[:, 0], :] v1 = vertices[:, faces[:, 1], :] v2 = vertices[:, faces[:, 2], :] batch_size, num_faces = v0.shape[:2] v0 = cf.reshape(v0, (batch_size * num_faces, 3)) v1 = cf.reshape(v1, (batch_size * num_faces, 3)) v2 = cf.reshape(v2, (batch_size * num_faces, 3)) norms = neural_renderer.cross(v1 - v0, v2 - v0) # [bs * nf, 3] norms = cf.normalize(norms) v0_t = (v0 + norms).data v1_t = (v1 + norms).data v2_t = (v2 + norms).data loss_v0 = cf.sum(cf.sqrt(cf.sum(cf.square(v0_t - v0), 1) + eps)) loss_v1 = cf.sum(cf.sqrt(cf.sum(cf.square(v1_t - v1), 1) + eps)) loss_v2 = cf.sum(cf.sqrt(cf.sum(cf.square(v2_t - v2), 1) + eps)) loss = loss_v0 + loss_v1 + loss_v2 loss /= batch_size return loss
def sample_points(faces, num_points=16384): if faces.ndim == 3: faces = faces[None, :, :] batch_size = faces.shape[0] v1 = faces[:, :, 1] - faces[:, :, 0] v2 = faces[:, :, 2] - faces[:, :, 0] s = (neural_renderer.cross(v1.reshape((-1, 3)), v2.reshape( (-1, 3))).data**2).sum(-1)**0.5 s = s.reshape((batch_size, -1)) s = s / s.sum() c = s.cumsum(1) p = cp.tile(np.arange(0, 1, 1. / num_points)[None, :], (batch_size, 1)) i = (p[:, :, None] <= c[:, None, :]).argmax(2) vs = cp.zeros((batch_size, num_points, 3), 'float32') for bn in range(batch_size): v0 = faces[bn, i[bn], 0] v1 = faces[bn, i[bn], 1] v2 = faces[bn, i[bn], 2] r1 = cp.tile(cp.random.uniform(0, 1, v1.shape[0])[:, None], (1, 3)) r2 = cp.tile(cp.random.uniform(0, 1, v1.shape[0])[:, None], (1, 3)) v = (v0 * (1 - (r1**0.5)) + v1 * (r1**0.5) * (1 - r2) + v2 * (r1**0.5) * r2) vs[bn] = v return vs