def _simple_renderer(rn, meshes, yrot=0, texture=None, use_light=False): mesh = meshes[0] if texture is not None: if not hasattr(mesh, 'ft'): mesh.ft = _copy(mesh.f) vt = _copy(mesh.v[:, :2]) vt -= _np.min(vt, axis=0).reshape((1, -1)) vt /= _np.max(vt, axis=0).reshape((1, -1)) mesh.vt = vt mesh.texture_filepath = rn.texture_image # Set camera parameters if texture is not None: rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, ft=mesh.ft, vt=mesh.vt, bgcolor=_np.ones(3)) else: rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, bgcolor=_np.ones(3)) for next_mesh in meshes[1:]: _stack_with(rn, next_mesh, texture) # Construct light. if use_light: albedo = rn.vc rn.vc = _odr_l.LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=_rotateY( _np.array([-200, -100, -100]), yrot), vc=albedo, light_color=_np.array([1, 1, 1])) # Construct Left Light rn.vc += _odr_l.LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=_rotateY( _np.array([800, 10, 300]), yrot), vc=albedo, light_color=_np.array([1, 1, 1])) # Construct Right Light rn.vc += _odr_l.LambertianPointLight( f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=_rotateY(_np.array([-500, 500, 1000]), yrot), vc=albedo, light_color=_np.array([.7, .7, .7])) return rn.r
def simple_renderer(rn, meshes, yrot=0): """Create a renderer, optionally with texture.""" mesh = meshes[0] if hasattr(rn, 'texture_image'): if not hasattr(mesh, 'ft'): mesh.ft = _copy(mesh.f) vt = _copy(mesh.v[:, :2]) vt -= _np.min(vt, axis=0).reshape((1, -1)) vt /= _np.max(vt, axis=0).reshape((1, -1)) mesh.vt = vt mesh.texture_filepath = rn.texture_image rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, ft=mesh.ft, vt=mesh.vt, bgcolor=_np.ones(3)) else: rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, bgcolor=_np.ones(3)) for next_mesh in meshes[1:]: _stack_with(rn, next_mesh) # pylint: disable=undefined-variable albedo = rn.vc # Construct Back Light (on back right corner) rn.vc = _odr_l.LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=_rotateY( _np.array([-200, -100, -100]), yrot), vc=albedo, light_color=_np.array([1, 1, 1])) # Construct Left Light rn.vc += _odr_l.LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=_rotateY( _np.array([800, 10, 300]), yrot), vc=albedo, light_color=_np.array([1, 1, 1])) # Construct Right Light rn.vc += _odr_l.LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=_rotateY( _np.array([-500, 500, 1000]), yrot), vc=albedo, light_color=_np.array([.7, .7, .7])) return rn.r
def render(model, resolution, cam, steps, center=(0, 0), segmented=False, use_light=False, path_to_mesh=None): # pylint: disable=too-many-arguments """Render a sequence of views from a fitted body model.""" assert steps >= 1 if segmented: texture = _os.path.join(_os.path.dirname(__file__), '..', 'models', '3D', 'mask_filled.png') else: texture = _os.path.join(_os.path.dirname(__file__), '..', 'models', '3D', 'mask_filled_uniform.png') if path_to_mesh is None: mesh = _copy(_TEMPLATE_MESH) else: mesh = _copy(_Mesh(path_to_mesh)) # render ply model.betas[:len(cam['betas'])] = cam['betas'] model.pose[:] = cam['pose'] model.trans[:] = cam['trans'] mesh.v = model.r w, h = resolution[0], resolution[1] dist = _np.abs(cam['t'][2] - _np.mean(mesh.v, axis=0)[2]) rn = _create_renderer( w=w, h=h, near=1., far=20. + dist, rt=_np.array(cam['rt']), t=_np.array(cam['t']), f=_np.array([cam['f'], cam['f']]), # c=_np.array(cam['cam_c']), texture=texture) light_yrot = _np.radians(120) baked_mesh = bake_vertex_colors(mesh) base_mesh = _copy(baked_mesh) mesh.f = base_mesh.f mesh.vc = base_mesh.vc renderings = [] for angle in _np.linspace(0., 2. * (1. - 1. / steps) * _np.pi, steps): mesh.v = _rotateY(base_mesh.v, angle) imtmp = _simple_renderer(rn=rn, meshes=[mesh], yrot=light_yrot, texture=texture, use_light=use_light) im = _np.zeros(h * w * 3).reshape(((h, w, 3))) im[:h, :w, :] = imtmp * 255. renderings.append(im) return renderings
def render(model, resolution, cam, steps, center=(0,0), segmented=False, use_light=False, path_to_mesh=None, color_type="segment"): # pylint: disable=too-many-arguments """Render a sequence of views from a fitted body model.""" assert steps >= 1 if segmented: texture = _os.path.join(_os.path.dirname(__file__), '..', 'models', '3D', 'mask_filled.png') else: texture = _os.path.join(_os.path.dirname(__file__), '..', 'models', '3D', 'mask_filled_uniform.png') if path_to_mesh is None: mesh = _copy(_TEMPLATE_MESH) else: mesh = _copy(_Mesh(path_to_mesh)) # render ply model.betas[:len(cam['betas'])] = cam['betas'] model.pose[:] = cam['pose'] model.trans[:] = cam['trans'] mesh.v = model.r w, h = resolution[0], resolution[1] dist = _np.abs(cam['t'][2] - _np.mean(mesh.v, axis=0)[2]) rn = _create_renderer(w=w, h=h, near=1., far=20.+dist, rt=_np.array(cam['rt']), t=_np.array(cam['t']), f=_np.array([cam['f'], cam['f']]), # c=_np.array(cam['cam_c']), texture=None, depth=(color_type == "depth")) light_yrot = _np.radians(120) meshes = [] if color_type == "segment" or color_type == "depth": baked_mesh = bake_vertex_colors(mesh) base_mesh = _copy(baked_mesh) mesh.f = base_mesh.f mesh.vc = base_mesh.vc meshes = [_copy(mesh)] elif color_type == "weight": base_mesh = _copy(mesh) npweights = _np.array(model.weights) print(npweights) for i in range(24): nv = len(mesh.v) vc = _np.zeros_like(mesh.v) for iv, v in enumerate(mesh.v): v = mesh.v[iv] warray = npweights[iv] vc[iv] = (warray[i], 0, 0) mesh.vc = vc meshes.append(_copy(mesh)) else: _LOGGER.warn("this color type is not yet supported") renderings = [] for mesh in meshes: for angle in _np.linspace(0., 2. * (1. - 1. / steps) * _np.pi, steps): mesh.v = _rotateY(base_mesh.v, angle) imtmp = _simple_renderer(rn=rn, meshes=[mesh], yrot=light_yrot, texture=None, use_light=use_light, depth=(color_type == "depth")) if color_type == "depth": import scipy.misc as sm sm.imsave("/tmp/depth.png", imtmp) _np.set_printoptions(threshold=40) im = _np.zeros(h*w).reshape(((h, w))) im[:h, :w] = imtmp else: im = _np.zeros(h*w*3).reshape(((h, w, 3))) im[:h, :w, :] = imtmp*255. renderings.append(im) return renderings
def run_single_fit( img, # pylint: disable=too-many-statements, too-many-locals j2d, scale, do_degrees=None): """Run the fit for one specific image.""" global _DEPTH_EST, _SHAPE_EST, _ROT_EST, _POSE_EST # pylint: disable=global-statement assert j2d.shape[0] == 3 assert j2d.shape[1] == 91 conf = j2d[2, :].copy().reshape((-1, )) j2d = j2d[:2, :].copy() j2d_norm = j2d * scale # Center the data. mean = _np.mean(j2d_norm, axis=1) j2d_norm = (j2d_norm.T - mean + 513. / 2.).T _LOGGER.debug("Running fit...") if do_degrees is None: do_degrees = [] # Prepare the estimators if necessary. if _DEPTH_EST is None: _DEPTH_EST = [ None, _pymp.shared.array(j2d.shape, dtype='float32'), _pymp.shared.array((3, ), dtype='float32'), _pymp.shared.queue(), _pymp.shared.lock(), _pymp.shared.queue() ] _DEPTH_EST[0] = _multiprocessing.Process(target=_depth_estimator, args=tuple(_DEPTH_EST[1:])) _DEPTH_EST[0].start() _DEPTH_EST[5].get() if _ROT_EST is None: _ROT_EST = [ None, _pymp.shared.array(j2d.shape, dtype='float32'), _pymp.shared.array((3, ), dtype='float32'), _pymp.shared.queue(), _pymp.shared.lock(), _pymp.shared.queue() ] _ROT_EST[0] = _multiprocessing.Process(target=_rot_estimator, args=tuple(_ROT_EST[1:])) _ROT_EST[0].start() _ROT_EST[5].get() if _SHAPE_EST is None: _SHAPE_EST = [ None, _pymp.shared.array(j2d.shape, dtype='float32'), _pymp.shared.array((10, ), dtype='float32'), _pymp.shared.queue(), _pymp.shared.lock(), _pymp.shared.queue() ] _SHAPE_EST[0] = _multiprocessing.Process(target=_shape_estimator, args=tuple(_SHAPE_EST[1:])) _SHAPE_EST[0].start() _SHAPE_EST[5].get() if _POSE_EST is None: _POSE_EST = [ None, _pymp.shared.array(j2d.shape, dtype='float32'), _pymp.shared.array((69, ), dtype='float32'), _pymp.shared.queue(), _pymp.shared.lock(), _pymp.shared.queue() ] _POSE_EST[0] = _multiprocessing.Process(target=_pose_estimator, args=tuple(_POSE_EST[1:])) _POSE_EST[0].start() _POSE_EST[5].get() # Copy the data to the processes. with _POSE_EST[4]: _POSE_EST[1][...] = j2d_norm with _SHAPE_EST[4]: _SHAPE_EST[1][...] = j2d_norm with _ROT_EST[4]: _ROT_EST[1][...] = j2d_norm with _DEPTH_EST[4]: _DEPTH_EST[1][...] = j2d_norm # Run it. before_fit = _time() _POSE_EST[3].put('go') _ROT_EST[3].put('go') _SHAPE_EST[3].put('go') _DEPTH_EST[3].put('go') _LOGGER.info("Running...") _DEPTH_EST[5].get() _POSE_EST[5].get() _SHAPE_EST[5].get() _ROT_EST[5].get() _LOGGER.info("Prediction available in %ss.", str(_time() - before_fit)) # Extract the results. pose = _np.zeros((72, ), dtype='float32') betas = _np.zeros((10, ), dtype='float32') trans = _np.zeros((3, ), dtype='float32') with _POSE_EST[4]: pose[3:] = _POSE_EST[2] with _SHAPE_EST[4]: betas[:] = _SHAPE_EST[2] with _ROT_EST[4]: pose[:3] = _ROT_EST[2] with _DEPTH_EST[4]: trans[:] = _DEPTH_EST[2] trans[2] *= scale # Get the projected landmark locations from the model. param_dict = { 't': [0, 0, 0], 'rt': [0, 0, 0], 'f': _FLENGTH_GUESS, 'pose': pose, 'trans': trans, 'betas': betas } # Optimize depth and global rotation. opt_globrot, opt_trans, dmin, dmax = _fit_rot_trans( _MODEL_NEUTRAL, j2d, [img.shape[1] // 2, img.shape[0] // 2], trans, pose, conf, _FLENGTH_GUESS) pose[:3] = opt_globrot trans[:] = opt_trans """ proj_landmark_positions = get_landmark_positions(param_dict, (513, 513), _LANDMARK_MAPPING) # Get the right offset to match the original. offset = _np.mean(j2d, axis=1) - _np.mean(proj_landmark_positions, axis=1) """ # Render the optimized mesh. _LOGGER.info("Rendering...") mesh = _copy(_TEMPLATE_MESH) model = _MODEL_NEUTRAL model.betas[:len(betas)] = betas # Get the full rendered mesh. model.pose[:] = pose model.trans[:] = trans mesh.v = model.r mesh.vc = [.7, .7, .9] base_mesh_v = mesh.v.copy() images = [] for deg in do_degrees: mesh.v = _rotateY(base_mesh_v.copy(), deg) rn = create_renderer(w=img.shape[1], h=img.shape[0], near=dmin - 1., far=dmax + 1., rt=[0., 0., 0.], t=[0., 0., 0.], f=[_FLENGTH_GUESS, _FLENGTH_GUESS], c=[img.shape[1] // 2, img.shape[0] // 2]) # + offset[1]]) light_yrot = _np.radians(120) im = (simple_renderer(rn=rn, meshes=[mesh], yrot=light_yrot) * 255.).astype('uint8') images.append(im) #param_dict['j2d'] = (proj_landmark_positions.T + offset).T _LOGGER.info("Estimation done.") return param_dict, images
def _simple_renderer(rn, meshes, yrot=0, texture=None, use_light=False): mesh = meshes[0] if texture is not None: if not hasattr(mesh, 'ft'): mesh.ft = _copy(mesh.f) vt = _copy(mesh.v[:, :2]) vt -= _np.min(vt, axis=0).reshape((1, -1)) vt /= _np.max(vt, axis=0).reshape((1, -1)) mesh.vt = vt mesh.texture_filepath = rn.texture_image # Set camera parameters if texture is not None: rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, ft=mesh.ft, vt=mesh.vt, bgcolor=_np.ones(3)) else: rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, bgcolor=_np.ones(3)) for next_mesh in meshes[1:]: _stack_with(rn, next_mesh, texture) # Construct light. if use_light: albedo = rn.vc rn.vc = _odr_l.LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=_rotateY( _np.array([-200, -100, -100]), yrot), vc=albedo, light_color=_np.array([1, 1, 1])) # Construct Left Light rn.vc += _odr_l.LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=_rotateY( _np.array([800, 10, 300]), yrot), vc=albedo, light_color=_np.array([1, 1, 1])) # Construct Right Light rn.vc += _odr_l.LambertianPointLight( f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=_rotateY(_np.array([-500, 500, 1000]), yrot), vc=albedo, light_color=_np.array([.7, .7, .7])) flipXRotation = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, -1.0, 0., 0.0], [0.0, 0., -1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) rn.camera.openglMat = flipXRotation # this is from setupcamera in utils rn.glMode = 'glfw' rn.sharedWin = None rn.overdraw = True rn.nsamples = 8 rn.msaa = True # Without anti-aliasing optimization often does not work. rn.initGL() rn.debug = False return rn.r