class Camera(object): CONFIG = { "background_image": None, "frame_config": {}, "pixel_width": DEFAULT_PIXEL_WIDTH, "pixel_height": DEFAULT_PIXEL_HEIGHT, "frame_rate": DEFAULT_FRAME_RATE, # Note: frame height and width will be resized to match # the pixel aspect ratio "background_color": BLACK, "background_opacity": 1, # Points in vectorized mobjects with norm greater # than this value will be rescaled. "max_allowable_norm": FRAME_WIDTH, "image_mode": "RGBA", "n_channels": 4, "pixel_array_dtype": 'uint8', "light_source_position": [-10, 10, 10], # Measured in pixel widths, used for vector graphics "anti_alias_width": 1.5, # Although vector graphics handle antialiasing fine # without multisampling, for 3d scenes one might want # to set samples to be greater than 0. "samples": 0, } def __init__(self, ctx=None, **kwargs): digest_config(self, kwargs, locals()) self.rgb_max_val = np.iinfo(self.pixel_array_dtype).max self.background_rgba = [ *Color(self.background_color).get_rgb(), self.background_opacity ] self.init_frame() # 初始化OpenGL self.init_context(ctx) self.init_shaders() self.init_textures() self.init_light_source() self.refresh_perspective_uniforms() self.static_mobject_to_render_group_list = {} def init_frame(self): self.frame = CameraFrame(**self.frame_config) def init_context(self, ctx=None): if ctx is None: ctx = moderngl.create_standalone_context() fbo = self.get_fbo(ctx, 0) else: fbo = ctx.detect_framebuffer() # For multisample antialiasing fbo_msaa = self.get_fbo(ctx, self.samples) fbo_msaa.use() ctx.enable(moderngl.BLEND) ctx.blend_func = (moderngl.SRC_ALPHA, moderngl.ONE_MINUS_SRC_ALPHA, moderngl.ONE, moderngl.ONE) self.ctx = ctx self.fbo = fbo self.fbo_msaa = fbo_msaa def init_light_source(self): self.light_source = Point(self.light_source_position) # Methods associated with the frame buffer def get_fbo(self, ctx, samples=0): pw = self.pixel_width ph = self.pixel_height return ctx.framebuffer(color_attachments=ctx.texture( (pw, ph), components=self.n_channels, samples=samples, ), depth_attachment=ctx.depth_renderbuffer( (pw, ph), samples=samples)) def clear(self): self.fbo.clear(*self.background_rgba) self.fbo_msaa.clear(*self.background_rgba) def reset_pixel_shape(self, new_width, new_height): self.pixel_width = new_width self.pixel_height = new_height self.refresh_perspective_uniforms() def get_raw_fbo_data(self, dtype='f1'): # Copy blocks from the fbo_msaa to the drawn fbo using Blit pw, ph = (self.pixel_width, self.pixel_height) gl.glBindFramebuffer(gl.GL_READ_FRAMEBUFFER, self.fbo_msaa.glo) gl.glBindFramebuffer(gl.GL_DRAW_FRAMEBUFFER, self.fbo.glo) gl.glBlitFramebuffer(0, 0, pw, ph, 0, 0, pw, ph, gl.GL_COLOR_BUFFER_BIT, gl.GL_LINEAR) return self.fbo.read( viewport=self.fbo.viewport, components=self.n_channels, dtype=dtype, ) def get_image(self, pixel_array=None): return Image.frombytes('RGBA', self.get_pixel_shape(), self.get_raw_fbo_data(), 'raw', 'RGBA', 0, -1) def get_pixel_array(self): raw = self.get_raw_fbo_data(dtype='f4') flat_arr = np.frombuffer(raw, dtype='f4') arr = flat_arr.reshape([*self.fbo.size, self.n_channels]) # Convert from float return (self.rgb_max_val * arr).astype(self.pixel_array_dtype) # Needed? def get_texture(self): texture = self.ctx.texture(size=self.fbo.size, components=4, data=self.get_raw_fbo_data(), dtype='f4') return texture # Getting camera attributes def get_pixel_shape(self): return self.fbo.viewport[2:4] # return (self.pixel_width, self.pixel_height) def get_pixel_width(self): return self.get_pixel_shape()[0] def get_pixel_height(self): return self.get_pixel_shape()[1] def get_frame_height(self): return self.frame.get_height() def get_frame_width(self): return self.frame.get_width() def get_frame_shape(self): return (self.get_frame_width(), self.get_frame_height()) def get_frame_center(self): return self.frame.get_center() def resize_frame_shape(self, fixed_dimension=0): """ Changes frame_shape to match the aspect ratio of the pixels, where fixed_dimension determines whether frame_height or frame_width remains fixed while the other changes accordingly. """ pixel_height = self.get_pixel_height() pixel_width = self.get_pixel_width() frame_height = self.get_frame_height() frame_width = self.get_frame_width() aspect_ratio = fdiv(pixel_width, pixel_height) if fixed_dimension == 0: frame_height = frame_width / aspect_ratio else: frame_width = aspect_ratio * frame_height self.frame.set_height(frame_height) self.frame.set_width(frame_width) def pixel_coords_to_space_coords(self, px, py, relative=False): pw, ph = self.fbo.size fw, fh = self.get_frame_shape() fc = self.get_frame_center() if relative: return 2 * np.array([px / pw, py / ph, 0]) else: # Only scale wrt one axis scale = fh / ph return fc + scale * np.array([(px - pw / 2), (py - ph / 2), 0]) # Rendering def capture(self, *mobjects, **kwargs): # 刷新摄像机 self.refresh_perspective_uniforms() # 绘制每一个对象 for mobject in mobjects: for render_group in self.get_render_group_list(mobject): self.render(render_group) def render(self, render_group): shader_wrapper = render_group["shader_wrapper"] shader_program = render_group["prog"] self.set_shader_uniforms(shader_program, shader_wrapper) self.update_depth_test(shader_wrapper) # 渲染顶点数据 render_group["vao"].render(int(shader_wrapper.render_primitive)) # if render_group["single_use"]: self.release_render_group(render_group) def update_depth_test(self, shader_wrapper): if shader_wrapper.depth_test: self.ctx.enable(moderngl.DEPTH_TEST) else: self.ctx.disable(moderngl.DEPTH_TEST) def get_render_group_list(self, mobject): try: return self.static_mobject_to_render_group_list[id(mobject)] except KeyError: return map(self.get_render_group, mobject.get_shader_wrapper_list()) def get_render_group(self, shader_wrapper, single_use=True): # Data buffers vbo = self.ctx.buffer(shader_wrapper.vert_data.tobytes()) if shader_wrapper.vert_indices is None: ibo = None else: vert_index_data = shader_wrapper.vert_indices.astype( 'i4').tobytes() if vert_index_data: ibo = self.ctx.buffer(vert_index_data) else: ibo = None # Program and vertex array shader_program, vert_format = self.get_shader_program(shader_wrapper) # vao = self.ctx.vertex_array( program=shader_program, content=[(vbo, vert_format, *shader_wrapper.vert_attributes)], index_buffer=ibo, ) return { "vbo": vbo, "ibo": ibo, "vao": vao, "prog": shader_program, "shader_wrapper": shader_wrapper, "single_use": single_use, } def release_render_group(self, render_group): for key in ["vbo", "ibo", "vao"]: if render_group[key] is not None: render_group[key].release() def set_mobjects_as_static(self, *mobjects): # Creates buffer and array objects holding each mobjects shader data for mob in mobjects: self.static_mobject_to_render_group_list[id(mob)] = [ self.get_render_group(sw, single_use=False) for sw in mob.get_shader_wrapper_list() ] def release_static_mobjects(self): for rg_list in self.static_mobject_to_render_group_list.values(): for render_group in rg_list: self.release_render_group(render_group) self.static_mobject_to_render_group_list = {} # Shaders def init_shaders(self): # Initialize with the null id going to None self.id_to_shader_program = {"": None} def get_shader_program(self, shader_wrapper): sid = shader_wrapper.get_program_id() if sid not in self.id_to_shader_program: # Create shader program for the first time, then cache # in the id_to_shader_program dictionary program = self.ctx.program(**shader_wrapper.get_program_code()) vert_format = moderngl.detect_format( program, shader_wrapper.vert_attributes) self.id_to_shader_program[sid] = (program, vert_format) return self.id_to_shader_program[sid] def set_shader_uniforms(self, shader, shader_wrapper): for name, path in shader_wrapper.texture_paths.items(): tid = self.get_texture_id(path) shader[name].value = tid for name, value in it.chain(shader_wrapper.uniforms.items(), self.perspective_uniforms.items()): try: shader[name].value = value except KeyError: pass def refresh_perspective_uniforms(self): frame = self.frame pw, ph = self.get_pixel_shape() fw, fh = frame.get_shape() # TODO, this should probably be a mobject uniform, with # the camera taking care of the conversion factor anti_alias_width = self.anti_alias_width / (ph / fh) # Orient light rotation = frame.get_inverse_camera_rotation_matrix() light_pos = self.light_source.get_location() light_pos = np.dot(rotation, light_pos) self.perspective_uniforms = { "frame_shape": frame.get_shape(), "anti_alias_width": anti_alias_width, "camera_center": tuple(frame.get_center()), "camera_rotation": tuple(np.array(rotation).T.flatten()), "light_source_position": tuple(light_pos), "focal_distance": frame.get_focal_distance(), } def init_textures(self): self.path_to_texture_id = {} def get_texture_id(self, path): if path not in self.path_to_texture_id: # A way to increase tid's sequentially tid = len(self.path_to_texture_id) im = Image.open(path) texture = self.ctx.texture( size=im.size, components=len(im.getbands()), data=im.tobytes(), ) texture.use(location=tid) self.path_to_texture_id[path] = tid return self.path_to_texture_id[path]
class Camera(object): CONFIG = { "background_image": None, "frame_config": {}, "pixel_width": DEFAULT_PIXEL_WIDTH, "pixel_height": DEFAULT_PIXEL_HEIGHT, "fps": DEFAULT_FPS, # Note: frame height and width will be resized to match # the pixel aspect ratio "background_color": BLACK, "background_opacity": 1, # Points in vectorized mobjects with norm greater # than this value will be rescaled. "max_allowable_norm": FRAME_WIDTH, "image_mode": "RGBA", "n_channels": 4, "pixel_array_dtype": 'uint8', "light_source_position": [-10, 10, 10], # Measured in pixel widths, used for vector graphics "anti_alias_width": 1.5, # Although vector graphics handle antialiasing fine # without multisampling, for 3d scenes one might want # to set samples to be greater than 0. "samples": 0, } def __init__(self, ctx: moderngl.Context | None = None, **kwargs): digest_config(self, kwargs, locals()) self.rgb_max_val: float = np.iinfo(self.pixel_array_dtype).max self.background_rgba: list[float] = list( color_to_rgba(self.background_color, self.background_opacity)) self.init_frame() self.init_context(ctx) self.init_shaders() self.init_textures() self.init_light_source() self.refresh_perspective_uniforms() # A cached map from mobjects to their associated list of render groups # so that these render groups are not regenerated unnecessarily for static # mobjects self.mob_to_render_groups = {} def init_frame(self) -> None: self.frame = CameraFrame(**self.frame_config) def init_context(self, ctx: moderngl.Context | None = None) -> None: if ctx is None: ctx = moderngl.create_standalone_context() fbo = self.get_fbo(ctx, 0) else: fbo = ctx.detect_framebuffer() self.ctx = ctx self.fbo = fbo self.set_ctx_blending() # For multisample antialiasing fbo_msaa = self.get_fbo(ctx, self.samples) fbo_msaa.use() self.fbo_msaa = fbo_msaa def set_ctx_blending(self, enable: bool = True) -> None: if enable: self.ctx.enable(moderngl.BLEND) else: self.ctx.disable(moderngl.BLEND) self.ctx.blend_func = ( moderngl.SRC_ALPHA, moderngl.ONE_MINUS_SRC_ALPHA, # moderngl.ONE, moderngl.ONE ) def set_ctx_depth_test(self, enable: bool = True) -> None: if enable: self.ctx.enable(moderngl.DEPTH_TEST) else: self.ctx.disable(moderngl.DEPTH_TEST) def init_light_source(self) -> None: self.light_source = Point(self.light_source_position) # Methods associated with the frame buffer def get_fbo(self, ctx: moderngl.Context, samples: int = 0) -> moderngl.Framebuffer: pw = self.pixel_width ph = self.pixel_height return ctx.framebuffer(color_attachments=ctx.texture( (pw, ph), components=self.n_channels, samples=samples, ), depth_attachment=ctx.depth_renderbuffer( (pw, ph), samples=samples)) def clear(self) -> None: self.fbo.clear(*self.background_rgba) self.fbo_msaa.clear(*self.background_rgba) def reset_pixel_shape(self, new_width: int, new_height: int) -> None: self.pixel_width = new_width self.pixel_height = new_height self.refresh_perspective_uniforms() def get_raw_fbo_data(self, dtype: str = 'f1') -> bytes: # Copy blocks from the fbo_msaa to the drawn fbo using Blit pw, ph = (self.pixel_width, self.pixel_height) gl.glBindFramebuffer(gl.GL_READ_FRAMEBUFFER, self.fbo_msaa.glo) gl.glBindFramebuffer(gl.GL_DRAW_FRAMEBUFFER, self.fbo.glo) gl.glBlitFramebuffer(0, 0, pw, ph, 0, 0, pw, ph, gl.GL_COLOR_BUFFER_BIT, gl.GL_LINEAR) return self.fbo.read( viewport=self.fbo.viewport, components=self.n_channels, dtype=dtype, ) def get_image(self) -> Image.Image: return Image.frombytes('RGBA', self.get_pixel_shape(), self.get_raw_fbo_data(), 'raw', 'RGBA', 0, -1) def get_pixel_array(self) -> np.ndarray: raw = self.get_raw_fbo_data(dtype='f4') flat_arr = np.frombuffer(raw, dtype='f4') arr = flat_arr.reshape([*self.fbo.size, self.n_channels]) # Convert from float return (self.rgb_max_val * arr).astype(self.pixel_array_dtype) # Needed? def get_texture(self) -> moderngl.Texture: texture = self.ctx.texture(size=self.fbo.size, components=4, data=self.get_raw_fbo_data(), dtype='f4') return texture # Getting camera attributes def get_pixel_shape(self) -> tuple[int, int]: return self.fbo.viewport[2:4] # return (self.pixel_width, self.pixel_height) def get_pixel_width(self) -> int: return self.get_pixel_shape()[0] def get_pixel_height(self) -> int: return self.get_pixel_shape()[1] def get_frame_height(self) -> float: return self.frame.get_height() def get_frame_width(self) -> float: return self.frame.get_width() def get_frame_shape(self) -> tuple[float, float]: return (self.get_frame_width(), self.get_frame_height()) def get_frame_center(self) -> np.ndarray: return self.frame.get_center() def get_location(self) -> tuple[float, float, float]: return self.frame.get_implied_camera_location() def resize_frame_shape(self, fixed_dimension: bool = False) -> None: """ Changes frame_shape to match the aspect ratio of the pixels, where fixed_dimension determines whether frame_height or frame_width remains fixed while the other changes accordingly. """ pixel_height = self.get_pixel_height() pixel_width = self.get_pixel_width() frame_height = self.get_frame_height() frame_width = self.get_frame_width() aspect_ratio = fdiv(pixel_width, pixel_height) if not fixed_dimension: frame_height = frame_width / aspect_ratio else: frame_width = aspect_ratio * frame_height self.frame.set_height(frame_height) self.frame.set_width(frame_width) # Rendering def capture(self, *mobjects: Mobject, **kwargs) -> None: self.refresh_perspective_uniforms() for mobject in mobjects: for render_group in self.get_render_group_list(mobject): self.render(render_group) def render(self, render_group: dict[str]) -> None: shader_wrapper = render_group["shader_wrapper"] shader_program = render_group["prog"] self.set_shader_uniforms(shader_program, shader_wrapper) self.set_ctx_depth_test(shader_wrapper.depth_test) render_group["vao"].render(int(shader_wrapper.render_primitive)) if render_group["single_use"]: self.release_render_group(render_group) def get_render_group_list(self, mobject: Mobject) -> Iterable[dict[str]]: if mobject.is_changing(): return self.generate_render_group_list(mobject) # Otherwise, cache result for later use key = id(mobject) if key not in self.mob_to_render_groups: self.mob_to_render_groups[key] = list( self.generate_render_group_list(mobject)) return self.mob_to_render_groups[key] def generate_render_group_list(self, mobject: Mobject) -> Iterable[dict[str]]: return (self.get_render_group(sw, single_use=mobject.is_changing()) for sw in mobject.get_shader_wrapper_list()) def get_render_group(self, shader_wrapper: ShaderWrapper, single_use: bool = True) -> dict[str]: # Data buffers vbo = self.ctx.buffer(shader_wrapper.vert_data.tobytes()) if shader_wrapper.vert_indices is None: ibo = None else: vert_index_data = shader_wrapper.vert_indices.astype( 'i4').tobytes() if vert_index_data: ibo = self.ctx.buffer(vert_index_data) else: ibo = None # Program and vertex array shader_program, vert_format = self.get_shader_program(shader_wrapper) vao = self.ctx.vertex_array( program=shader_program, content=[(vbo, vert_format, *shader_wrapper.vert_attributes)], index_buffer=ibo, ) return { "vbo": vbo, "ibo": ibo, "vao": vao, "prog": shader_program, "shader_wrapper": shader_wrapper, "single_use": single_use, } def release_render_group(self, render_group: dict[str]) -> None: for key in ["vbo", "ibo", "vao"]: if render_group[key] is not None: render_group[key].release() def refresh_static_mobjects(self) -> None: for render_group in it.chain(*self.mob_to_render_groups.values()): self.release_render_group(render_group) self.mob_to_render_groups = {} # Shaders def init_shaders(self) -> None: # Initialize with the null id going to None self.id_to_shader_program: dict[int | str, tuple[moderngl.Program, str] | None] = { "": None } def get_shader_program( self, shader_wrapper: ShaderWrapper) -> tuple[moderngl.Program, str]: sid = shader_wrapper.get_program_id() if sid not in self.id_to_shader_program: # Create shader program for the first time, then cache # in the id_to_shader_program dictionary program = self.ctx.program(**shader_wrapper.get_program_code()) vert_format = moderngl.detect_format( program, shader_wrapper.vert_attributes) self.id_to_shader_program[sid] = (program, vert_format) return self.id_to_shader_program[sid] def set_shader_uniforms(self, shader: moderngl.Program, shader_wrapper: ShaderWrapper) -> None: for name, path in shader_wrapper.texture_paths.items(): tid = self.get_texture_id(path) shader[name].value = tid for name, value in it.chain(self.perspective_uniforms.items(), shader_wrapper.uniforms.items()): try: if isinstance(value, np.ndarray) and value.ndim > 0: value = tuple(value) shader[name].value = value except KeyError: pass def refresh_perspective_uniforms(self) -> None: frame = self.frame pw, ph = self.get_pixel_shape() fw, fh = frame.get_shape() # TODO, this should probably be a mobject uniform, with # the camera taking care of the conversion factor anti_alias_width = self.anti_alias_width / (ph / fh) # Orient light rotation = frame.get_inverse_camera_rotation_matrix() offset = frame.get_center() light_pos = np.dot(rotation, self.light_source.get_location() + offset) cam_pos = self.frame.get_implied_camera_location() # TODO self.perspective_uniforms = { "frame_shape": frame.get_shape(), "anti_alias_width": anti_alias_width, "camera_offset": tuple(offset), "camera_rotation": tuple(np.array(rotation).T.flatten()), "camera_position": tuple(cam_pos), "light_source_position": tuple(light_pos), "focal_distance": frame.get_focal_distance(), } def init_textures(self) -> None: self.n_textures: int = 0 self.path_to_texture: dict[str, tuple[int, moderngl.Texture]] = {} def get_texture_id(self, path: str) -> int: if path not in self.path_to_texture: if self.n_textures == 15: # I have no clue why this is needed self.n_textures += 1 tid = self.n_textures self.n_textures += 1 im = Image.open(path).convert("RGBA") texture = self.ctx.texture( size=im.size, components=len(im.getbands()), data=im.tobytes(), ) texture.use(location=tid) self.path_to_texture[path] = (tid, texture) return self.path_to_texture[path][0] def release_texture(self, path: str): tid_and_texture = self.path_to_texture.pop(path, None) if tid_and_texture: tid_and_texture[1].release() return self