def __init__(self, shfile=None, vshader_source=None, fshader_source=None):
"""
Arguments:
*shfile*
Pathname without vs or fs ending i.e. "shaders/uv_light"
*vshader_source*
String with the code for the vertex shader.
*fshader_source*
String with the code for the fragment shader.
"""
try:
assert Loadable.is_display_thread()
except AssertionError as err:
LOGGER.error('load_opengl must be called on main thread for %s',
self)
raise err
# TODO: the rest of the constructor should be split into load_disk
# and load_opengl so that we can delete that assert.
self.program = opengles.glCreateProgram()
self.shfile = shfile
def make_shader(src, suffix, shader_type):
src = src or self._load_shader(shfile + suffix)
if type(
src
) != bytes: #if ..shader_source passed as a string to __init__
src = src.encode()
characters = ctypes.c_char_p(src)
shader = opengles.glCreateShader(shader_type)
src_len = ctypes.c_int(len(src))
opengles.glShaderSource(shader, 1, characters,
ctypes.byref(src_len))
opengles.glCompileShader(shader)
self.showshaderlog(shader, src)
opengles.glAttachShader(self.program, shader)
return shader, src
self.vshader, self.vshader_source = make_shader(
vshader_source, '.vs', GL_VERTEX_SHADER)
self.fshader, self.fshader_source = make_shader(
fshader_source, '.fs', GL_FRAGMENT_SHADER)
opengles.glLinkProgram(self.program)
self.showprogramlog(self.program)
self.attr_vertex = opengles.glGetAttribLocation(
self.program, b'vertex')
self.attr_normal = opengles.glGetAttribLocation(
self.program, b'normal')
self.unif_modelviewmatrix = opengles.glGetUniformLocation(
self.program, b'modelviewmatrix')
self.unif_unif = opengles.glGetUniformLocation(self.program, b'unif')
self.unif_unib = opengles.glGetUniformLocation(self.program, b'unib')
self.attr_texcoord = opengles.glGetAttribLocation(
self.program, b'texcoord')
opengles.glEnableVertexAttribArray(self.attr_texcoord)
self.unif_tex = []
self.textures = []
for i in range(8):
s = 'tex{}'.format(i).encode()
self.unif_tex.append(opengles.glGetUniformLocation(
self.program, s))
self.textures.append(None)
"""
*NB*
for *uv* shaders tex0=texture tex1=normal map tex2=reflection
for *mat* shaders tex0=normal map tex1=reflection
"""
self.use()
def draw(self, shape=None, M=None, unif=None, shader=None,
textures=None, ntl=None, shny=None, fullset=True):
"""Draw this Buffer, called by the parent Shape.draw()
Keyword arguments:
*shape*
Shape object this Buffer belongs to, has to be passed at draw to avoid
circular reference
*shader*
Shader object
*textures*
array of Texture objects
*ntl*
multiple for tiling normal map which can be less than or greater
than 1.0. 0.0 disables the normal mapping, float
*shiny*
how strong to make the reflection 0.0 to 1.0, float
"""
self.load_opengl()
shader = shader or self.shader or shape.shader or Shader.instance()
shader.use()
opengles.glUniformMatrix4fv(shader.unif_modelviewmatrix, GLsizei(3),
GLboolean(0), M.ctypes.data)
opengles.glUniform3fv(shader.unif_unif, GLsizei(20), unif)
textures = textures or self.textures
if ntl is not None:
self.unib[0] = ntl
if shny is not None:
self.unib[1] = shny
self._select()
opengles.glVertexAttribPointer(shader.attr_vertex, GLint(3), GL_FLOAT, GLboolean(0), self.N_BYTES, 0)
opengles.glEnableVertexAttribArray(shader.attr_vertex)
if self.N_BYTES > 12:
opengles.glVertexAttribPointer(shader.attr_normal, GLint(3), GL_FLOAT, GLboolean(0), self.N_BYTES, 12)
opengles.glEnableVertexAttribArray(shader.attr_normal)
if self.N_BYTES > 24:
opengles.glVertexAttribPointer(shader.attr_texcoord, GLint(2), GL_FLOAT, GLboolean(0), self.N_BYTES, 24)
opengles.glEnableVertexAttribArray(shader.attr_texcoord)
opengles.glDisable(GL_BLEND)
self.unib[2] = 0.6
for t, texture in enumerate(textures):
if (self.disp.last_textures[t] != texture or self.disp.last_shader != shader or
self.disp.offscreen_tex): # very slight speed increase for sprites
opengles.glActiveTexture(GL_TEXTURE0 + t)
assert texture.tex(), 'There was an empty texture in your Buffer.'
opengles.glBindTexture(GL_TEXTURE_2D, texture.tex())
opengles.glUniform1i(shader.unif_tex[t], GLint(t))
self.disp.last_textures[t] = texture
if texture.blend:
# i.e. if any of the textures set to blend then all will for this shader.
self.unib[2] = 0.05
if self.unib[2] != 0.6 or shape.unif[13] < 1.0 or shape.unif[14] < 1.0:
#use unib[2] as flag to indicate if any Textures to be blended
#needs to be done outside for..textures so materials can be transparent
opengles.glEnable(GL_BLEND)
self.unib[2] = 0.05
self.disp.last_shader = shader
opengles.glUniform3fv(shader.unif_unib, GLsizei(5), self.unib)
opengles.glEnable(GL_DEPTH_TEST) # TODO find somewhere more efficient to do this
opengles.glDrawElements(self.draw_method, GLsizei(self.ntris * 3), GL_UNSIGNED_SHORT, 0)
def __init__(self, shfile=None, vshader_source=None, fshader_source=None):
"""
Arguments:
*shfile*
Pathname without vs or fs ending i.e. "shaders/uv_light"
*vshader_source*
String with the code for the vertex shader.
*fshader_source*
String with the code for the fragment shader.
"""
try:
assert Loadable.is_display_thread()
except AssertionError as err:
LOGGER.error('load_opengl must be called on main thread for %s', self)
raise err
# TODO: the rest of the constructor should be split into load_disk
# and load_opengl so that we can delete that assert.
self.program = opengles.glCreateProgram()
self.shfile = shfile
def make_shader(src, suffix, shader_type):
src = src or self._load_shader(shfile + suffix)
if type(src) != bytes: #if ..shader_source passed as a string to __init__
src = src.encode()
characters = ctypes.c_char_p(src)
shader = opengles.glCreateShader(shader_type)
src_len = ctypes.c_int(len(src))
opengles.glShaderSource(shader, 1, characters, ctypes.byref(src_len))
opengles.glCompileShader(shader)
self.showshaderlog(shader, src)
opengles.glAttachShader(self.program, shader)
return shader, src
self.vshader, self.vshader_source = make_shader(
vshader_source, '.vs', GL_VERTEX_SHADER)
self.fshader, self.fshader_source = make_shader(
fshader_source, '.fs', GL_FRAGMENT_SHADER)
opengles.glLinkProgram(self.program)
self.showprogramlog(self.program)
self.attr_vertex = opengles.glGetAttribLocation(self.program, b'vertex')
self.attr_normal = opengles.glGetAttribLocation(self.program, b'normal')
self.unif_modelviewmatrix = opengles.glGetUniformLocation(
self.program, b'modelviewmatrix')
self.unif_unif = opengles.glGetUniformLocation(self.program, b'unif')
self.unif_unib = opengles.glGetUniformLocation(self.program, b'unib')
self.attr_texcoord = opengles.glGetAttribLocation(self.program, b'texcoord')
opengles.glEnableVertexAttribArray(self.attr_texcoord)
self.unif_tex = []
self.textures = []
for i in range(8):
s = 'tex{}'.format(i).encode()
self.unif_tex.append(opengles.glGetUniformLocation(self.program, s))
self.textures.append(None)
"""
*NB*
for *uv* shaders tex0=texture tex1=normal map tex2=reflection
for *mat* shaders tex0=normal map tex1=reflection
"""
self.use()
def draw(self) -> None:
import numpy as np
from scipy.ndimage.filters import gaussian_filter
from pi3d.Camera import Camera
from pi3d.constants import (
opengles,
GL_SRC_ALPHA,
GL_ONE_MINUS_SRC_ALPHA,
GLsizei,
GLboolean,
GLint,
GL_FLOAT,
GL_ARRAY_BUFFER,
GL_UNSIGNED_SHORT,
GL_TEXTURE_2D,
GL_UNSIGNED_BYTE,
)
time_logging = False
if self.should_prepare:
self._prepare()
if self.lights.alarm_program.factor != -1:
self.alarm_factor = max(0.001, self.lights.alarm_program.factor)
else:
self.alarm_factor = 0
then = time.time()
self.display.loop_running()
now = self.display.time
self.time_delta = now - self.last_loop
self.last_loop = now
self.time_elapsed += self.time_delta
if time_logging:
print(f"{time.time() - then} main loop")
then = time.time()
# use a sliding window to smooth the spectrum with a gauss function
# truncating does not save significant time (~3% for this step)
# new_frame = np.array(self.cava.current_frame, dtype="float32")
new_frame = gaussian_filter(self.cava.current_frame, sigma=1.5, mode="nearest")
new_frame = new_frame[self.SPECTRUM_CUT : -self.SPECTRUM_CUT]
new_frame = -0.5 * new_frame ** 3 + 1.5 * new_frame
new_frame *= 255
current_frame = new_frame
if time_logging:
print(f"{time.time() - then} spectrum smoothing")
then = time.time()
# Value used for circle shake and background color cycle
# select the first few values and compute their average
bass_elements = math.ceil(self.BASS_MAX * self.cava.bars)
self.bass_value = sum(current_frame[0:bass_elements]) / bass_elements / 255
self.bass_value = max(self.bass_value, self.alarm_factor)
self.total_bass = self.total_bass + self.bass_value
# the fraction of time that there was bass
self.bass_fraction = self.total_bass / self.time_elapsed / self.lights.UPS
self.uniform_values = {
48: self.width / self.scale,
49: self.height / self.scale,
50: self.scale,
51: self.FFT_HIST,
52: self.NUM_PARTICLES,
53: self.PARTICLE_SPAWN_Z,
54: self.time_elapsed,
55: self.time_delta,
56: self.alarm_factor,
57: self.bass_value,
58: self.total_bass,
59: self.bass_fraction,
}
# start rendering to the smaller OffscreenTexture
# we decrease the size of the texture so it only allocates that much memory
# otherwise it would use as much as the displays size, negating its positive effect
self.post.ix = int(self.post.ix / self.scale)
self.post.iy = int(self.post.iy / self.scale)
opengles.glViewport(
GLint(0),
GLint(0),
GLsizei(int(self.width / self.scale)),
GLsizei(int(self.height / self.scale)),
)
self.post._start()
self.post.ix = self.width
self.post.iy = self.height
self._set_unif(self.background, [48, 49, 54, 56, 58])
self.background.draw()
if time_logging:
print(f"{time.time() - then} background draw")
then = time.time()
# enable additive blending so the draw order of overlapping particles does not matter
opengles.glBlendFunc(1, 1)
self._set_unif(self.particle_sprite, [53, 54, 59])
# copied code from pi3d.Shape.draw()
# we don't need modelmatrices, normals ord textures and always blend
self.particle_sprite.load_opengl()
camera = Camera.instance()
if not camera.mtrx_made:
camera.make_mtrx()
self.particle_sprite.MRaw = self.particle_sprite.tr1
self.particle_sprite.M[0, :, :] = self.particle_sprite.MRaw[:, :]
self.particle_sprite.M[1, :, :] = np.dot(
self.particle_sprite.MRaw, camera.mtrx
)[:, :]
# Buffer.draw()
buf = self.particle_sprite.buf[0]
buf.load_opengl()
shader = buf.shader
shader.use()
opengles.glUniformMatrix4fv(
shader.unif_modelviewmatrix,
GLsizei(2),
GLboolean(0),
self.particle_sprite.M.ctypes.data,
)
opengles.glUniform3fv(shader.unif_unif, GLsizei(20), self.particle_sprite.unif)
buf._select()
opengles.glVertexAttribPointer(
shader.attr_vertex, GLint(3), GL_FLOAT, GLboolean(0), buf.N_BYTES, 0
)
opengles.glEnableVertexAttribArray(shader.attr_vertex)
opengles.glVertexAttribPointer(
shader.attr_texcoord, GLint(2), GL_FLOAT, GLboolean(0), buf.N_BYTES, 24
)
opengles.glEnableVertexAttribArray(shader.attr_texcoord)
buf.disp.last_shader = shader
opengles.glUniform3fv(shader.unif_unib, GLsizei(5), buf.unib)
opengles.glBindBuffer(GL_ARRAY_BUFFER, self.instance_vbo)
opengles.glDrawElementsInstanced(
buf.draw_method,
GLsizei(buf.ntris * 3),
GL_UNSIGNED_SHORT,
0,
self.NUM_PARTICLES,
)
# restore normal blending
opengles.glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
if time_logging:
print(f"{time.time() - then} particle draw")
then = time.time()
# roll the history one further, insert the current one.
# we use a texture with four channels eventhough we only need one, refer to this post:
# https://community.khronos.org/t/updating-textures-per-frame/75020/3
# basically the gpu converts it anyway, so other formats would be slower
history = np.zeros(
(self.FFT_HIST, self.cava.bars - 2 * self.SPECTRUM_CUT, 4), dtype="uint8"
)
self.fft = np.roll(self.fft, 1, 0)
self.fft[0] = current_frame
history[:, :, 0] = self.fft
if time_logging:
print(f"{time.time() - then} spectrum roll")
then = time.time()
# change the spectrum part of the texture (the lower 256xFFT_HIST pixels)
opengles.glBindTexture(GL_TEXTURE_2D, self.dynamic_texture._tex)
iformat = self.dynamic_texture._get_format_from_array(
history, self.dynamic_texture.i_format
)
opengles.glTexSubImage2D(
GL_TEXTURE_2D,
0,
0,
self.dynamic_texture.ix,
history.shape[1],
history.shape[0],
iformat,
GL_UNSIGNED_BYTE,
history.ctypes.data_as(ctypes.POINTER(ctypes.c_ubyte)),
)
if time_logging:
print(f"{time.time() - then} glTexImage2D")
then = time.time()
self._set_unif(self.spectrum, [48, 49, 51, 52, 53, 54, 55, 57, 58])
self.spectrum.draw()
if time_logging:
print(f"{time.time() - then} spectrum draw")
then = time.time()
self._set_unif(self.logo, [48, 49, 51, 54, 57, 58])
self.logo.draw()
if time_logging:
print(f"{time.time() - then} logo draw")
then = time.time()
self._set_unif(self.after, [48, 49, 54, 57])
self.after.draw()
if time_logging:
print(f"{time.time() - then} after draw")
then = time.time()
self.post._end()
opengles.glViewport(
GLint(0), GLint(0), GLsizei(self.width), GLsizei(self.height)
)
self._set_unif(self.post_sprite, [50])
self.post_sprite.draw()
if time_logging:
print(f"{time.time() - then} post draw")
print(f"scale: {self.scale}")
print("=====")
def _prepare(self) -> None:
self.should_prepare = False
import numpy as np
# pi3d has to be imported in the same thread that it will draw in
# Thus, import it here instead of at the top of the file
import pi3d
from pi3d.util.OffScreenTexture import OffScreenTexture
from pi3d.constants import (
opengles,
GL_CLAMP_TO_EDGE,
GL_ALWAYS,
)
# used for reimplementing the draw call with instancing
from pi3d.constants import (
GLsizei,
GLint,
GLboolean,
GL_FLOAT,
GLuint,
GL_ARRAY_BUFFER,
GL_STATIC_DRAW,
GLfloat,
)
from PIL import Image
# Setup display and initialise pi3d
self.display = pi3d.Display.create(
w=self.width, h=self.height, window_title="Raveberry"
)
# error 0x500 after Display create
# error = opengles.glGetError()
# Set a pink background color so mistakes are clearly visible
# self.display.set_background(1, 0, 1, 1)
self.display.set_background(0, 0, 0, 1)
# print OpenGL Version, useful for debugging
# import ctypes
# def print_char_p(addr):
# g = (ctypes.c_char*32).from_address(addr)
# i = 0
# while True:
# c = g[i]
# if c == b'\x00':
# break
# sys.stdout.write(c.decode())
# i += 1
# sys.stdout.write('\n')
# print_char_p(opengles.glGetString(GL_VERSION))
# Visualization is split into five parts:
# The background, the particles, the spectrum, the logo and after effects.
# * The background is a vertical gradient that cycles through HSV color space,
# speeding up with strong bass.
# * Particles are multiple sprites that are created at a specified x,y-coordinate
# and fly towards the camera.
# Due to the projection into screenspace they seem to move away from the center.
# * The spectrum is a white circle that represents the fft-transformation of the
# currently played audio. It is smoothed to avoid strong spikes.
# * The logo is a black circle on top of the spectrum containing the logo.
# * After effects add a vignette.
# Each of these parts is represented with pi3d Shapes.
# They have their own shader and are ordered on the z-axis to ensure correct overlapping.
background_shader = pi3d.Shader(
os.path.join(settings.BASE_DIR, "core/lights/circle/background")
)
self.background = pi3d.Sprite(w=2, h=2)
self.background.set_shader(background_shader)
self.background.positionZ(0)
self.particle_shader = pi3d.Shader(
os.path.join(settings.BASE_DIR, "core/lights/circle/particle")
)
# create one sprite for all particles
self.particle_sprite = pi3d.Sprite(w=self.PARTICLE_SIZE, h=self.PARTICLE_SIZE)
self.particle_sprite.set_shader(self.particle_shader)
self.particle_sprite.positionZ(0)
# this array containes the position and speed for all particles
particles = self._initial_particles()
# This part was modified from https://learnopengl.com/Advanced-OpenGL/Instancing
self.instance_vbo = GLuint()
opengles.glGenBuffers(GLsizei(1), ctypes.byref(self.instance_vbo))
opengles.glBindBuffer(GL_ARRAY_BUFFER, self.instance_vbo)
particles_raw = particles.ctypes.data_as(ctypes.POINTER(GLfloat))
opengles.glBufferData(
GL_ARRAY_BUFFER, particles.nbytes, particles_raw, GL_STATIC_DRAW
)
opengles.glBindBuffer(GL_ARRAY_BUFFER, GLuint(0))
attr_particle = opengles.glGetAttribLocation(
self.particle_shader.program, b"particle"
)
opengles.glEnableVertexAttribArray(attr_particle)
opengles.glBindBuffer(GL_ARRAY_BUFFER, self.instance_vbo)
opengles.glVertexAttribPointer(
attr_particle, GLint(4), GL_FLOAT, GLboolean(0), 0, 0
)
opengles.glBindBuffer(GL_ARRAY_BUFFER, GLuint(0))
opengles.glVertexAttribDivisor(attr_particle, GLuint(1))
spectrum_shader = pi3d.Shader(
os.path.join(settings.BASE_DIR, "core/lights/circle/spectrum")
)
# use the ratio to compute small sizes for the sprites
ratio = self.width / self.height
self.spectrum = pi3d.Sprite(w=2 / ratio, h=2)
self.spectrum.set_shader(spectrum_shader)
self.spectrum.positionZ(0)
# initialize the spectogram history with zeroes
self.fft = np.zeros(
(self.FFT_HIST, self.cava.bars - 2 * self.SPECTRUM_CUT), dtype=np.uint8
)
logo_shader = pi3d.Shader(
os.path.join(settings.BASE_DIR, "core/lights/circle/logo")
)
self.logo = pi3d.Sprite(w=1.375 / ratio, h=1.375)
self.logo.set_shader(logo_shader)
self.logo.positionZ(0)
logo_image = Image.open(
os.path.join(settings.STATIC_ROOT, "graphics/raveberry_square.png")
)
self.logo_array = np.frombuffer(logo_image.tobytes(), dtype=np.uint8)
self.logo_array = self.logo_array.reshape(
(logo_image.size[1], logo_image.size[0], 3)
)
# add space for the spectrum
self.logo_array = np.concatenate(
(
self.logo_array,
np.zeros((self.FFT_HIST, logo_image.size[0], 3), dtype=np.uint8),
),
axis=0,
)
# add alpha channel
self.logo_array = np.concatenate(
(
self.logo_array,
np.ones(
(self.logo_array.shape[0], self.logo_array.shape[1], 1),
dtype=np.uint8,
),
),
axis=2,
)
# In order to save memory, the logo and the spectrum share one texture.
# The upper 256x256 pixels are the raveberry logo.
# Below are 256xFFT_HIST pixels for the spectrum.
# The lower part is periodically updated every frame while the logo stays static.
self.dynamic_texture = pi3d.Texture(self.logo_array)
# Prevent interpolation from opposite edge
self.dynamic_texture.m_repeat = GL_CLAMP_TO_EDGE
self.spectrum.set_textures([self.dynamic_texture])
self.logo.set_textures([self.dynamic_texture])
after_shader = pi3d.Shader(
os.path.join(settings.BASE_DIR, "core/lights/circle/after")
)
self.after = pi3d.Sprite(w=2, h=2)
self.after.set_shader(after_shader)
self.after.positionZ(0)
# create an OffscreenTexture to allow scaling.
# By first rendering into a smaller Texture a lot of computation is saved.
# This OffscreenTexture is then drawn at the end of the draw loop.
self.post = OffScreenTexture("scale")
self.post_sprite = pi3d.Sprite(w=2, h=2)
post_shader = pi3d.Shader(
os.path.join(settings.BASE_DIR, "core/lights/circle/scale")
)
self.post_sprite.set_shader(post_shader)
self.post_sprite.set_textures([self.post])
self.total_bass = 0
self.last_loop = time.time()
self.time_elapsed = 0
opengles.glDepthFunc(GL_ALWAYS)
def draw(self, shape=None, M=None, unif=None, shader=None,
textures=None, ntl=None, shny=None, fullset=True):
"""Draw this Buffer, called by the parent Shape.draw()
Keyword arguments:
*shape*
Shape object this Buffer belongs to, has to be passed at draw to avoid
circular reference
*shader*
Shader object
*textures*
array of Texture objects
*ntl*
multiple for tiling normal map which can be less than or greater
than 1.0. 0.0 disables the normal mapping, float
*shiny*
how strong to make the reflection 0.0 to 1.0, float
"""
self.load_opengl()
shader = shader or self.shader or shape.shader or Shader.instance()
shader.use()
opengles.glUniformMatrix4fv(shader.unif_modelviewmatrix, 3,
ctypes.c_ubyte(0), M.ctypes.data)
opengles.glUniform3fv(shader.unif_unif, 20, unif)
textures = textures or self.textures
if ntl is not None:
self.unib[0] = ntl
if shny is not None:
self.unib[1] = shny
self._select()
opengles.glVertexAttribPointer(shader.attr_vertex, 3, GL_FLOAT, 0, self.N_BYTES, 0)
opengles.glEnableVertexAttribArray(shader.attr_vertex)
if self.N_BYTES > 12:
opengles.glVertexAttribPointer(shader.attr_normal, 3, GL_FLOAT, 0, self.N_BYTES, 12)
opengles.glEnableVertexAttribArray(shader.attr_normal)
if self.N_BYTES > 24:
opengles.glVertexAttribPointer(shader.attr_texcoord, 2, GL_FLOAT, 0, self.N_BYTES, 24)
opengles.glEnableVertexAttribArray(shader.attr_texcoord)
opengles.glDisable(GL_BLEND)
self.unib[2] = 0.6
for t, texture in enumerate(textures):
if (self.disp.last_textures[t] != texture or self.disp.last_shader != shader or
self.disp.offscreen_tex): # very slight speed increase for sprites
opengles.glActiveTexture(GL_TEXTURE0 + t)
assert texture.tex(), 'There was an empty texture in your Buffer.'
opengles.glBindTexture(GL_TEXTURE_2D, texture.tex())
opengles.glUniform1i(shader.unif_tex[t], t)
self.disp.last_textures[t] = texture
if texture.blend:
# i.e. if any of the textures set to blend then all will for this shader.
self.unib[2] = 0.05
if self.unib[2] != 0.6 or shape.unif[13] < 1.0 or shape.unif[14] < 1.0:
#use unib[2] as flag to indicate if any Textures to be blended
#needs to be done outside for..textures so materials can be transparent
opengles.glEnable(GL_BLEND)
self.unib[2] = 0.05
self.disp.last_shader = shader
opengles.glUniform3fv(shader.unif_unib, 5, self.unib)
opengles.glEnable(GL_DEPTH_TEST) # TODO find somewhere more efficient to do this
opengles.glDrawElements(self.draw_method, self.ntris * 3, GL_UNSIGNED_SHORT, 0)
