class Paintbrush(Widget):
def __init__(self, **kwargs):
super(Paintbrush, self).__init__(**kwargs)
self.fbo = Fbo(size=(10, 10))
self.mesh = Mesh()
self.points = []
self.vertices = []
self.indices = []
self.line_widths = []
self.cap_vertices_index = 0
self.cap_indices_index = 0
self.mask_lines = []
self.mask_alphas = []
self.canvas = RenderContext()
self.canvas.shader.fs = mask_shader
self.buffer_container = None
self.rgb = (0, 1, 1)
# We'll update our glsl variables in a clock
Clock.schedule_interval(self.update_glsl, 0)
# Maintain a window of history for autocorrelations
self.ac_window = []
self.ac_position = 0
self.periodicity_factor = 1.0
def update_glsl(self, *largs):
# This is needed for the default vertex shader.
self.canvas['projection_mat'] = Window.render_context['projection_mat']
self.canvas['modelview_mat'] = Window.render_context['modelview_mat']
def on_size(self, instance, value):
self.canvas.clear()
with self.canvas:
self.fbo = Fbo(size=value)
self.mask_fbo = Fbo(size=(value[0] // 5, value[1] // 5), clear_color=(1, 1, 1, 1))
Color(*self.rgb, 0.9)
BindTexture(texture=self.mask_fbo.texture, index=1)
self.buffer_container = Rectangle(pos=self.pos, size=value, texture=self.fbo.texture)
#Rectangle(pos=self.pos, size=value, texture=self.mask_fbo.texture)
self.canvas['texture1'] = 1
with self.fbo:
Color(1, 1, 1)
self.mesh = Mesh(mode='triangle_strip')
def on_pos(self, instance, value):
if not self.buffer_container: return
self.buffer_container.pos = value
def build_line_segment(self, start, end, future, start_width=8.0, end_width=8.0):
"""Builds a line segment knowing the start and end, as well as one point in the future."""
start = np.array([start[0], start[1]])
end = np.array([end[0], end[1]])
future = np.array([future[0], future[1]])
length = np.linalg.norm(end - start)
num_interpolants = max(int(length / DISTANCE_PER_POINT), 3)
normal = (end - start) / length * start_width / 2.0
normal = np.array([-normal[1], normal[0]])
end_normal = (future - end) / max(np.linalg.norm(future - end), 0.1) * end_width / 2.0
end_normal = np.array([-end_normal[1], end_normal[0]])
delta_sign = None
# if (self.last_normal is not None and
# np.linalg.norm(normal - self.last_normal) > np.linalg.norm(normal + self.last_normal)):
# self.last_normal *= -1
# Add points deviating in alternating directions around the actual path
for i in range(num_interpolants):
path_point = start + (i / num_interpolants) * (end - start)
delta = normal + (i / num_interpolants) * (end_normal - normal)
if delta_sign is None:
delta_sign = 1
if len(self.points) > 3:
second_last_vertex = np.array(self.vertices[-8:-6])
option_1 = path_point + delta
option_2 = path_point - delta
if (np.linalg.norm(option_2 - second_last_vertex) <
np.linalg.norm(option_1 - second_last_vertex)):
delta_sign *= -1
self.vertices.extend([*(path_point + delta * delta_sign), 0, 0])
self.indices.append(len(self.indices))
delta_sign *= -1
def add_cap(self, width):
"""Adds a round line cap to the end of the vertex/index list."""
self.cap_vertices_index = len(self.vertices)
self.cap_indices_index = len(self.indices)
if len(self.points) < 3:
return
# Extend the current line segment using a circular interpolation of line widths
start = np.array([self.points[-1][0], self.points[-1][1]])
prev = np.array([self.points[-2][0], self.points[-2][1]])
end = start + (start - prev) / max(np.linalg.norm(start - prev), 0.001) * width / 2.0
length = np.linalg.norm(end - start)
num_interpolants = max(int(length / DISTANCE_PER_POINT) * 2, 3)
normal = (end - start) / length * width / 2.0
normal = np.array([-normal[1], normal[0]])
end_normal = np.zeros(2)
delta_sign = None
# Add points deviating in alternating directions around the actual path
for i in range(num_interpolants):
path_point = start + (i / (num_interpolants - 1)) * (end - start)
circ_progress = 1 - np.sqrt(1 - (i / (num_interpolants - 1)) ** 2)
delta = normal + circ_progress * (end_normal - normal)
if delta_sign is None:
delta_sign = 1
if len(self.points) > 3:
second_last_vertex = np.array(self.vertices[-8:-6])
option_1 = path_point + delta
option_2 = path_point - delta
if (np.linalg.norm(option_2 - second_last_vertex) <
np.linalg.norm(option_1 - second_last_vertex)):
delta_sign *= -1
self.vertices.extend([*(path_point + delta * delta_sign), 0, 0])
self.indices.append(len(self.indices))
delta_sign *= -1
def remove_cap(self):
"""Removes a cap on the line."""
if self.cap_vertices_index > 0 and self.cap_vertices_index <= len(self.vertices):
del self.vertices[self.cap_vertices_index:]
del self.indices[self.cap_indices_index:]
self.cap_vertices_index = 0
self.cap_indices_index = 0
def current_line_width(self, depth, window=5):
"""Computes the current line width of the previous `window` points."""
max_width = 120.0
min_width = 5.0
min_dist = 40.0
max_dist = 140.0
last_point = self.points[-1]
old_point = self.points[max(0, len(self.points) - window)]
if PAINTBRUSH_MODE == 0:
dist = np.linalg.norm(np.array([last_point[0], last_point[1]]) -
np.array([old_point[0], old_point[1]]))
else:
dist = 120.0
width = (max_dist - dist) * (max_width * 0.8 - min_width) / (max_dist - min_dist)
if PAINTBRUSH_MODE != 0:
depth_factor = 1 / (1 + np.exp(-(depth - 0.5) * 4))
width *= depth_factor
if PAINTBRUSH_MODE == 2:
width *= self.periodicity_factor
return np.clip(width, min_width, max_width)
def update_periodicity(self, point):
"""Computes a new autocorrelation magnitude by adding the given point."""
self.ac_window.append(point)
if len(self.ac_window) > AUTOCORRELATION_WINDOW:
del self.ac_window[0]
self.ac_position += 1
if self.ac_position % 8 == 0 and len(self.ac_window) == AUTOCORRELATION_WINDOW:
ac_window = np.array(self.ac_window)
x_fft = np.abs(np.fft.rfft(ac_window[:,0] * np.hanning(AUTOCORRELATION_WINDOW)))
y_fft = np.abs(np.fft.rfft(ac_window[:,1] * np.hanning(AUTOCORRELATION_WINDOW)))
x_fft = x_fft[4:20] / np.mean(x_fft[4:20])
y_fft = y_fft[4:20] / np.mean(y_fft[4:20])
# if self.ac_position > 200:
# plt.figure()
# plt.subplot(121)
# plt.plot(ac_window[:,0], ac_window[:,1])
# plt.subplot(122)
# plt.plot(x_fft, label='x')
# plt.plot(y_fft, label='y')
# plt.show()
self.periodicity_factor = ((max(1.0, np.max(x_fft) / 4.0) *
max(1.0, np.max(y_fft) / 4.0)) - 1) ** 2 + 1
def add_point(self, point, depth=None, width=None, alpha=None):
"""
point: a point in window space to add to the paintbrush trajectory (x, y).
depth: a 0-1 value indicating the depth into the screen of the current point.
alpha: a manual 0-1 alpha level for this point.
Returns the current line width.
"""
point = (point[0] - self.pos[0], point[1] - self.pos[1])
self.points.append(point)
# Build a segment of line
line_width = 0
if len(self.points) > 2:
self.update_periodicity(point)
line_width = self.current_line_width(depth) if depth is not None else width
old_line_width = (sum(self.line_widths) / len(self.line_widths)
if self.line_widths else line_width)
self.line_widths.append(line_width)
if len(self.line_widths) > LINE_WIDTH_WINDOW:
self.line_widths.pop(0)
if width is None:
line_width = sum(self.line_widths) / len(self.line_widths)
# Clamp the amount by which the line width can change - results in
# smoother lines
# line_width = old_line_width + np.clip(line_width - old_line_width, -2.0, 2.0)
self.remove_cap()
self.build_line_segment(*self.points[-3:], old_line_width, line_width)
self.add_cap(line_width)
# Update mask
if len(self.points) % MASK_INTERVAL == 0 and len(self.points) > MASK_INTERVAL:
self.mask_lines.append(Line(points=(self.points[-MASK_INTERVAL - 1][0] / 5,
self.points[-MASK_INTERVAL - 1][1] /5 ,
self.points[-1][0] / 5,
self.points[-1][1] / 5),
width=(line_width + 8.0) / 10))
if alpha is not None:
self.mask_alphas.append(alpha)
with self.mask_fbo:
self.mask_fbo.clear()
self.mask_fbo.clear_buffer()
if len(self.mask_alphas) == len(self.mask_lines):
white_values = self.mask_alphas
else:
white_values = 1 / (1 + np.exp(-((np.arange(len(self.mask_lines)) -
len(self.mask_lines)) / FADE_FACTOR + 3)))
white_values = white_values * (1 - BASE_FADE) + BASE_FADE
for i, (white, line) in enumerate(zip(white_values, self.mask_lines)):
Color(white, white, white, 1)
self.mask_fbo.add(line)
# if len(self.points) % 100 == 20:
# plt.figure()
# plt.plot(self.vertices[::4], self.vertices[1::4])
# plt.plot(self.vertices[::4], self.vertices[1::4], 'b.')
# plt.plot([x[0] for x in self.points], [x[1] for x in self.points], 'ro')
# plt.plot([x[0] for x in self.points], [x[1] for x in self.points], 'r-')
# plt.show()
# self.vertices.extend([point[0], point[1], 0, 0])
# if len(self.points) > 1:
# self.indices.extend([len(self.points) - 2, len(self.points) - 1])
self.mesh.vertices = self.vertices
self.mesh.indices = self.indices
return line_width
def clear(self):
self.points = []
self.vertices = []
self.indices = []
self.mesh.vertices = []
self.mesh.indices = []
self.periodicity_factor = 1.0
self.ac_window = []
self.ac_position = 0
with self.fbo:
self.fbo.clear_buffer()
self.mask_lines = []
self.mask_colors = []
with self.mask_fbo:
self.mask_fbo.clear()
self.mask_fbo.clear_buffer()
self.on_size(self, self.size)
class XWindow(Widget):
__events__ = [
'on_window_map',
'on_window_resize',
'on_window_unmap',
'on_window_destroy',
]
active = BooleanProperty(False)
invalidate_pixmap = BooleanProperty(False)
pixmap = ObjectProperty(None, allownone=True)
refresh_rate = NumericProperty()
texture = ObjectProperty(None, allownone=True)
def __init__(self, manager, window=None, **kwargs):
super().__init__(**kwargs)
self.manager = manager
if window:
self._window = window
else:
self._window = manager.screen.root.create_window(
x=0,
y=0,
width=self.width,
height=self.height,
depth=self.manager.screen.root_depth,
border_width=0,
window_class=Xlib.X.InputOutput,
visual=Xlib.X.CopyFromParent,
)
refresh_hz = int(os.environ.get('KIVYWM_REFRESH_HZ', 60))
self.refresh_rate = 1 / refresh_hz if refresh_hz > 0 else 0
self.canvas = RenderContext(use_parent_projection=True,
use_parent_modelview=True,
use_parent_frag_modelview=True)
with self.canvas:
self.rect = Rectangle(size=self.size)
def __repr__(self):
if hasattr(self, '_window') and self._window is not None:
return f'<{self.__class__.__name__} id: {hex(self.id)}>'
else:
return f'<{self.__class__.__name__} (No Window Bound)>'
def focus(self):
self._win.set_input_focus(revert_to=Xlib.X.RevertToParent,
time=Xlib.X.CurrentTime)
def redraw(self, *args):
self.canvas.ask_update()
return self.active
def on_invalidate_pixmap(self, *args):
if not self.invalidate_pixmap or not self._window:
return
try:
self.release_texture()
self.release_pixmap()
self.create_pixmap()
self.create_texture()
except (Xlib.error.BadDrawable, Xlib.error.BadWindow,
KeyboardInterrupt):
self.active = False
self.invalidate_pixmap = False
def on_active(self, *args):
if self.active:
Clock.schedule_interval(self.redraw, self.refresh_rate)
else:
self.release_texture()
self.release_pixmap()
def map(self, *args):
try:
self._window.map()
except AttributeError:
pass
self.invalidate_pixmap = True
self.start()
def unmap(self, *args):
try:
self._window.unmap()
except AttributeError:
pass
self.stop()
def start(self, *args):
self.active = True
def stop(self, *args):
self.active = False
def destroy(self, *args, **kwargs):
window = self._window
self._window = None
self.active = False
self.unmap()
self.release_texture()
self.release_pixmap()
self.canvas.clear()
window.destroy()
@property
def id(self):
try:
return self._window.id
except AttributeError:
return None
def on_size(self, *args):
Logger.trace(f'WindowMgr: {self}: on_size: {self.size}')
try:
self._window.configure(
width=round(self.width),
height=round(self.height),
)
except AttributeError:
return
self.invalidate_pixmap = True
def on_pos(self, *args):
try:
self._window.configure(
x=round(self.x),
y=round(self.y),
)
except AttributeError:
return
def on_window_map(self):
Logger.trace(f'WindowMgr: {self}: on_window_map')
self.invalidate_pixmap = True
def on_window_resize(self):
Logger.trace(f'WindowMgr: {self}: on_window_resize')
self.invalidate_pixmap = True
def on_window_unmap(self):
Logger.trace(f'WindowMgr: {self}: on_window_unmap')
self.stop()
def on_window_destroy(self):
Logger.trace(f'WindowMgr: {self}: on_window_destroy')
def create_pixmap(self):
ec = Xlib.error.CatchError(Xlib.error.BadMatch)
try:
self.pixmap = self._window.composite_name_window_pixmap(onerror=ec)
except AttributeError:
pass
if ec.get_error():
self.pixmap = None
def release_pixmap(self):
if self.pixmap:
self.pixmap.free()
self.pixmap = None
def create_texture(self):
if not self._window:
return
try:
geom = self._window.get_geometry()
self.texture = Texture.create_from_pixmap(
self.pixmap.id, (geom.width, geom.height))
except AttributeError:
return
else:
self.rect.texture = self.texture
self.rect.size = self.texture.size
def release_texture(self):
self.texture = None
class Section(Widget):
source = ObjectProperty(None)
block = ObjectProperty(None)
def __init__(self, client, **kwargs):
self.canvas = RenderContext(use_parent_projection=True)
self.client = client
super(Section, self).__init__(**kwargs)
self.canvas.shader.fs = open(resource_find('shaders/section_fragment.glsl')).read()
self.canvas.shader.vs = open(resource_find('shaders/section_vertex.glsl')).read()
self.recalc()
def recalc(self, *args, **kwargs):
block = copy.deepcopy(self.block)
block['x'] -= block['blend_left'] / 2
block['y'] -= block['blend_top'] / 2
block['width'] += block['blend_left'] / 2 + block['blend_right'] / 2
block['height'] += block['blend_top'] / 2 + block['blend_bottom'] / 2
print(block['x'], block['y'], block['width'], block['height'])
if block['x'] < 0: block['x'] = 0
if block['y'] < 0: block['x'] = 0
if block['width'] > 1: block['width'] = 1
if block['height'] > 1: block['height'] = 1
w, h = self.source.texture.width, self.source.texture.height
sw, sh = self.source.width / self.source.texture.width, self.source.height / self.source.texture.height
# self.texture = self.source.texture.get_region(
# sw * min(block['x'] * w, w) - (block['blend_left'] * w / 2),
# sh * min(block['y'] * h, h) - (block['blend_top'] * h / 2),
# sw * min(block['width'] * w, w) + (block['blend_right'] * w / 2),
# sh * min(block['height'] * h, h) + (block['blend_bottom'] * h / 2)
# )
self.texture = self.source.texture.get_region(
sw * min(block['x'] * w, w),
sh * min(block['y'] * h, h),
sw * min(block['width'] * w, w),
sh * min(block['height'] * h, h)
)
before = [
[-1, -1],
[1, -1],
[1, 1],
[-1, 1]
]
# Adjust size of section if edge blending is used
points = block['points']
points[3] = [points[3][0] - block['blend_left'], points[3][1] + block['blend_bottom']]
points[2] = [points[2][0] + block['blend_right'], points[2][1] + block['blend_bottom']]
points[0] = [points[0][0] - block['blend_left'], points[0][1] - block['blend_top']]
points[1] = [points[1][0] + block['blend_right'], points[1][1] - block['blend_top']]
after = numpy.array(points)
A = []
for a, b in zip(after, before):
A.append([
b[0], 0, -a[0] * b[0],
b[1], 0, -a[0] * b[1], 1, 0]);
A.append([
0, b[0], -a[1] * b[0],
0, b[1], -a[1] * b[1], 0, 1]);
A = numpy.array(A)
B = numpy.array([[c for p in block['points'] for c in p]])
B = B.transpose()
m = numpy.dot(numpy.linalg.inv(A), B)
m = m.transpose().reshape(-1,).tolist()
matrix = Matrix()
matrix.set([
m[0], m[1], 0, m[2],
m[3], m[4], 0, m[5],
0, 0, 1, 0,
m[6], m[7], 0, 1
])
self.canvas['uTransformMatrix'] = matrix
self.canvas['brightness'] = float(block.get('brightness', 1))
self.canvas['alpha_mask'] = int(block.get('alpha_mask', False)) # Because Kivy can't pass booleans to shaders, apparently.
self.canvas['adjust'] = float(self.client.minion['settings'].get('displayminion_color_adjust_range', 0))
self.canvas['tex_x'] = block['x']
self.canvas['tex_y'] = block['y']
self.canvas['tex_width'] = block['width']
self.canvas['tex_height'] = block['height']
self.canvas['blend_top'] = float(block['blend_top'])
self.canvas['blend_bottom'] = float(block['blend_bottom'])
self.canvas['blend_left'] = float(block['blend_left'])
self.canvas['blend_right'] = float(block['blend_right'])
self.canvas.clear()
with self.canvas:
self.rect = Rectangle(texture = self.texture, size = (2, 2), pos = (-1, -1))
class Renderer(Widget):
_curr_mode = StringProperty("")
_nframes = NumericProperty(-1)
_zoom_speed = 0.03
def __init__(self,
smpl_faces_path=None,
keypoints_spec=None,
obj_mesh_path=None):
if smpl_faces_path is not None:
self._smpl_faces = np.load(smpl_faces_path)
if keypoints_spec is not None:
self.keypoints_spec = keypoints_spec.copy()
self.keypoints_spec.sort(key=lambda kpnt: kpnt["smpl_indx"])
# Extract the parent indices from the keypoints dictionary
self.parents = [
next(
(indx for (indx, kpnt) in enumerate(self.keypoints_spec)
if kpnt["name"] == p_kpnt["parent"]),
-1,
) for p_kpnt in self.keypoints_spec
]
if obj_mesh_path is not None:
self._monkey_scene = ObjFile(obj_mesh_path)
# Make a canvas and add simple view
self.canvas = RenderContext(compute_normal_mat=True)
shader_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
"simple.glsl")
self._init_shader(shader_path)
super().__init__()
self._create_mesh_fn = {
"monkey": self._create_monkey_mesh,
"monkey_no_norms": self._create_monkey_mesh_no_norms,
"random": self._create_rand_mesh,
"smpl_mesh": self._create_smpl_mesh,
"smpl_kpnts": self._create_smpl_kpnts,
"error_vectors": self._create_error_vectors,
}
self._curr_mode = "triangles"
self.curr_obj = None
self._nframes = 0
self.reset_scene()
self._store_quat = None
self._dx_acc, self._dy_acc = 0, 0
def _init_shader(self, path):
self.canvas.shader.source = resource_find(path)
self.canvas["ambient_light"] = (0.2, 0.1, 0.2)
self.reset_highlight()
def setup_scene(self, rendered_obj, opts: dict = {}):
self.curr_obj = rendered_obj
self._recalc_normals = True
self._update_glsl()
with self.canvas:
self.cb = Callback(self._setup_gl_context)
PushMatrix()
self.trans = Translate(0, 0, -3)
self.rotx = Rotate(
1, 1, 0, 0) # so that the object does not break continuity
self.scale = Scale(1, 1, 1)
self.yaw = Rotate(0, 0, 0, 1)
self.pitch = Rotate(0, -1, 0, 0)
self.roll = Rotate(0, 0, 1, 0)
self.quat = euler_to_quaternion([0, 0, 0])
UpdateNormalMatrix()
if rendered_obj in self._create_mesh_fn.keys():
self._create_mesh_fn[rendered_obj](**opts)
# self.trans.x += 1 # move everything to the right
PopMatrix()
self.cb = Callback(self._reset_gl_context)
def _update_glsl(self):
asp = self.width / float(self.height)
proj = Matrix().view_clip(-asp, asp, -1, 1, 1.5, 100, 1)
self.canvas["projection_mat"] = proj
def _setup_gl_context(self, *args):
glEnable(GL_DEPTH_TEST)
def _reset_gl_context(self, *args):
glDisable(GL_DEPTH_TEST)
def reset_scene(self):
self.canvas.clear()
self._nframes = 0
def _create_monkey_mesh(self):
m = list(self._monkey_scene.objects.values())[0]
self._mesh = Mesh(
vertices=m.vertices,
indices=m.indices,
fmt=m.vertex_format,
mode=self._curr_mode,
)
self._mesh_data = GLMeshData(vertices=np.array(m.verts_raw))
self._mesh_data.verts_gl = m.vertices
self._mesh_data.indices = m.indices
def _create_monkey_mesh_no_norms(self):
m = list(self._monkey_scene.objects.values())[0]
self._mesh_data = GLMeshData(vertices=np.array(m.verts_raw),
faces=m.faces)
self._mesh = Mesh(
vertices=self._mesh_data.verts_gl,
indices=self._mesh_data.indices,
fmt=self._mesh_data.vertex_format,
mode=self._curr_mode,
)
def _create_rand_mesh(self):
verts = np.random.logistic(scale=0.5, size=(10000, 3))
# verts = np.random.normal(scale=0.7, size=(10000, 3))
# verts = np.random.laplace(scale=0.6, size=(10000, 3))
# verts = np.random.lognormal(size=(10000, 3))
self._mesh_data = GLMeshData(vertices=verts, faces=self._smpl_faces)
self._mesh = Mesh(
vertices=self._mesh_data.verts_gl,
indices=self._mesh_data.indices,
fmt=self._mesh_data.vertex_format,
mode=self._curr_mode,
)
def _create_smpl_mesh(self):
verts = np.random.rand(6890, 3) * 2 - 1
self._mesh_data = GLMeshData(vertices=verts, faces=self._smpl_faces)
self._curr_mode = "triangles"
self._mesh = Mesh(
vertices=self._mesh_data.verts_gl,
indices=self._mesh_data.indices,
fmt=self._mesh_data.vertex_format,
mode=self._curr_mode,
)
self.rotx.angle += 180
# For debugging rotations: x,y,z axis
# Mesh(
# vertices=[1, 0, 0, -1, -1, -1, 0, 0, 0, 0, 0, -1, -1, -1, 0, 0],
# indices=[0, 1],
# fmt=GLMeshData.vertex_format,
# mode='lines'
# )
# Mesh(
# vertices=[0, 1, 0, -1, -1, -1, 0, 0, 0, 0, 0, -1, -1, -1, 0, 0],
# indices=[0, 1],
# fmt=GLMeshData.vertex_format,
# mode='lines'
# )
# Mesh(
# vertices=[0, 0, 1, -1, -1, -1, 0, 0, 0, 0, 0, -1, -1, -1, 0, 0],
# indices=[0, 1],
# fmt=GLMeshData.vertex_format,
# mode='lines'
# )
def _create_smpl_kpnts(self):
verts = np.random.rand(24, 3) * 2 - 1
indices = []
for indx, kpnt in enumerate(self.parents):
if kpnt < 0:
continue
indices.extend([kpnt, indx])
self._mesh_data = GLMeshData(verts,
normals=-np.ones((24, 3)),
indices=indices)
self._curr_mode = "lines"
self._mesh = Mesh(
vertices=self._mesh_data.verts_gl,
indices=self._mesh_data.indices,
fmt=self._mesh_data.vertex_format,
mode=self._curr_mode,
)
self.trans.z += 0.2
self.rotx.angle += 180
def _create_error_vectors(self, start_verts, direction_vecs):
"""Render lines starting at :param: `start_verts` and with direction :param: `direction_vecs`.
Parameters
----------
starts_verts: `numpy.array`, (N x 3)
The starting points' 3D coordinates of the N lines
direction_vecs: `numpy.array`, (N x 3)
The direction vectors of the N error arrows
"""
for i in range(start_verts.shape[0]):
start_point = np.expand_dims(start_verts[i, :], axis=1).transpose()
dir_vec = direction_vecs[i, :]
self._create_arrow(start_point, dir_vec)
self.rotx.angle += 180
self.trans.z += 0.2
def _create_arrow(self, start_point: np.array, dir_vec: np.array):
"""Renders an arrow starting from `start_point` and pointing towards `dir_vec`.
Parameters
----------
start_point : `numpy.array`, (3 x 1)
The coordinates of the origin of the arrow.
dir_vec : `numpy.array`, (3 x 1)
The direction vector of the arrow.
Returns
-------
arrow_shaft : `kivy.graphics.Mesh`
The mesh data of the arrow's shaft.
arrw_head : `kivy.graphics.Mesh`
The mesh data of the arrow's head.
"""
dir_vec /= 2
end_point = start_point + dir_vec
shaft_data = GLMeshData(
np.concatenate((start_point, end_point), axis=0),
normals=-np.ones((2, 3)),
indices=[0, 1],
)
arrow_shaft = Mesh(
vertices=shaft_data.verts_gl,
indices=shaft_data.indices,
fmt=shaft_data.vertex_format,
mode="lines",
)
# Create the points of the arrow head's base circle
bases = self._get_base_vecs(dir_vec)
npoints = 20
step = 2 * math.pi / npoints
points = []
for j in range(npoints):
points.append(0.03 * (bases[0] * math.sin(step * j) +
bases[1] * math.cos(step * j)))
points = np.array(points)
points += start_point + dir_vec * 0.8
# Add to the points of the circle the tip of the arrow and set up the faces to create the mesh
points = np.append(points, end_point, axis=0)
faces = []
for j in range(npoints):
faces.append([j, npoints, (j + 1) % npoints])
head_data = GLMeshData(points, faces=faces)
arrow_head = Mesh(
vertices=head_data.verts_gl,
indices=head_data.indices,
fmt=head_data.vertex_format,
mode="triangles",
)
return (arrow_shaft, arrow_head)
def _get_base_vecs(self, vec: np.array):
"""Create base 3 base vectors from a given vector.
Parameters
----------
vec : `numpy.array`, (3, )
One vector of the resulting base vectors.
Returns
-------
base1, base2 : `numpy.array`, (3, )
The perpendicular vectors to the :param: `vec` and to each other. The three vectors `vec`, `base1`
and `base2` form the base.
"""
base1 = np.random.randn(3)
base1 -= np.dot(vec, base1) * vec
base1 /= np.linalg.norm(base1)
base2 = np.cross(vec, base1)
return base1, base2
def set_vertices(self, vertices, keypoints):
"""Set the vertices of the currently rendered mesh.
Parameters
----------
vertices: `numpy.array`, (N x 3)
The 3D coordinates of the mesh's N vertices.
keypoints : `numpy.array`, (24 x 3)
The 3D coordinates of the 24 SMPL keypoints.
"""
if not hasattr(self, "_mesh"):
return
if self._recalc_normals and self.curr_obj == "smpl_mesh":
self._mesh_data.populate_normals_and_indices(vertices)
self._recalc_normals = False
if self.curr_obj == "smpl_mesh":
self._mesh_data.vertices = vertices
elif self.curr_obj == "smpl_kpnts":
self._mesh_data.vertices = keypoints
self._curr_keypoints = keypoints
self._highlight_keypoint()
self._mesh.vertices = self._mesh_data.verts_gl
self._nframes += 1
def setup_highlight(self, kpnt_indx: int):
"""Setup the highlighting around the given keypoint.
Parameters
----------
kpnt_indx : `int`
The SMPL index of the keypoint to highlight.
"""
self._highlighted_kpnt_indx = kpnt_indx
self.canvas["sphere_color"] = (0.415, 0.878, 0.662)
self._highlight_keypoint()
def _highlight_keypoint(self):
"""Highlight the selected keypoint for each frame."""
if not hasattr(self, "_highlighted_kpnt_indx"):
return
sphere_center = self._curr_keypoints[self._highlighted_kpnt_indx]
sphere_radius = (
self.keypoints_spec[self._highlighted_kpnt_indx]["hradius"] *
self.scale.x)
self.canvas["sphere_center"] = tuple(
[float(coord) for coord in sphere_center])
self.canvas["sphere_radius"] = float(sphere_radius)
def reset_highlight(self):
if not hasattr(self, "_highlighted_kpnt_indx"):
return
del self._highlighted_kpnt_indx
self.canvas["sphere_color"] = (1.0, 1.0, 1.0)
self.canvas["sphere_radius"] = 0.0
def play_animation(self, animation_spec):
if not hasattr(self, "_mesh"):
raise UnboundLocalError("The mesh does not exist")
if animation_spec == "correct_repetition":
start_color = self.canvas["object_color"]
target_color = (0.0, 0.6, 0.0)
Clock.schedule_interval(
partial(self._reach_color_anim, start_color, target_color,
True), 0.01)
def _reach_color_anim(self, start_color, target_color, reverse, dt):
"""Play a color animation on the rendered object. The animations starts from a starting color,
reaches a target color and if :param: reverse is True, returns slowly back to the starting color.
Parameters
----------
start_color: tuple
The rgb components specifying the starting color in the range [0,1].
target_color: tuple
The rgb components specifying the target color in the range [0, 1].
reverse: bool
If True, plays the animation in reverse when the target color is reached.
If False, the animation ends when the target color is reached.
"""
new_col = []
for i in range(3):
color_dif = target_color[i] - start_color[i]
col_comp = self.canvas["object_color"][i] + color_dif * 0.05
if (color_dif < 0 and col_comp < target_color[i]) or (
color_dif > 0 and col_comp > target_color[i]):
col_comp = target_color[i]
new_col.append(col_comp)
self.canvas["object_color"] = tuple(new_col)
if self.canvas["object_color"] == target_color:
# When the target color is reached, play in reverse or stop the animation.
if reverse:
Clock.schedule_interval(
partial(self._reach_color_anim, target_color, start_color,
False),
0.07,
)
return False
def _scale_anim(self, delta):
step = ((self._nframes) % 360) * np.pi / 180.0
scale_factors = (np.sin(step) * 0.7 + 0.9, -np.sin(step) * 0.7 + 0.9,
1)
self.scale.xyz = scale_factors
def _rotate_anim(self, delta):
self.roty.angle += delta * 50
def _deform_anim(self, delta):
vertices = (
self._mesh_data.vertices +
np.random.normal(size=self._mesh_data.vertices.shape) * 0.02)
self.set_vertices(vertices)
def change_mesh_mode(self):
cur_indx = self._mesh_MODES.index(self._curr_mode)
self._curr_mode = self._mesh_MODES[(cur_indx + 1) %
len(self._mesh_MODES)]
if hasattr(self, "_mesh"):
self._mesh.mode = self._curr_mode
def on_touch_down(self, touch):
"""Initialize potential rotation and handle scaling of the mesh."""
if hasattr(self, "_mesh"):
if self.collide_point(*touch.pos):
# Zoom in and out functionality
if touch.is_mouse_scrolling:
prev_scale = list(self.scale.xyz)
if touch.button == "scrolldown":
new_scale = [
sc_ax + self._zoom_speed for sc_ax in prev_scale
]
elif touch.button == "scrollup":
new_scale = [
sc_ax - self._zoom_speed for sc_ax in prev_scale
]
self.scale.xyz = tuple(new_scale)
# Accumulators for rotation
self._dx_acc, self._dy_acc = 0, 0
self._store_quat = self.quat
touch.grab(self)
return super().on_touch_down(touch)
def on_touch_move(self, touch):
"""On mouse click and move, handle the rotation of the mesh."""
if touch.grab_current is self:
self._dx_acc += touch.dx
self._dy_acc += touch.dy
new_quat = euler_to_quaternion(
[0.01 * self._dx_acc, 0.01 * self._dy_acc, 0])
self.quat = quat_mult(self._store_quat, new_quat)
euler_radians = quaternion_to_euler(self.quat)
self.roll.angle, self.pitch.angle, self.yaw.angle = euler_to_roll_pitch_yaw(
euler_radians)
return super().on_touch_down(touch)
def on_touch_up(self, touch):
"""If it was a single click, handle it."""
if touch.grab_current is self:
if touch.pos == touch.opos and not touch.is_mouse_scrolling:
# If the position didnt change, this was a single click
if hasattr(self, "single_click_handle"):
kpnts_2d = self._kpnts_to_2D()
closest_kpnts = self._find_closest_kpnts(
touch.pos, kpnts_2d)
self.single_click_handle(touch.pos, closest_kpnts)
touch.ungrab(self)
return super().on_touch_down(touch)
def _kpnts_to_2D(self):
width, height = self.width, self.height
verts_3d = self._curr_keypoints
modelview = np.array(self.canvas["modelview_mat"].tolist())
projection = np.array(self.canvas["projection_mat"].tolist())
transforms_mat = self._get_transformation_matrix()
verts_2d = np.zeros(shape=(verts_3d.shape[0], 2))
for i, vert in enumerate(verts_3d):
hom_vert = np.expand_dims(np.append(vert, 1), axis=1)
pos = np.dot(modelview, hom_vert)
pos = np.dot(projection, pos)
pos = np.dot(transforms_mat, pos)
pos[:3] /= pos[3]
pos[0] = (pos[0] + 1) * width / 2.0
pos[1] = (pos[1] + 1) * height / 2.0
verts_2d[i, :] = pos[:2].flatten()
return verts_2d
def _get_transformation_matrix(self):
scale = np.array(self.scale.matrix.tolist())
rotx = np.array(self.rotx.matrix.tolist())
roll = np.array(self.roll.matrix.tolist())
pitch = np.array(self.pitch.matrix.tolist())
yaw = np.array(self.yaw.matrix.tolist())
transl = np.array(self.trans.matrix.tolist())
mat = np.dot(transl, scale)
mat = np.dot(roll, mat)
mat = np.dot(pitch, mat)
mat = np.dot(yaw, mat)
mat = np.dot(rotx, mat)
return mat
def _find_closest_kpnts(self, pos, kpnts_2d):
dists = np.linalg.norm(kpnts_2d - pos, axis=1)
return self.keypoints_spec[dists.argmin()]["name"]
class Renderer(Widget):
def __init__(self, **kwargs):
super(Renderer, self).__init__(**kwargs)
self.canvas = RenderContext(compute_normal_mat=True)
self.canvas.shader.source = resource_find('simple.glsl')
self._touches = []
def render(self, obj_file):
self.canvas.clear()
self.scene = ObjFile(obj_file)
with self.canvas:
self.cb = Callback(lambda args: glEnable(GL_DEPTH_TEST))
PushMatrix()
self._setup_scene()
PopMatrix()
self.cb = Callback(lambda args: glDisable(GL_DEPTH_TEST))
Clock.schedule_interval(self._update_glsl, 1 / 60.)
def _update_glsl(self, *_):
p = self.parent.parent
asp = float(p.width) / p.height * p.size_hint_y / p.size_hint_x
proj = Matrix().view_clip(-asp, asp, -1, 1, 1, 100, 1)
self.canvas['projection_mat'] = proj
self.canvas['diffuse_light'] = (1.0, 1.0, 0.8)
self.canvas['ambient_light'] = (0.1, 0.1, 0.1)
def _setup_scene(self):
for mi, m in enumerate(self.scene.objects.values()):
Color(1, 1, 1, 1)
PushMatrix()
Translate(0, -0.3, -1.8)
setattr(self, 'mesh%03d_rotx' % mi, Rotate(180, 1, 0, 0))
setattr(self, 'mesh%03d_roty' % mi, Rotate(0, 0, 1, 0))
setattr(self, 'mesh%03d_scale' % mi, Scale(1))
UpdateNormalMatrix()
mesh = Mesh(
vertices=m.vertices,
indices=m.indices,
fmt=m.vertex_format,
mode='triangles',
)
setattr(self, 'mesh%03d' % mi, mesh)
PopMatrix()
def _angle_from_touch(self, touch):
x_angle = (touch.dx / self.width) * 360
y_angle = -1 * (touch.dy / self.height) * 360
return x_angle, y_angle
def on_touch_down(self, touch):
self._touch = touch
touch.grab(self)
self._touches.append(touch)
def _scale_objects(self, scale):
objs = self.scene.objects.values()
for mi in range(len(objs)):
mesh_scale = getattr(self, 'mesh%03d_scale' % mi)
xyz = mesh_scale.xyz
if scale != 0:
mesh_scale.xyz = tuple(p + scale for p in xyz)
def on_touch_move(self, touch):
if touch.grab_current is self:
scale_factor = 0.01
self._update_glsl()
if touch in self._touches:
if len(self._touches) == 1:
ax, ay = self._angle_from_touch(touch)
for mi in range(len(self.scene.objects.values())):
rot = getattr(self, 'mesh%03d_rotx' % mi)
rot.angle += ay
rot = getattr(self, 'mesh%03d_roty' % mi)
rot.angle -= ax
elif len(self._touches) == 2:
# Use two touches to determine if we need scale
touch1, touch2 = self._touches
old_pos1 = (touch1.x - touch1.dx, touch1.y - touch1.dy)
old_pos2 = (touch2.x - touch2.dx, touch2.y - touch2.dy)
old_dx = old_pos1[0] - old_pos2[0]
old_dy = old_pos1[1] - old_pos2[1]
old_distance = old_dx**2 + old_dy**2
s = "old_distance = %s; " % old_distance
new_dx = touch1.x - touch2.x
new_dy = touch1.y - touch2.y
new_distance = new_dx**2 + new_dy**2
s += "new_distance = %s -> " % new_distance
if new_distance > old_distance:
scale = scale_factor
s += "scale up"
elif new_distance == old_distance:
scale = 0
else:
scale = -1 * scale_factor
s += "scale down"
Logger.debug(s)
self._scale_objects(scale)
def on_touch_up(self, touch):
if touch.grab_current is self:
touch.ungrab(self)
self._touches.remove(touch)
