def demo(video_frame = None):
user_input = {'down':False, 'mouse':(0,0)}
# Callback functions
def on_resize(window,w, h):
h = max(h,1)
w = max(w,1)
gui.update_window(w,h)
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
# norm_size = normalize((w,h),glfwGetWindowSize(window))
# fb_size = denormalize(norm_size,glfwGetFramebufferSize(window))
adjust_gl_view(w,h)
glfwMakeContextCurrent(active_window)
track.set_window_size(w,h)
def on_iconify(window,iconfied):
pass
def on_key(window, key, scancode, action, mods):
gui.update_key(key,scancode,action,mods)
def on_char(window,char):
gui.update_char(char)
def on_button(window,button, action, mods):
gui.update_button(button,action,mods)
if action == GLFW_PRESS:
user_input['down'] = True
user_input['mouse'] = glfwGetCursorPos(window)
if action == GLFW_RELEASE:
user_input['down'] = False
# pos = normalize(pos,glfwGetWindowSize(window))
# pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
def on_pos(window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
gui.update_mouse(x,y)
if user_input['down']:
old_x,old_y = user_input['mouse']
track.drag_to(x-old_x,y-old_y)
user_input['mouse'] = x,y
def on_scroll(window,x,y):
gui.update_scroll(x,y)
track.zoom_to(y)
# get glfw started
glfwInit()
window = glfwCreateWindow(width, height, "pyglui demo", None, None)
glfwMakeContextCurrent(window)
if not window:
exit()
glfwSetWindowPos(window,0,0)
# Register callbacks window
glfwSetFramebufferSizeCallback(window,on_resize)
glfwSetWindowIconifyCallback(window,on_iconify)
glfwSetKeyCallback(window,on_key)
glfwSetCharCallback(window,on_char)
glfwSetMouseButtonCallback(window,on_button)
glfwSetCursorPosCallback(window,on_pos)
glfwSetScrollCallback(window,on_scroll)
init()
glutInit()
basic_gl_setup()
gui = ui.UI()
track = Trackball()
on_resize(window,*glfwGetFramebufferSize(window))
if video_frame == None:
video_frame = (np.linspace(0,1,num=(400*400*4))*255).astype(np.uint8).reshape((400,400,4)) #the randomized image
test_sphere = geometry.Sphere([0,5,0],1)
test_ellipse = geometry.Ellipse((0,3),5,3,0)
#return window
#def demo_update(window):
while not glfwWindowShouldClose(window):
clear_gl_screen()
track.push()
#glutils.draw_polyline3d([(0,0,2),(1,1,2)],color=RGBA(0.4,0.5,0.3,0.5))
draw_sphere(test_sphere)
draw_ellipse(test_ellipse)
draw_video_screen(video_frame)
draw_coordinate_system(20)
track.pop()
glfwSwapBuffers(window)
glfwPollEvents()
#def demo_close(window):
glfwDestroyWindow(window)
glfwTerminate()
logger.debug("Process done")
class Visualizer():
def __init__(self,name = "unnamed", focal_length = 554.25625, intrinsics = None, run_independently = False):
self.sphere = geometry.Sphere() #the eyeball, initialized as something random
self.ellipses = [] #collection of ellipses
self.circles = [] #collection of all 3D circles on the sphere
self.video_frame = (np.linspace(0,1,num=(400*400*4))*255).astype(np.uint8).reshape((400,400,4)) #the randomized image, should be video frame
# self.screen_points = [] #collection of points
if intrinsics == None:
intrinsics = np.identity(3)
if focal_length != None:
intrinsics[0,0] = focal_length
intrinsics[1,1] = focal_length
logger.warning('no camera intrinsic input, set to focal length')
else:
logger.warning('no camera intrinsic input, set to default identity matrix')
# transformation matrices
self.intrinsics = intrinsics #camera intrinsics of our webcam.
self.scale = 1
self.name = name
self._window = None
self.input = None
self.trackball = None
self.run_independently = run_independently
self.window_should_close = False
############## MATRIX FUNCTIONS ##############################
def get_pixel_space_matrix(self):
# returns a homoegenous matrix
temp = self.get_anthropomorphic_matrix()
temp[3,3] *= self.scale
return temp
def get_adjusted_pixel_space_matrix(self,scale):
# returns a homoegenous matrix
temp = self.get_pixel_space_matrix()
temp[3,3] *= scale
return temp
def get_anthropomorphic_matrix(self):
temp = np.identity(4)
temp[2,2] *=-1 #consistent with our 3d coord system
return temp
def get_image_space_matrix(self,scale=1.):
temp = self.get_adjusted_pixel_space_matrix(scale)
temp[1,1] *=-1 #image origin is top left
temp[0,3] = -self.intrinsics[0,2] #cx
temp[1,3] = self.intrinsics[1,2] #cy
temp[2,3] = -self.intrinsics[0,0] #focal length
return temp.T
def get_pupil_transformation_matrix(self,circle):
"""
OpenGL matrix convention for typical GL software
with positive Y=up and positive Z=rearward direction
RT = right
UP = up
BK = back
POS = position/translation
US = uniform scale
float transform[16];
[0] [4] [8 ] [12]
[1] [5] [9 ] [13]
[2] [6] [10] [14]
[3] [7] [11] [15]
[RT.x] [UP.x] [BK.x] [POS.x]
[RT.y] [UP.y] [BK.y] [POS.y]
[RT.z] [UP.z] [BK.z] [POS.Z]
[ ] [ ] [ ] [US ]
"""
temp = self.get_anthropomorphic_matrix()
right = temp[:3,0]
up = temp[:3,1]
back = temp[:3,2]
translation = temp[:3,3]
back[:] = np.array(circle.normal)
back[-2] *=-1 #our z axis is inverted
back[-0] *=-1 #our z axis is inverted
back[:] /= np.linalg.norm(back)
right[:] = get_perpendicular_vector(back)/np.linalg.norm(get_perpendicular_vector(back))
up[:] = np.cross(right,back)/np.linalg.norm(np.cross(right,back))
translation[:] = np.array((circle.center[0],circle.center[1],-circle.center[2]))
return temp.T
def get_rotated_sphere_matrix(self,circle,sphere):
"""
OpenGL matrix convention for typical GL software
with positive Y=up and positive Z=rearward direction
RT = right
UP = up
BK = back
POS = position/translation
US = uniform scale
float transform[16];
[0] [4] [8 ] [12]
[1] [5] [9 ] [13]
[2] [6] [10] [14]
[3] [7] [11] [15]
[RT.x] [UP.x] [BK.x] [POS.x]
[RT.y] [UP.y] [BK.y] [POS.y]
[RT.z] [UP.z] [BK.z] [POS.Z]
[ ] [ ] [ ] [US ]
"""
temp = self.get_anthropomorphic_matrix()
right = temp[:3,0]
up = temp[:3,1]
back = temp[:3,2]
translation = temp[:3,3]
back[:] = np.array(circle.normal)
back[-2] *=-1 #our z axis is inverted
back[-0] *=-1 #our z axis is inverted
back[:] /= np.linalg.norm(back)
right[:] = get_perpendicular_vector(back)/np.linalg.norm(get_perpendicular_vector(back))
up[:] = np.cross(right,back)/np.linalg.norm(np.cross(right,back))
translation[:] = np.array((sphere.center[0],sphere.center[1],-sphere.center[2]))
return temp.T
############## DRAWING FUNCTIONS ##############################
def draw_rect(self):
glBegin(GL_QUADS)
glColor4f(0.0, 0.0, 0.5,0.2) #set color to light blue
glVertex2f(0,0)
glVertex2f(0 + 32, 0)
glVertex2f(0 + 32, 0 + 16)
glVertex2f(0, 0 + 16)
glEnd()
def draw_frustum(self, scale=1):
# average focal length
#f = (K[0, 0] + K[1, 1]) / 2
# compute distances for setting up the camera pyramid
W = self.intrinsics[0,2]
H = self.intrinsics[1,2]
Z = self.intrinsics[0,0]
# scale the pyramid
W *= scale
H *= scale
Z *= scale
# draw it
glColor4f( 1, 0.5, 0, 0.5 )
glBegin( GL_LINE_LOOP )
glVertex3f( 0, 0, 0 )
glVertex3f( -W, H, Z )
glVertex3f( W, H, Z )
glVertex3f( 0, 0, 0 )
glVertex3f( W, H, Z )
glVertex3f( W, -H, Z )
glVertex3f( 0, 0, 0 )
glVertex3f( W, -H, Z )
glVertex3f( -W, -H, Z )
glVertex3f( 0, 0, 0 )
glVertex3f( -W, -H, Z )
glVertex3f( -W, H, Z )
glEnd( )
def draw_coordinate_system(self,l=1):
# Draw x-axis line. RED
glLineWidth(2)
glColor3f( 1, 0, 0 )
glBegin( GL_LINES )
glVertex3f( 0, 0, 0 )
glVertex3f( l, 0, 0 )
glEnd( )
# Draw y-axis line. GREEN.
glColor3f( 0, 1, 0 )
glBegin( GL_LINES )
glVertex3f( 0, 0, 0 )
glVertex3f( 0, l, 0 )
glEnd( )
# Draw z-axis line. BLUE
glColor3f( 0, 0,1 )
glBegin( GL_LINES )
glVertex3f( 0, 0, 0 )
glVertex3f( 0, 0, l )
glEnd( )
def draw_sphere(self,circle):
# this function draws the location of the eye sphere
glPushMatrix()
glColor4f(0.0, 1.0, 1.0,0.2) #set color to blue
glLoadMatrixf(self.get_rotated_sphere_matrix(circle,self.sphere))
glutSolidSphere(self.sphere.radius,20,10)
glColor4f(0.0, 0.0, 1.0,0.8) #set color to blue
glutWireSphere(self.sphere.radius,20,10)
glPopMatrix()
def draw_all_ellipses(self):
# draws all ellipses in self.ellipses.
glPushMatrix()
for ellipse in self.ellipses[-10:]:
glColor3f(0.0, 1.0, 0.0) #set color to green
pts = cv2.ellipse2Poly( (int(ellipse.center[0]),int(ellipse.center[1])),
(int(ellipse.major_radius),int(ellipse.minor_radius)),
int(ellipse.angle*180/scipy.pi),0,360,15)
draw_polyline(pts,2,color = RGBA(0,1,1,.5))
glPopMatrix()
def draw_circle(self,circle):
glPushMatrix()
glLoadMatrixf(self.get_pupil_transformation_matrix(circle))
draw_points(((0,0),),color=RGBA(1.1,0.2,.8))
glColor4f(0.0, 0.0, 0.0,0.5) #set color to green
glBegin(GL_POLYGON) #draw circle
for i in xrange(45):
rad = i*16*scipy.pi/360.
glVertex2f(np.cos(rad)*circle.radius,np.sin(rad)*circle.radius)
glEnd()
glPopMatrix()
def draw_video_screen(self):
# Function to draw self.video_frame.
# Need to scale
glPushMatrix()
tex_id = create_named_texture(self.video_frame.shape)
update_named_texture(tex_id,self.video_frame) #since image doesn't change, do not need to put in while loop
draw_named_texture(tex_id)
glPopMatrix()
########## Setup functions I don't really understand ############
def basic_gl_setup(self):
glEnable(GL_POINT_SPRITE )
glEnable(GL_VERTEX_PROGRAM_POINT_SIZE) # overwrite pointsize
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
glEnable(GL_BLEND)
glClearColor(.8,.8,.8,1.)
glEnable(GL_LINE_SMOOTH)
# glEnable(GL_POINT_SMOOTH)
glHint(GL_LINE_SMOOTH_HINT, GL_NICEST)
glEnable(GL_LINE_SMOOTH)
glEnable(GL_POLYGON_SMOOTH)
glHint(GL_POLYGON_SMOOTH_HINT, GL_NICEST)
def adjust_gl_view(self,w,h):
"""
adjust view onto our scene.
"""
glViewport(0, 0, w, h)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
glOrtho(0, w, h, 0, -1, 1)
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
def clear_gl_screen(self):
glClearColor(.9,.9,0.9,1.)
glClear(GL_COLOR_BUFFER_BIT)
########### Open, update, close #####################
def open_window(self):
if not self._window:
self.input = {'button':None, 'mouse':(0,0)}
self.trackball = Trackball()
# get glfw started
if self.run_independently:
glfwInit()
window = glfwGetCurrentContext()
self._window = glfwCreateWindow(640, 480, self.name, None, window)
glfwMakeContextCurrent(self._window)
if not self._window:
exit()
glfwSetWindowPos(self._window,2000,0)
# Register callbacks window
glfwSetFramebufferSizeCallback(self._window,self.on_resize)
glfwSetWindowIconifyCallback(self._window,self.on_iconify)
glfwSetKeyCallback(self._window,self.on_key)
glfwSetCharCallback(self._window,self.on_char)
glfwSetMouseButtonCallback(self._window,self.on_button)
glfwSetCursorPosCallback(self._window,self.on_pos)
glfwSetScrollCallback(self._window,self.on_scroll)
glfwSetWindowCloseCallback(self._window,self.on_close)
# get glfw started
if self.run_independently:
init()
glutInit()
self.basic_gl_setup()
self.glfont = fs.Context()
self.glfont.add_font('opensans',get_opensans_font_path())
self.glfont.set_size(22)
self.glfont.set_color_float((0.2,0.5,0.9,1.0))
# self.gui = ui.UI()
self.on_resize(self._window,*glfwGetFramebufferSize(self._window))
def update_window(self, g_pool = None,model = None):
if self.window_should_close:
self.close_window()
if self._window != None:
glfwMakeContextCurrent(self._window)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glAlphaFunc(GL_GREATER, 0)
self.clear_gl_screen()
self.trackball.push()
# print glGetFloatv(GL_MODELVIEW_MATRIX).T
#THINGS I NEED TO DRAW
# 1. in anthromorphic space, draw pupil sphere and circles on it
glLoadMatrixf(self.get_anthropomorphic_matrix())
if model: #if we are feeding in spheres to draw
for pupil in model.observations[-10:]: #draw the last 10
self.draw_circle(pupil.circle)
self.draw_sphere(pupil.circle) #draw the eyeball
self.draw_coordinate_system(4)
glTranslatef(*self.sphere.center)
glScalef(0.1,0.1,0.1)
self.glfont.draw_multi_line_text(0,0,'Eyeball location: \n %s'%self.sphere.center)
# 1a. draw frustum in pixel scale, but retaining origin
glLoadMatrixf(self.get_adjusted_pixel_space_matrix(30))
self.draw_frustum()
# 2. in pixel space, draw ellipses, and video frame
glLoadMatrixf(self.get_image_space_matrix(30))
if g_pool: #if display eye camera frames
draw_named_texture(g_pool.image_tex,quad=((0,480),(640,480),(640,0),(0,0)),alpha=0.5)
self.draw_all_ellipses()
# 3. display text on screen
# glWindowPos2i(100, 100)
# glutPrint("SAMPLE TEXT")
# glFontBegin(font);
# glScalef(8.0, 8.0, 8.0);
# glTranslatef(30, 30, 0);
# glFontTextOut("Test", 5, 5, 0);
# glFontEnd();
# glFlush();
self.trackball.pop()
glfwSwapBuffers(self._window)
glfwPollEvents()
return True
def close_window(self):
if self.window_should_close == True:
glfwDestroyWindow(self._window)
if self.run_independently:
glfwTerminate()
self._window = None
self.window_should_close = False
logger.debug("Process done")
############ window callbacks #################
def on_resize(self,window,w, h):
h = max(h,1)
w = max(w,1)
self.trackball.set_window_size(w,h)
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
self.adjust_gl_view(w,h)
glfwMakeContextCurrent(active_window)
def on_iconify(self,window,x,y): pass
def on_key(self,window, key, scancode, action, mods): pass
#self.gui.update_key(key,scancode,action,mods)
def on_char(window,char): pass
# self.gui.update_char(char)
def on_button(self,window,button, action, mods):
# self.gui.update_button(button,action,mods)
if action == GLFW_PRESS:
self.input['button'] = button
self.input['mouse'] = glfwGetCursorPos(window)
if action == GLFW_RELEASE:
self.input['button'] = None
# pos = normalize(pos,glfwGetWindowSize(window))
# pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
def on_pos(self,window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
# self.gui.update_mouse(x,y)
if self.input['button']==GLFW_MOUSE_BUTTON_RIGHT:
old_x,old_y = self.input['mouse']
self.trackball.drag_to(x-old_x,y-old_y)
self.input['mouse'] = x,y
if self.input['button']==GLFW_MOUSE_BUTTON_LEFT:
old_x,old_y = self.input['mouse']
self.trackball.pan_to(x-old_x,y-old_y)
self.input['mouse'] = x,y
def on_scroll(self,window,x,y):
# self.gui.update_scroll(x,y)
self.trackball.zoom_to(y)
def on_close(self,window=None):
self.window_should_close = True
class Visualizer():
def __init__(self,name = "unnamed"):
self.name = name
self.sphere = geometry.Sphere()
self.ellipses = [] #collection of ellipses to display
self.projected_lines = [] #collection of projected lines to display
self.frame = None #video frame from eye
self._window = None
self.input = None
self.trackball = None
self.window_should_close = False
self.video_frame = (np.linspace(0,1,num=(400*400*4))*255).astype(np.uint8).reshape((400,400,4)) #the randomized image
self.test_sphere = geometry.Sphere([0,5,0],1)
self.test_ellipse = geometry.Ellipse((0,3),5,3,0)
############## DRAWING FUNCTIONS ##############################
def draw_coordinate_system(self,l=1):
# Draw x-axis line. RED
glColor3f( 1, 0, 0 )
glBegin( GL_LINES )
glVertex3f( 0, 0, 0 )
glVertex3f( l, 0, 0 )
glEnd( )
# Draw y-axis line. GREEN. #not working... why?
glColor3f( 0, 1, 0 )
glBegin( GL_LINES )
glVertex3f( 0, 0, 0 )
glVertex3f( 0, l, 0 )
glEnd( )
# Draw z-axis line. BLUE
glColor3f( 0, 0,1 )
glBegin( GL_LINES )
glVertex3f( 0, 0, 0 )
glVertex3f( 0, 0, l )
glEnd( )
def draw_sphere(self,sphere):
# this function draws the location of the eye sphere
glPushMatrix()
glColor3f(0.0, 0.0, 1.0)
glTranslate(sphere.center[0], sphere.center[1], sphere.center[2])
glutWireSphere(sphere.radius,20,20)
glPopMatrix()
def draw_ellipse(self,ellipse):
glPushMatrix()
glTranslate(ellipse.center[0], ellipse.center[1], 0)
glBegin(GL_LINE_LOOP)
for i in xrange(360):
rad = i*2*scipy.pi/360.
glVertex2f(np.cos(rad)*ellipse.major_radius,np.sin(rad)*ellipse.minor_radius)
glEnd()
glPopMatrix()
def draw_projected_line(self,line):
#draw a line from projected sphere center to the ellipse on frame.
""" TO BE IMPLEMENTED """
pass
def draw_video_screen(self,frame):
glPushMatrix()
tex_id = create_named_texture(frame.shape)
update_named_texture(tex_id,frame) #since image doesn't change, do not need to put in while loop
draw_named_texture(tex_id)
glPopMatrix()
########## Setup functions I don't really understand ############
def basic_gl_setup(self):
glEnable(GL_POINT_SPRITE )
glEnable(GL_VERTEX_PROGRAM_POINT_SIZE) # overwrite pointsize
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
glEnable(GL_BLEND)
glClearColor(.8,.8,.8,1.)
glEnable(GL_LINE_SMOOTH)
# glEnable(GL_POINT_SMOOTH)
glHint(GL_LINE_SMOOTH_HINT, GL_NICEST)
glEnable(GL_LINE_SMOOTH)
glEnable(GL_POLYGON_SMOOTH)
glHint(GL_POLYGON_SMOOTH_HINT, GL_NICEST)
def adjust_gl_view(self,w,h):
"""
adjust view onto our scene.
"""
glViewport(0, 0, w, h)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
glOrtho(0, w, h, 0, -1, 1)
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
def clear_gl_screen(self):
glClearColor(.9,.9,0.9,1.)
glClear(GL_COLOR_BUFFER_BIT)
########### Open, update, close #####################
def open_window(self):
if not self._window:
self.input = {'down':False, 'mouse':(0,0)}
self.trackball = Trackball()
# get glfw started
glfwInit()
self._window = glfwCreateWindow(width, height, self.name, None, None)
glfwMakeContextCurrent(self._window)
if not self._window:
exit()
glfwSetWindowPos(self._window,2000,0)
# Register callbacks window
glfwSetFramebufferSizeCallback(self._window,self.on_resize)
glfwSetWindowIconifyCallback(self._window,self.on_iconify)
glfwSetKeyCallback(self._window,self.on_key)
glfwSetCharCallback(self._window,self.on_char)
glfwSetMouseButtonCallback(self._window,self.on_button)
glfwSetCursorPosCallback(self._window,self.on_pos)
glfwSetScrollCallback(self._window,self.on_scroll)
glfwSetWindowCloseCallback(self._window,self.on_close)
init()
glutInit()
self.basic_gl_setup()
self.gui = ui.UI()
self.on_resize(self._window,*glfwGetFramebufferSize(self._window))
def update_window(self):
if self.window_should_close:
self.close_window()
if self._window != None:
self.clear_gl_screen()
self.trackball.push()
#glutils.draw_polyline3d([(0,0,2),(1,1,2)],color=RGBA(0.4,0.5,0.3,0.5))
self.draw_sphere(self.test_sphere)
self.draw_ellipse(self.test_ellipse)
self.draw_video_screen(self.video_frame)
self.draw_coordinate_system(20)
self.trackball.pop()
glfwSwapBuffers(self._window)
glfwPollEvents()
return True
def close_window(self):
if self.window_should_close == True:
glfwDestroyWindow(self._window)
glfwTerminate()
self._window = None
self.window_should_close = False
logger.debug("Process done")
############ window callbacks #################
def on_resize(self,window,w, h):
h = max(h,1)
w = max(w,1)
self.trackball.set_window_size(w,h)
active_window = glfwGetCurrentContext()
glfwMakeContextCurrent(window)
self.adjust_gl_view(w,h)
glfwMakeContextCurrent(active_window)
def on_iconify(self,window,x,y): pass
def on_key(self,window, key, scancode, action, mods):
self.gui.update_key(key,scancode,action,mods)
def on_char(window,char):
self.gui.update_char(char)
def on_button(self,window,button, action, mods):
self.gui.update_button(button,action,mods)
if action == GLFW_PRESS:
self.input['down'] = True
self.input['mouse'] = glfwGetCursorPos(window)
if action == GLFW_RELEASE:
self.input['down'] = False
# pos = normalize(pos,glfwGetWindowSize(window))
# pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels
def on_pos(self,window,x, y):
hdpi_factor = float(glfwGetFramebufferSize(window)[0]/glfwGetWindowSize(window)[0])
x,y = x*hdpi_factor,y*hdpi_factor
self.gui.update_mouse(x,y)
if self.input['down']:
old_x,old_y = self.input['mouse']
self.trackball.drag_to(x-old_x,y-old_y)
self.input['mouse'] = x,y
def on_scroll(self,window,x,y):
self.gui.update_scroll(x,y)
self.trackball.zoom_to(y)
def on_close(self,window=None):
self.window_should_close = True
