def main():
webcam = Webcam()
webcam.start()
mnp = MarkNplay()
while True:
image = webcam.get_current_frame()
mnp.screen_thread(image)
cv2.imshow('AR Book Reader', image)
cv2.waitKey(10)
def take_imgs(chessboard_size=(11,7), kSaveImageDeltaTime=1):
sys.path.append("../")
os.makedirs("./calib_images", exist_ok=True)
camera_num = 0
if len(sys.argv) == 2:
camera_num = int(sys.argv[1])
print('opening camera: ', camera_num)
webcam = Webcam(camera_num)
webcam.start()
lastSaveTime = time.time()
while True:
# get image from webcam
image = webcam.get_current_frame()
if image is not None:
# check if pattern found
ret, corners = cv2.findChessboardCorners(cv2.cvtColor(image,cv2.COLOR_BGR2GRAY), chessboard_size, None)
if ret == True:
print('found chessboard')
# save image
filename = datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss%f') + '.bmp'
image_path="./calib_images/" + filename
elapsedTimeSinceLastSave = time.time() - lastSaveTime
do_save = elapsedTimeSinceLastSave > kSaveImageDeltaTime
print(elapsedTimeSinceLastSave, kSaveImageDeltaTime)
if do_save:
lastSaveTime = time.time()
print('saving file ', image_path)
cv2.imwrite(image_path, image)
# draw the corners
image = cv2.drawChessboardCorners(image, chessboard_size, corners, ret)
cv2.imshow('camera', image)
else:
pass
#print('empty image')
if cv2.waitKey(1) & 0xFF == ord('q'):
break
return img
cbrow = 6
cbcol = 9
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
objp = np.zeros((cbrow * cbcol, 3), np.float32)
objp[:, :2] = np.mgrid[0:cbrow, 0:cbcol].T.reshape(-1, 2)
axis = np.float32([[0, 0, 0], [0, 3, 0], [3, 3, 0], [3, 0, 0], [0, 0, -3],
[0, 3, -3], [3, 3, -3], [3, 0, -3]])
while True:
# get image from webcam
img = webcam.get_current_frame()
# for fname in glob.glob('*.jpg'):
# img = cv2.imread(fname)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret, corners = cv2.findChessboardCorners(gray, (cbrow, cbcol), None)
if ret == True:
corners2 = cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1),
criteria)
# Find the rotation and translation vectors.
_, rvecs, tvecs, inliers = cv2.solvePnPRansac(objp, corners2, mtx,
dist)
# project 3D points to image plane
imgpts, jac = cv2.projectPoints(axis, rvecs, tvecs, mtx, dist)
class ArucoFootball:
# constants
INVERSE_MATRIX = np.array([[1.0, 1.0, 1.0, 1.0],
[-1.0, -1.0, -1.0, -1.0],
[-1.0, -1.0, -1.0, -1.0],
[1.0, 1.0, 1.0, 1.0]])
def __init__(self, btAddr):
# init needed values
self.btAddr = btAddr
self.player = None
self.texture_background = None
self.set_ids = []
self.set_players = []
self.players = []
self.calc_values()
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
# connect wiimote and create model
wiiModel = WiiModel.WiiModel(self.btAddr)
# init openGl, Qt and Wiimote
self.initOpenGL()
self.initGUI()
# run wiimote-connection-loop
thread = threading.Thread(target=wiiModel.wiimoteLoop, args=(self.mainWindow, self.cursor))
thread.start()
# run opengl and camera in a thread
thread = threading.Thread(target=glutMainLoop, args=())
thread.start()
# run Qt
self.app.exec_()
def initGUI(self):
self.app = QApplication(sys.argv)
self.mainWindow = MainWindow(self.players)
self.mainWindow.show()
self.set_player_widget = self.mainWindow.listWidgetB
self.unset_player_widget = self.mainWindow.listWidgetA
self.unset_player_widget.itemChanged.connect(self.removeID)
self.mainWindow.setFocus()
self.mainWindow.setWindowTitle("Tactic-Window")
self.mainWindow.resize(600, 800)
self.cursor = QCursor()
def initOpenGL(self):
# setup OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(1280, 960)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("Footballfield")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# assign shapes
player_model = "models/football-player-new.obj"
self.player1 = OBJ(player_model, 1)
self.player2 = OBJ(player_model, 2)
self.player3 = OBJ(player_model, 3)
# self.player4 = OBJ(player_model, 4)
# add Players to list
self.players.append(Player("Dani", "1", "player_images/dani-img.jpg", self.player1))
self.players.append(Player("Maxi", "2", "player_images/maxi-img.jpg", self.player2))
self.players.append(Player("Jonas", "3", "player_images/jonas-img.jpg", self.player3))
# self.players.append(Player("Michi", "4", "player_images/michi-img.jpg", self.player4))
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
# if player removed from list -> remove set id
def removeID(self):
for index in range(self.unset_player_widget.count()):
item = self.unset_player_widget.item(index)
if item:
data = item.data(Qt.UserRole)
if data:
for player in self.players:
if player.number == data[0] and player.marker_num is not None:
self.set_ids.remove(player.marker_num)
player.marker_num = None
# add players to set_players for all players on "Field"
def setChangedListItems(self):
items = []
for index in range(self.set_player_widget.count()):
item = self.set_player_widget.item(index)
if item:
data = item.data(Qt.UserRole)
for player in self.players:
if player.number == data[0]:
items.append(player)
self.set_players = items
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -10.0)
self._draw_background()
glPopMatrix()
# handle glyphs
image = self._handle_aruco(image)
glutSwapBuffers()
def calc_values(self):
path = 'calib_images/*.jpg'
self.ret, self.mtx, self.dist, self.rvecs, self.tvecs = Tracker.calculate_camera_values(path)
def _handle_aruco(self, image):
img = image
corners, ids, _ = Tracker.preprocess(img)
if np.all(ids is not None):
# check for OpenCV output in different versions
params = aruco.estimatePoseSingleMarkers(corners, 1, self.mtx, self.dist)
if len(params) == 2:
rvec, tvec = params
else:
rvec, tvec, _ = params
else:
return
# set all players to list from "Field"
self.setChangedListItems()
for i in range(len(ids)):
rvecs, tvecs, glyph_name = rvec[i], tvec[i], ids[i][0]
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0], rmtx[0][1], rmtx[0][2], tvecs[0][0]],
[rmtx[1][0], rmtx[1][1], rmtx[1][2], tvecs[0][1]],
[rmtx[2][0], rmtx[2][1], rmtx[2][2], tvecs[0][2]],
[0.0, 0.0, 0.0, 1.0]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw shape
glPushMatrix()
glLoadMatrixd(view_matrix)
# check if ID is set or not and set it
if ids[i] not in self.set_ids:
for player in self.set_players:
if player.marker_num is None:
player.marker_num = ids[i]
self.set_ids.append(ids[i])
break
# if ID is set project model
for player in self.set_players:
if player.marker_num == ids[i]:
glCallList(player.model.gl_list)
glPopMatrix()
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
class OpenGLGlyphs:
# constants
INVERSE_MATRIX = np.array([[1.0, 1.0, 1.0, 1.0], [-1.0, -1.0, -1.0, -1.0],
[-1.0, -1.0, -1.0, -1.0], [1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
# textures
self.texture_background = None
self.texture_cube = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# enable textures
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
self.texture_cube = glGenTextures(1)
# create cube texture
image = Image.open("sample2.jpg")
ix = image.size[0]
iy = image.size[1]
image = image.tostring("raw", "RGBX", 0, -1)
glBindTexture(GL_TEXTURE_2D, self.texture_cube)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
image)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tostring("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -10.0)
self._draw_background()
glPopMatrix()
# handle glyph
image = self._handle_glyph(image)
glutSwapBuffers()
def _handle_glyph(self, image):
# attempt to detect glyph
rvecs = None
tvecs = None
try:
rvecs, tvecs = detect_glyph(image)
except Exception as ex:
print(ex)
if rvecs == None or tvecs == None:
return
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0], rmtx[0][1], rmtx[0][2], tvecs[0]],
[rmtx[1][0], rmtx[1][1], rmtx[1][2], tvecs[1]],
[rmtx[2][0], rmtx[2][1], rmtx[2][2], tvecs[2]],
[0.0, 0.0, 0.0, 1.0]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glBindTexture(GL_TEXTURE_2D, self.texture_cube)
glPushMatrix()
glLoadMatrixd(view_matrix)
self._draw_cube()
glPopMatrix()
def _draw_cube(self):
# draw cube
glBegin(GL_QUADS)
glTexCoord2f(0.0, 0.0)
glVertex3f(0.0, 0.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 0.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, 0.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(0.0, 1.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(0.0, 0.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(0.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, 0.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(0.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(0.0, 1.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 1.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(0.0, 0.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 0.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, 0.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(0.0, 0.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 0.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, 0.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(0.0, 0.0, -1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(0.0, 0.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(0.0, 1.0, 0.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(0.0, 1.0, -1.0)
glEnd()
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("OpenGL Glyphs")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class OpenGLGlyphs:
INVERSE_MATRIX = np.array([[1.0, 1.0, 1.0, 1.0], [-1.0, -1.0, -1.0, -1.0],
[-1.0, -1.0, -1.0, -1.0], [1.0, 1.0, 1.0, 1.0]])
def __init__(self):
self.webcam = Webcam()
self.webcam.start()
self.glyphs = Glyphs()
self.cone = None
self.sphere = None
self.texture_background = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
self.cone = OBJ('cone.obj')
self.sphere = OBJ('sphere.obj')
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
image = self.webcam.get_current_frame()
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tostring("raw", "BGRX", 0, -1)
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
bg_image)
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -10.0)
self._draw_background()
glPopMatrix()
image = self._handle_glyphs(image)
glutSwapBuffers()
def _handle_glyphs(self, image):
glyphs = []
try:
glyphs = self.glyphs.detect(image)
except Exception as ex:
print(ex)
if not glyphs:
return
for glyph in glyphs:
rvecs, tvecs, glyph_name = glyph
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array(
[[rmtx[0][0], rmtx[0][1], rmtx[0][2], tvecs[0]],
[rmtx[1][0], rmtx[1][1], rmtx[1][2], tvecs[1]],
[rmtx[2][0], rmtx[2][1], rmtx[2][2], tvecs[2]],
[0.0, 0.0, 0.0, 1.0]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
glPushMatrix()
glLoadMatrixd(view_matrix)
print(view_matrix)
if glyph_name == SHAPE_CONE:
glCallList(self.cone.gl_list)
elif glyph_name == SHAPE_SPHERE:
glCallList(self.sphere.gl_list)
glPopMatrix()
def _draw_background(self):
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
def main(self):
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("nichi Technologies")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class SaltwashAR:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise config
self.config_provider = ConfigProvider()
# initialise robots
self.rocky_robot = RockyRobot()
self.sporty_robot = SportyRobot()
# initialise webcam
self.webcam = Webcam()
# initialise glyphs
self.glyphs = Glyphs()
self.glyphs_cache = None
# initialise browser
self.browser = None
if self.config_provider.browser:
self.browser = Browser()
# initialise texture
self.texture_background = None
def _init_gl(self):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# load robots frames
self.rocky_robot.load_frames(self.config_provider.animation)
self.sporty_robot.load_frames(self.config_provider.animation)
# start threads
self.webcam.start()
if self.browser:
self.browser.start()
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# reset robots
self.rocky_robot.is_detected = False
self.sporty_robot.is_detected = False
# get image from webcam
image = self.webcam.get_current_frame()
# handle background
self._handle_background(image)
# handle glyphs
self._handle_glyphs(image)
# handle browser
self._handle_browser()
glutSwapBuffers()
def _handle_background(self, image):
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tostring("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
glPopMatrix()
def _handle_glyphs(self, image):
# attempt to detect glyphs
glyphs = []
try:
glyphs = self.glyphs.detect(image)
except Exception as ex:
print(ex)
# manage glyphs cache
if glyphs:
self.glyphs_cache = glyphs
elif self.glyphs_cache:
glyphs = self.glyphs_cache
self.glyphs_cache = None
else:
return
for glyph in glyphs:
rvecs, tvecs, _, glyph_name = glyph
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glPushMatrix()
glLoadMatrixd(view_matrix)
if glyph_name == ROCKY_ROBOT:
self.rocky_robot.is_detected = True
if self.browser and self.browser.is_speaking:
self.rocky_robot.next_frame(True)
else:
self.rocky_robot.next_frame(False)
elif glyph_name == SPORTY_ROBOT:
self.sporty_robot.is_detected = True
if self.browser and self.browser.is_speaking:
self.sporty_robot.next_frame(True)
else:
self.sporty_robot.next_frame(False)
glColor3f(1.0, 1.0, 1.0)
glPopMatrix()
def _handle_browser(self):
# check browser instantiated
if not self.browser: return
# handle browser
if self.rocky_robot.is_detected:
self.browser.load(ROCK)
elif self.sporty_robot.is_detected:
self.browser.load(SPORT)
else:
self.browser.halt()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("SaltwashAR")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl()
glutMainLoop()
class SaltwashAR:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise config
self.config_provider = ConfigProvider()
# initialise robots
self.rocky_robot = RockyRobot()
self.sporty_robot = SportyRobot()
# initialise webcam
self.webcam = Webcam()
# initialise markers
self.markers = Markers()
self.markers_cache = None
# initialise features
self.features = Features(self.config_provider)
# initialise texture
self.texture_background = None
def _init_gl(self):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# load robots frames
self.rocky_robot.load_frames(self.config_provider.animation)
self.sporty_robot.load_frames(self.config_provider.animation)
# start webcam thread
self.webcam.start()
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# reset robots
self.rocky_robot.reset()
self.sporty_robot.reset()
# get image from webcam
image = self.webcam.get_current_frame()
# handle background
self._handle_background(image.copy())
# handle markers
self._handle_markers(image.copy())
# handle features
self.features.handle(self.rocky_robot, self.sporty_robot, image.copy())
glutSwapBuffers()
def _handle_background(self, image):
# let features update background image
image = self.features.update_background_image(image)
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes('raw', 'BGRX', 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
glPopMatrix()
def _handle_markers(self, image):
# attempt to detect markers
markers = []
try:
markers = self.markers.detect(image)
except Exception as ex:
print(ex)
# manage markers cache
if markers:
self.markers_cache = markers
elif self.markers_cache:
markers = self.markers_cache
self.markers_cache = None
else:
return
for marker in markers:
rvecs, tvecs, marker_rotation, marker_name = marker
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glPushMatrix()
glLoadMatrixd(view_matrix)
if marker_name == ROCKY_ROBOT:
self.rocky_robot.next_frame(marker_rotation, self.features.is_speaking(), self.features.get_emotion())
elif marker_name == SPORTY_ROBOT:
self.sporty_robot.next_frame(marker_rotation, self.features.is_speaking(), self.features.get_emotion())
glColor3f(1.0, 1.0, 1.0)
glPopMatrix()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(100, 100)
self.window_id = glutCreateWindow('SaltwashAR')
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl()
glutMainLoop()
class HandTracker:
def __init__(self):
self.webcam = Webcam()
self.webcam.start()
self.detection = Detection()
self.x_axis = 0.0
self.z_axis = 0.0
self.show_cube = False
self.texture_background = None
self.texture_cube = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(45.0, float(Width) / float(Height), 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# enable texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
self.texture_cube = glGenTextures(1)
# create cube texture
image = Image.open("/home/annus/Pictures/image.jpeg")
ix = image.size[0]
iy = image.size[1]
image = image.tobytes("raw", "RGBX", 0, -1)
glBindTexture(GL_TEXTURE_2D, self.texture_cube)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
image)
def _draw_scene(self):
# handle any hand gesture
self._handle_gesture()
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -11.2)
self._draw_background()
glPopMatrix()
# draw cube if enabled
if self.show_cube:
glColor4f(1.0, 1.0, 1.0, 1.0)
glBlendFunc(GL_SRC_ALPHA, GL_ONE)
glEnable(GL_BLEND)
glDisable(GL_DEPTH_TEST)
glBindTexture(GL_TEXTURE_2D, self.texture_cube)
glPushMatrix()
glTranslatef(0.0, 0.0, -7.0)
glRotatef(self.x_axis, 1.0, 0.0, 0.0)
glRotatef(0.0, 0.0, 1.0, 0.0)
glRotatef(self.z_axis, 0.0, 0.0, 1.0)
self._draw_cube()
glPopMatrix()
glDisable(GL_BLEND)
glEnable(GL_DEPTH_TEST)
# update rotation values
self.x_axis = self.x_axis - 10
self.z_axis = self.z_axis - 10
glutSwapBuffers()
def _handle_gesture(self):
# get image from webcam
image = self.webcam.get_current_frame()
# detect hand gesture in image
is_okay = self.detection.is_item_detected_in_image(
'haarcascade_okaygesture.xml', image.copy())
is_vicky = self.detection.is_item_detected_in_image(
'haarcascade_vickygesture.xml', image.copy())
if is_okay:
# okay gesture shows cube
self.show_cube = True
elif is_vicky:
# vicky gesture hides cube
self.show_cube = False
# convert image to OpenGL texture format
image = cv2.flip(image, 0)
gl_image = Image.fromarray(image)
ix = gl_image.size[0]
iy = gl_image.size[1]
gl_image = gl_image.tostring("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
gl_image)
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 4.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 4.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 4.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 4.0)
glEnd()
def _draw_cube(self):
# draw cube
glBegin(GL_QUADS)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glEnd()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
glutCreateWindow("OpenGL Hand Tracker")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class ArkwoodAR:
def __init__(self):
# initialise webcams
self.webcam_one = Webcam(0)
self.webcam_two = Webcam(1)
# initialise config
self.config_provider = ConfigProvider()
# initialise features
self.features = Features(self.config_provider)
# initialise texture
self.texture_background = None
def _init_gl(self):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# start webcam threads
self.webcam_one.start()
self.webcam_two.start()
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
window_half_width = glutGet(GLUT_WINDOW_WIDTH) / 2
window_height = glutGet(GLUT_WINDOW_HEIGHT)
# get image from webcams
image_one = self.webcam_one.get_current_frame()
image_two = self.webcam_two.get_current_frame()
# detect feature in images
detection = self.features.detect(image_one, image_two)
# render first image
glViewport(0, 0, window_half_width, window_height)
if detection:
image_one = self.features.render(image_one, detection[0])
self._handle_background(image_one)
# render second image
glViewport(window_half_width, 0, window_half_width, window_height)
if detection:
image_two = self.features.render(image_two, detection[1])
self._handle_background(image_two)
# swap buffers
glutSwapBuffers()
# handle background
def _handle_background(self, image):
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes('raw', 'BGRX', 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -10.0)
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
glPopMatrix()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 360)
glutInitWindowPosition(100, 100)
glutCreateWindow('ArkwoodAR')
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl()
glutMainLoop()
class SaltwashAR:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise config
self.config_provider = ConfigProvider()
# initialise robots
self.rocky_robot = RockyRobot()
self.sporty_robot = SportyRobot()
# initialise webcam
self.webcam = Webcam()
# initialise markers
self.markers = Markers()
self.markers_cache = None
# initialise features
self.features = Features(self.config_provider)
# initialise texture
self.texture_background = None
def _init_gl(self):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# load robots frames
self.rocky_robot.load_frames(self.config_provider.animation)
self.sporty_robot.load_frames(self.config_provider.animation)
# start webcam thread
self.webcam.start()
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# reset robots
self.rocky_robot.reset()
self.sporty_robot.reset()
# get image from webcam
image = self.webcam.get_current_frame()
# handle background
self._handle_background(image.copy())
# handle markers
self._handle_markers(image.copy())
# handle features
self.features.handle(self.rocky_robot, self.sporty_robot, image.copy())
glutSwapBuffers()
def _handle_background(self, image):
# let features update background image
image = self.features.update_background_image(image)
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes('raw', 'BGRX', 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
glPopMatrix()
def _handle_markers(self, image):
# attempt to detect markers
markers = []
try:
markers = self.markers.detect(image)
except Exception as ex:
print(ex)
# manage markers cache
if markers:
self.markers_cache = markers
elif self.markers_cache:
markers = self.markers_cache
self.markers_cache = None
else:
return
for marker in markers:
rvecs, tvecs, marker_rotation, marker_name = marker
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glPushMatrix()
glLoadMatrixd(view_matrix)
if marker_name == ROCKY_ROBOT:
self.rocky_robot.next_frame(marker_rotation, self.features.is_speaking(), self.features.get_emotion())
elif marker_name == SPORTY_ROBOT:
self.sporty_robot.next_frame(marker_rotation, self.features.is_speaking(), self.features.get_emotion())
glColor3f(1.0, 1.0, 1.0)
glPopMatrix()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(100, 100)
glutCreateWindow('SaltwashAR')
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl()
glutMainLoop()
class OpenGLGlyphs:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
# initialise glyphs
self.glyphs = Glyphs()
# initialise shapes
self.cone = None
self.sphere = None
# initialise texture
self.texture_background = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# assign shapes
self.cone = OBJ('cone.obj')
self.sphere = OBJ('sphere.obj')
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
self._draw_background()
glPopMatrix()
# handle glyphs
image = self._handle_glyphs(image)
glutSwapBuffers()
def _handle_glyphs(self, image):
# attempt to detect glyphs
glyphs = []
try:
glyphs = self.glyphs.detect(image)
except Exception as ex:
print(ex)
if not glyphs:
return
for glyph in glyphs:
rvecs, tvecs, glyph_name = glyph
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw shape
glPushMatrix()
glLoadMatrixd(view_matrix)
if glyph_name == SHAPE_CONE:
glCallList(self.cone.gl_list)
elif glyph_name == SHAPE_SPHERE:
glCallList(self.sphere.gl_list)
glPopMatrix()
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("OpenGL Glyphs")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class OpenGLGlyphs:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
# initialise glyphs
self.glyphs = Glyphs()
# initialise shapes
self.cone = None
self.sphere = None
# self.hp = HP()
# initialise texture
self.texture_background = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# assign shapes
self.cone = OBJ('texturedCube.obj')
self.sphere = OBJ('./sortedOut/hair.obj')
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture formatz
bg_image = cv2.flip(image, 0)
bg_image = np.asarray(bg_image)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
self._draw_background()
glPopMatrix()
# handle glyphs
image = self._handle_headpose(image)
glutSwapBuffers()
def _handle_headpose(self, image):
# attempt to detect glyphs
glyphs = []
# glyphs = self.hp.main(image)
try:
glyphs = self.glyphs.detect(image)
except Exception as ex:
print(ex)
if not glyphs:
return
for glyph in glyphs:
rvecs, tvecs, glyph_name = glyph
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw shape
glPushMatrix()
glLoadMatrixd(view_matrix)
if glyph_name == SHAPE_CONE:
glCallList(self.cone.gl_list)
elif glyph_name == SHAPE_SPHERE:
glCallList(self.sphere.gl_list)
glPopMatrix()
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow(b"OpenGL Glyphs")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class OpenGLGlyphs:
##############################################################초기화
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
self.find = fp()
self.find.set_img('book.jpg')
self.hei, self.wid = self.webcam.get_frame_shape()[:2]
# initialise cube
self.d_obj = None
self.img = None
# initialise texture
self.texture_background = None
self.K = None
self.mark_kp = None
self.mark_des = None
self.set_keypoint()
self.new_kp = None
self.mat_kp = None
self.mat_des = None
self.H = None
# self.Rt=None
##############################################################카메라 세팅
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
self.K = my_calibration((Height, Width))
fx = self.K[0, 0]
fy = self.K[1, 1]
fovy = 2 * np.arctan(0.5 * Height / fy) * 180 / np.pi
aspect = (float)(Width * fy) / (Height * fx)
# define the near and far clipping planes
near = 0.1
far = 100.0
# set perspective
gluPerspective(fovy, aspect, near, far)
glMatrixMode(GL_MODELVIEW)
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
##############################################################marker의 kp, des저장
def set_keypoint(self):
self.find.start()
self.mark_kp, self.mark_des = self.find.get_point()
##############################################################K값 구하기
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
Rt = self._my_cal(image)
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, self.wid, self.hei, 0, GL_RGBA,
GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
# glTranslatef(0.0,0.0,0.0)
gluLookAt(0.0, 0.0, 12, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0)
self._draw_background()
glPopMatrix()
################Rt를 구해서 매칭되는 이미지가 있는지 판단
if Rt is not None:
self._set_modelview_from_camera(Rt)
glEnable(GL_LIGHTING)
glEnable(GL_LIGHT0)
glEnable(GL_DEPTH_TEST)
glEnable(GL_NORMALIZE)
glClear(GL_DEPTH_BUFFER_BIT)
ObjLoader("jnu.obj").render_scene()
glutSwapBuffers()
##############################################################OpenGL용 Rt변환
def _set_modelview_from_camera(self, Rt):
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
Rx = np.array([[0.2, 0, 0], [0, 0, 0.2], [0, 0.2, 0]])
# set rotation to best approximation
R = Rt[:, :3]
# change sign of x-axis
R[0, :] = -R[0, :]
# set translation
t = Rt[:, 3]
t[0] = -t[0]
# setup 4*4 model view matrix
M = np.eye(4)
M[:3, :3] = np.dot(R, Rx)
M[:3, 3] = t
M[3, :3] = t
# transpose and flatten to get column order
M = M.T
m = M.flatten()
# replace model view with the new matrix
glLoadMatrixf(m)
##############################################################Rt반환
def _my_cal(self, image):
find_H = fp()
find_H.set_cv_img(image)
find_H.start()
kp, des = find_H.get_point()
self.H = self.match_images(self.mark_kp, self.mark_des, kp, des)
if self.H is not None:
cam1 = camera.Camera(
np.hstack((self.K, np.dot(self.K, np.array([[0], [0],
[-1]])))))
# Rt1=dot(linalg.inv(self.K),cam1.P)
cam2 = camera.Camera(np.dot(self.H, cam1.P))
A = np.dot(np.linalg.inv(self.K), cam2.P[:, :3])
A = np.array([A[:, 0], A[:, 1], np.cross(A[:, 0], A[:, 1])]).T
cam2.P[:, :3] = np.dot(self.K, A)
Rt = np.dot(np.linalg.inv(self.K), cam2.P)
return Rt
else:
return None
##############################################################match image
def match_images(self, kp1, des1, kp2, des2):
matcher = cv2.BFMatcher()
match_des = matcher.knnMatch(des1, des2, k=2)
matches = []
matA, matB = [], []
matC = []
for m in match_des:
if m[0].distance < 0.8 * m[1].distance:
matA.append(kp1[m[0].queryIdx])
matB.append(kp2[m[0].trainIdx])
matC.append(des1[m[0].queryIdx])
if len(matA) > 50:
ptsA = np.float32([m.pt for m in matA])
ptsB = np.float32([n.pt for n in matB])
H1 = []
H1, status = cv2.findHomography(ptsA, ptsB, cv2.RANSAC, 5.0)
H1 = self.homo_check(H1)
self.mat_kp = np.array(
[matB[i] for i in range(status.shape[0]) if status[i] == 1])
self.mat_des = np.array(
[matC[i] for i in range(status.shape[0]) if status[i] == 1])
return H1
else:
return None
##############################################################homography check
def homo_check(self, H1):
if self.H is None:
return H1
else:
if cv2.norm(H1, self.H) > 1.0:
return H1
else:
return self.H
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
glDeleteTextures(1)
def keyboard(self, *args):
if args[0] == GLUT_KEY_UP:
glutDestroyWindow(self.window_id)
self.webcam.finish()
sys.exit()
##############################################################OpenGL창 초기
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(self.wid, self.hei)
glutInitWindowPosition(200, 200)
self.window_id = glutCreateWindow(b"OpenGL Glyphs")
self._init_gl(self.wid, self.hei)
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
glutMainLoop()
import cv2
from glyphfunctions import *
from glyphdatabase import *
from webcam import Webcam
webcam = Webcam()
webcam.start()
QUADRILATERAL_POINTS = 4
BLACK_THRESHOLD = 100
WHITE_THRESHOLD = 155
while True:
# Stage 1: Read an image from our webcam
image = webcam.get_current_frame()
# Stage 2: Detect edges in image
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (5, 5), 0)
edges = cv2.Canny(gray, 100, 200)
# Stage 3: Find contours
contours, _ = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cv2.imshow('edges',edges)
# contours = sorted(contours, key=cv2.contourArea, reverse=True)[:10]
#
# for contour in contours:
#
# # Stage 4: Shape check
# perimeter = cv2.arcLength(contour, True)
class OpenGLGlyphs:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
#initialise
self.hBox = causalBox(winSize = 10)
self.vBox = causalBox(winSize = 10)
plyerName = ["John","Doe","Tommy","Emmanuel"]
self.game = GameCtrler(plyerName)
# initialise shapes
self.dragon = None
self.fly = None
self.ele = None
self.boat = None
self.horse = None
self.house = None
self.juk = None
# textures
self.texture_background = None
def _init_gl(self, Width, Height):
# initialPosition = (0,0,0)
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
# Projection matrix
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
fovy = 2*np.arctan(Height/1375.0)*180.0/np.pi
gluPerspective(fovy, float(Width)/float(Height), 0.1, 1375.1)
glViewport(0,0,Width,Height)
glMatrixMode(GL_MODELVIEW)
# assign shapes
print "loading model 1/7"
self.dragon = OBJ('Drgn9-6.obj')
print "loading model 2/7"
self.fly = OBJ('plane.obj')
print "loading model 3/7"
self.ele = OBJ('minion.obj')
print "loading model 4/7"
self.boat = OBJ('VikingShip.mtl.obj')
print "loading model 5/7"
self.horse = OBJ('Wooden_Toy_Truck.obj')
print "loading model 6/7"
self.house = OBJ('house_001.obj')
print "loading model 7/7"
self.juk = OBJ('Barrel_variation.obj')
print "loading model done"
# enable textures
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
self._draw_background(image)
# handle glyphs
image = self._handle_glyphs(image)
glutSwapBuffers()
def _handle_glyphs(self, image):
# attempt to detect glyphs
glyphs = []
try:
glyphs = detect_glyph(image, self.hBox, self.vBox, self.game)
except Exception as ex:
print(ex)
if not glyphs:
return
for glyph in glyphs:
rvecs, tvecs, glyph_name = glyph
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glPushMatrix()
glLoadIdentity()
glLoadMatrixd(view_matrix)
if glyph_name == "B juk":
glCallList(self.juk.gl_list)
elif glyph_name == "R juk":
glCallList(self.juk.gl_list)
elif glyph_name == "B Phao":
glCallList(self.house.gl_list)
elif glyph_name == "R Phao":
glCallList(self.house.gl_list)
elif glyph_name == "B Horse":
glCallList(self.horse.gl_list)
elif glyph_name == "R Horse":
glCallList(self.horse.gl_list)
elif glyph_name == "B Boat":
glCallList(self.boat.gl_list)
elif glyph_name == "R Boat":
glCallList(self.boat.gl_list)
elif glyph_name == "B Ele":
glCallList(self.ele.gl_list)
elif glyph_name == "R Ele":
glCallList(self.ele.gl_list)
elif glyph_name == "B Fly":
glCallList(self.fly.gl_list)
elif glyph_name == "R Fly":
glCallList(self.fly.gl_list)
elif glyph_name == "B T":
glCallList(self.dragon.gl_list)
elif glyph_name == "R T":
glCallList(self.dragon.gl_list)
else:
glCallList(self.dragon.gl_list)
glPopMatrix()
def _draw_background(self, image):
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glLoadIdentity()
glTranslatef(-100,100,-1375)
glBegin(GL_QUADS)
i, j = 1520/2, 820/2
glTexCoord2f(0.0, 1.0); glVertex3f(-i, -j, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( i, -j, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( i, j, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-i, j, 0.0)
glEnd()
glPopMatrix()
def main(self):
width = 1520
heigh = 820
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(760, 410)
glutInitWindowPosition(100, 100)
self.window_id = glutCreateWindow("OpenGL Glyphs")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(width, heigh)
glutMainLoop()
class OpenGLGlyphs:
##############################################################초기화
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
self.find = fp()
self.find.set_img('sample.jpg')
self.hei, self.wid = self.webcam.get_frame_shape()[:2]
# initialise cube
# self.d_obj = None
self.img = None
# initialise texture
self.texture_background = None
self.K = None
self.mark_kp = None
self.mark_des = None
self.set_keypoint()
self.new_kp = None
self.mat_kp = None
self.mat_des = None
self.H = None
# self.Rt=None
##############################################################카메라 세팅
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0) # 투명도 결정
glClearDepth(1.0) # 깊이 버퍼의 모든 픽셀에 설정될 초기값 지정
glDepthFunc(GL_LESS) # 언제나 새로 들어오는 값이 기준 GL_LESS를 설정했다고 하자. 이 경우에는 새로 들어온 값이
# 이미 저장되어 있는 값 보다 적을 경우에 depth buffer의 값을 새로 들어온 값으로 갱신하겠다,
glEnable(GL_DEPTH_TEST) # 요건 깊이 정보에 따라 이미지를 순서대로 나줌.
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
self.K = my_calibration((Height, Width))
fx = self.K[0, 0]
fy = self.K[1, 1]
fovy = 2 * arctan(0.5 * Height / fy) * 180 / pi
aspect = (float)(Width * fy) / (Height * fx)
# define the near and far clipping planes
near = 0.1
far = 100.0
# set perspective
gluPerspective(fovy, aspect, near, far)
glMatrixMode(GL_MODELVIEW)
# self.d_obj=[OBJ('Rocket.obj')]
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
# gluPerspective(33.7, 1.3, 0.1, 100.0)
##############################################################marker의 kp, des저장
def set_keypoint(self):
self.find.start()
self.mark_kp, self.mark_des = self.find.get_point()
##############################################################K값 구하기
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
Rt = self._my_cal(image)
"""
if Rt!=None:
box=ones((self.hei,self.wid),uint8)
H_box=cv2.warpPerspective(box,self.H,(self.wid, self.hei))
image=image*H_box[:,:,newaxis]
image=cv2.drawKeypoints(image,self.mat_kp,flags=cv2.DRAW_MATCHES_FLAGS_DEFAULT)
"""
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, self.wid, self.hei, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
# glTranslatef(0.0,0.0,0.0)
gluLookAt(0.0, 0.0, 12.5, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0)
self._draw_background()
glPopMatrix()
################Rt를 구해서 매칭되는 이미지가 있는지 판단
if Rt is not None:
self._set_modelview_from_camera(Rt)
glEnable(GL_LIGHTING)
glEnable(GL_LIGHT0)
glEnable(GL_DEPTH_TEST)
glEnable(GL_NORMALIZE)
glClear(GL_DEPTH_BUFFER_BIT)
glMaterialfv(GL_FRONT, GL_AMBIENT, [0.5, 0.5, 0.0, 1.0])
glMaterialfv(GL_FRONT, GL_DIFFUSE, [0.9, 0.9, 0.0, 1.0])
glMaterialfv(GL_FRONT, GL_SPECULAR, [1.0, 1.0, 1.0, 1.0])
glMaterialfv(GL_FRONT, GL_SHININESS, 0.25 * 128.0)
glutSolidTeapot(0.1)
glutSwapBuffers()
##############################################################OpenGL용 Rt변환
def _set_modelview_from_camera(self, Rt):
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
Rx = array([[1, 0, 0], [0, 0, 1], [0, 1, 0]])
# set rotation to best approximation
R = Rt[:, :3]
# change sign of x-axis
R[0, :] = -R[0, :]
# set translation
t = Rt[:, 3]
t[0] = -t[0]
# setup 4*4 model view matrix
M = eye(4)
M[:3, :3] = dot(R, Rx)
M[:3, 3] = t
M[3, :3] = t
# transpose and flatten to get column order
M = M.T
m = M.flatten()
# replace model view with the new matrix
glLoadMatrixf(m)
##############################################################Rt반환
def _my_cal(self, image):
find_H = fp()
find_H.set_cv_img(image)
find_H.start()
kp, des = find_H.get_point()
self.H = self.match_images(self.mark_kp, self.mark_des, kp, des)
if self.H is not None:
cam1 = camera.Camera(hstack((self.K, dot(self.K, array([[0], [0], [-1]])))))
# Rt1=dot(linalg.inv(self.K),cam1.P)
cam2 = camera.Camera(dot(self.H, cam1.P))
A = dot(linalg.inv(self.K), cam2.P[:, :3])
A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T
cam2.P[:, :3] = dot(self.K, A)
Rt = dot(linalg.inv(self.K), cam2.P)
return Rt
else:
return None
##############################################################match image
def match_images(self, kp1, des1, kp2, des2):
matcher = cv2.BFMatcher()
match_des = matcher.knnMatch(des1, des2, k=2)
matches = []
matA, matB = [], []
matC = []
for m in match_des:
if m[0].distance < 0.8 * m[1].distance:
matA.append(kp1[m[0].queryIdx])
matB.append(kp2[m[0].trainIdx])
matC.append(des1[m[0].queryIdx])
if len(matA) > 50:
ptsA = float32([m.pt for m in matA])
ptsB = float32([n.pt for n in matB])
H1 = []
H1, status = cv2.findHomography(ptsA, ptsB, cv2.RANSAC, 5.0)
H1 = self.homo_check(H1)
self.mat_kp = array([matB[i] for i in range(status.shape[0]) if status[i] == 1])
self.mat_des = array([matC[i] for i in range(status.shape[0]) if status[i] == 1])
return H1
else:
return None
##############################################################homography check
def homo_check(self, H1):
if self.H is None:
return H1
else:
if cv2.norm(H1, self.H) > 1.0:
return H1
else:
return self.H
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0);
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0);
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0);
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0);
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
glDeleteTextures(1)
def keyboard(self, *args):
if args[0] is GLUT_KEY_UP:
glutDestroyWindow(self.window_id)
self.webcam.finish()
sys.exit()
##############################################################OpenGL창 초기
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(self.wid, self.hei)
glutInitWindowPosition(400, 400)
self.window_id = glutCreateWindow(b"OpenGL Glyphs")
self._init_gl(self.wid, self.hei)
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
glutSpecialFunc(self.keyboard)
glutMainLoop()
class HandTracker:
def __init__(self):
self.webcam = Webcam()
self.webcam.start()
self.detection = Detection()
self.x_axis = 0.0
self.y_axis = 0.0
self.z_axis = 0.0
self.z_pos = -7.0
def _handle_gesture(self):
# get image from webcam
image = self.webcam.get_current_frame()
# detect hand gesture in image
is_okay = self.detection.is_item_detected_in_image(
'haarcascade_okaygesture.xml', image.copy())
is_vicky = self.detection.is_item_detected_in_image(
'haarcascade_vickygesture.xml', image.copy())
if is_okay:
# okay gesture moves cube towards us
self.z_pos = self.z_pos + 1.0
elif is_vicky:
# vicky gesture moves cube away from us
self.z_pos = self.z_pos - 1.0
def _draw_cube(self):
# draw cube
glBegin(GL_QUADS)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glEnd()
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(45.0, float(Width) / float(Height), 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# initialize lighting
glLightfv(GL_LIGHT0, GL_AMBIENT, (0.5, 0.5, 0.5, 1.0))
glLightfv(GL_LIGHT0, GL_DIFFUSE, (1.0, 0.8, 0.0, 1.0))
glEnable(GL_LIGHT0)
glEnable(GL_LIGHTING)
# initialize blending
glColor4f(0.2, 0.2, 0.2, 0.5)
glBlendFunc(GL_SRC_ALPHA, GL_ONE)
glEnable(GL_BLEND)
# initialize texture
image = open("devil.jpg")
ix = image.size[0]
iy = image.size[1]
image = image.tostring("raw", "RGBX", 0, -1)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
image)
glEnable(GL_TEXTURE_2D)
def _draw_scene(self):
# handle any hand gesture
self._handle_gesture()
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# position and rotate cube
glTranslatef(0.0, 0.0, self.z_pos)
glRotatef(self.x_axis, 1.0, 0.0, 0.0)
glRotatef(self.y_axis, 0.0, 1.0, 0.0)
glRotatef(self.z_axis, 0.0, 0.0, 1.0)
# position lighting
glLightfv(GL_LIGHT0, GL_POSITION, (0.0, 0.0, 2.0, 1.0))
# draw cube
self._draw_cube()
# update rotation values
self.x_axis = self.x_axis - 10
self.z_axis = self.z_axis - 10
glutSwapBuffers()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
glutCreateWindow("OpenGL Hand Tracker")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
#webcam.start()
def draw(img, corners, imgpts):
corner = tuple(corners[0].ravel())
img = cv2.line(img, corner, tuple(imgpts[0].ravel()), (255, 0, 0), 5)
img = cv2.line(img, corner, tuple(imgpts[1].ravel()), (0, 255, 0), 5)
img = cv2.line(img, corner, tuple(imgpts[2].ravel()), (0, 0, 255), 5)
return img
while True:
# get image from webcam
webcam.update_frame()
image = webcam.get_current_frame()
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
objp = np.zeros((3 * 5, 3), np.float32)
objp[:, :2] = np.mgrid[0:5, 0:3].T.reshape(-1, 2)
objpoints = [] # 3d point in real world space
imgpoints = []
axis = np.float32([[3, 0, 0], [0, 3, 0], [0, 0, -3]]).reshape(-1, 3)
# save image to file, if pattern found
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
ret, corners = cv2.findChessboardCorners(gray, (5, 3), None)
if True:
print(
"Found chess board-------------------------------------------------"
class AR_Project:
def __init__(self):
# sigint interrupt initialize
signal.signal(signal.SIGINT, self.signal_handler)
# initialize webcam
self.webcam = Webcam()
self.webcam.start()
self.x_axis = 0.0
self.y_axis = 0.0
self.z_axis = 0.0
self.z_pos = -7.0
self.win = 0
self.texture_background = None
self.texture_teapot = None
def signal_handler(self, signal, frame):
print('\nYou pressed Ctrl+C!')
self.webcam.close()
sys.exit()
def _get_background(self):
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture format
image = cv2.flip(image, 0)
image = cv2.flip(image, 1)
gl_image = Image.fromarray(image)
ix = gl_image.size[0]
iy = gl_image.size[1]
gl_image = gl_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, gl_image)
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 4.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 4.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 4.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 4.0)
glEnd()
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(45.0, float(Width)/float(Height)-.2, 0.1, 500.0)
glMatrixMode(GL_MODELVIEW)
# enable texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
# initialize lighting
#glLightfv(GL_LIGHT0, GL_AMBIENT, (0.5, 0.5, 0.5, 1.0))
#glLightfv(GL_LIGHT0, GL_DIFFUSE, (1.0, 0.8, 0.0, 1.0))
glEnable(GL_LIGHT0)
glEnable(GL_LIGHTING)
# initialize blending
glColor4f(0.2, 0.2, 0.2, 0.5)
glBlendFunc(GL_SRC_ALPHA, GL_ONE)
glEnable(GL_BLEND)
#glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
#glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
#glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, image)
#glEnable(GL_TEXTURE_2D)
def _draw_scene(self):
# handle any hand gesture
self._get_background()
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glLoadIdentity();
# draw background
#glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-11.2)
self._draw_background()
glPopMatrix()
# position teapot
glTranslatef(0.0,0.0,self.z_pos);
glRotatef(self.x_axis,1.0,0.0,0.0)
glRotatef(self.y_axis,0.0,1.0,0.0)
glRotatef(self.z_axis,0.0,0.0,1.0)
# draw teapot
glutSolidTeapot(1.2)
# rotate teapot
self.x_axis = self.x_axis - 2
self.z_axis = self.z_axis - 2
glutSwapBuffers()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.win = glutCreateWindow("COS 429 AR Project")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
