Esempio n. 1
0
            # Stage 6: Border check
            resized_shape = resize_image(topdown_quad, SHAPE_RESIZE)
            if resized_shape[5, 5] > BLACK_THRESHOLD: continue

            # Stage 7: Glyph pattern
            glyph_pattern = get_glyph_pattern(resized_shape, BLACK_THRESHOLD,
                                              WHITE_THRESHOLD)
            glyph_found, glyph_rotation, glyph_substitute = match_glyph_pattern(
                glyph_pattern)

            if glyph_found:

                # Stage 8: Substitute glyph
                substitute_image = cv2.imread('substitute.jpg')

                for _ in range(glyph_rotation):
                    substitute_image = cv2.imread(
                        '{}.jpg'.format(glyph_substitute))

                image = add_substitute_quad(image, substitute_image,
                                            approx.reshape(4, 2))

    # Stage 9: Show augmented reality
    cv2.imshow('2D Augmented Reality using Glyphs', image)
    if cv2.waitKey(100) & 0xFF == ord('q'):
        break

# When everything done, release the capture
cv2.destroyAllWindows()
webcam.finish()
Esempio n. 2
0
class OpenGLGlyphs:

    ##############################################################초기화
    def __init__(self):
        # initialise webcam and start thread
        self.webcam = Webcam()
        self.webcam.start()
        self.find = fp()
        self.find.set_img('sample.jpg')

        self.hei, self.wid = self.webcam.get_frame_shape()[:2]
        # initialise cube
        # self.d_obj = None
        self.img = None
        # initialise texture
        self.texture_background = None
        self.K = None
        self.mark_kp = None
        self.mark_des = None
        self.set_keypoint()
        self.new_kp = None

        self.mat_kp = None
        self.mat_des = None
        self.H = None

        # self.Rt=None

    ##############################################################카메라 세팅
    def _init_gl(self, Width, Height):
        glClearColor(0.0, 0.0, 0.0, 0.0) # 투명도 결정
        glClearDepth(1.0) # 깊이 버퍼의 모든 픽셀에 설정될 초기값 지정
        glDepthFunc(GL_LESS) # 언제나 새로 들어오는 값이 기준 GL_LESS를 설정했다고 하자. 이 경우에는 새로 들어온 값이
                                # 이미 저장되어 있는 값 보다 적을 경우에 depth buffer의 값을 새로 들어온 값으로 갱신하겠다,
        glEnable(GL_DEPTH_TEST) # 요건 깊이 정보에 따라 이미지를 순서대로 나줌.
        glShadeModel(GL_SMOOTH)
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()

        self.K = my_calibration((Height, Width))
        fx = self.K[0, 0]
        fy = self.K[1, 1]
        fovy = 2 * arctan(0.5 * Height / fy) * 180 / pi
        aspect = (float)(Width * fy) / (Height * fx)
        # define the near and far clipping planes
        near = 0.1
        far = 100.0
        # set perspective
        gluPerspective(fovy, aspect, near, far)

        glMatrixMode(GL_MODELVIEW)
        # self.d_obj=[OBJ('Rocket.obj')]
        glEnable(GL_TEXTURE_2D)
        self.texture_background = glGenTextures(1)
        # gluPerspective(33.7, 1.3, 0.1, 100.0)

    ##############################################################marker의 kp, des저장
    def set_keypoint(self):

        self.find.start()
        self.mark_kp, self.mark_des = self.find.get_point()

    ##############################################################K값 구하기
    def _draw_scene(self):
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
        glLoadIdentity()

        # get image from webcam
        image = self.webcam.get_current_frame()

        Rt = self._my_cal(image)
        """
        if Rt!=None:
            box=ones((self.hei,self.wid),uint8)
            H_box=cv2.warpPerspective(box,self.H,(self.wid, self.hei))
            image=image*H_box[:,:,newaxis]
            image=cv2.drawKeypoints(image,self.mat_kp,flags=cv2.DRAW_MATCHES_FLAGS_DEFAULT)
        """
        # convert image to OpenGL texture format
        bg_image = cv2.flip(image, 0)
        bg_image = Image.fromarray(bg_image)

        ix = bg_image.size[0]
        iy = bg_image.size[1]
        bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)

        # create background texture
        glBindTexture(GL_TEXTURE_2D, self.texture_background)
        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, self.wid, self.hei, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)

        # draw background
        glBindTexture(GL_TEXTURE_2D, self.texture_background)
        glPushMatrix()

        # glTranslatef(0.0,0.0,0.0)
        gluLookAt(0.0, 0.0, 12.5, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0)

        self._draw_background()
        glPopMatrix()
        ################Rt를 구해서 매칭되는 이미지가 있는지 판단

        if Rt is not None:
            self._set_modelview_from_camera(Rt)
            glEnable(GL_LIGHTING)
            glEnable(GL_LIGHT0)
            glEnable(GL_DEPTH_TEST)
            glEnable(GL_NORMALIZE)
            glClear(GL_DEPTH_BUFFER_BIT)
            glMaterialfv(GL_FRONT, GL_AMBIENT, [0.5, 0.5, 0.0, 1.0])
            glMaterialfv(GL_FRONT, GL_DIFFUSE, [0.9, 0.9, 0.0, 1.0])
            glMaterialfv(GL_FRONT, GL_SPECULAR, [1.0, 1.0, 1.0, 1.0])
            glMaterialfv(GL_FRONT, GL_SHININESS, 0.25 * 128.0)
            glutSolidTeapot(0.1)

        glutSwapBuffers()

    ##############################################################OpenGL용 Rt변환
    def _set_modelview_from_camera(self, Rt):

        glMatrixMode(GL_MODELVIEW)
        glLoadIdentity()
        Rx = array([[1, 0, 0], [0, 0, 1], [0, 1, 0]])

        # set rotation to best approximation
        R = Rt[:, :3]

        # change sign of x-axis
        R[0, :] = -R[0, :]
        # set translation
        t = Rt[:, 3]
        t[0] = -t[0]

        # setup 4*4 model view matrix
        M = eye(4)
        M[:3, :3] = dot(R, Rx)
        M[:3, 3] = t
        M[3, :3] = t

        # transpose and flatten to get column order
        M = M.T

        m = M.flatten()
        # replace model view with the new matrix
        glLoadMatrixf(m)

    ##############################################################Rt반환
    def _my_cal(self, image):
        find_H = fp()
        find_H.set_cv_img(image)
        find_H.start()
        kp, des = find_H.get_point()

        self.H = self.match_images(self.mark_kp, self.mark_des, kp, des)
        if self.H is not None:
            cam1 = camera.Camera(hstack((self.K, dot(self.K, array([[0], [0], [-1]])))))
            # Rt1=dot(linalg.inv(self.K),cam1.P)
            cam2 = camera.Camera(dot(self.H, cam1.P))

            A = dot(linalg.inv(self.K), cam2.P[:, :3])
            A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T
            cam2.P[:, :3] = dot(self.K, A)
            Rt = dot(linalg.inv(self.K), cam2.P)

            return Rt
        else:
            return None

    ##############################################################match image
    def match_images(self, kp1, des1, kp2, des2):
        matcher = cv2.BFMatcher()
        match_des = matcher.knnMatch(des1, des2, k=2)
        matches = []
        matA, matB = [], []
        matC = []

        for m in match_des:
            if m[0].distance < 0.8 * m[1].distance:
                matA.append(kp1[m[0].queryIdx])
                matB.append(kp2[m[0].trainIdx])
                matC.append(des1[m[0].queryIdx])

        if len(matA) > 50:
            ptsA = float32([m.pt for m in matA])
            ptsB = float32([n.pt for n in matB])
            H1 = []
            H1, status = cv2.findHomography(ptsA, ptsB, cv2.RANSAC, 5.0)
            H1 = self.homo_check(H1)
            self.mat_kp = array([matB[i] for i in range(status.shape[0]) if status[i] == 1])
            self.mat_des = array([matC[i] for i in range(status.shape[0]) if status[i] == 1])

            return H1
        else:
            return None

    ##############################################################homography check
    def homo_check(self, H1):
        if self.H is None:
            return H1
        else:
            if cv2.norm(H1, self.H) > 1.0:
                return H1
            else:
                return self.H

    def _draw_background(self):
        # draw background
        glBegin(GL_QUADS)
        glTexCoord2f(0.0, 1.0);
        glVertex3f(-4.0, -3.0, 0.0)
        glTexCoord2f(1.0, 1.0);
        glVertex3f(4.0, -3.0, 0.0)
        glTexCoord2f(1.0, 0.0);
        glVertex3f(4.0, 3.0, 0.0)
        glTexCoord2f(0.0, 0.0);
        glVertex3f(-4.0, 3.0, 0.0)
        glEnd()
        glDeleteTextures(1)

    def keyboard(self, *args):
        if args[0] is GLUT_KEY_UP:
            glutDestroyWindow(self.window_id)
            self.webcam.finish()
            sys.exit()

    ##############################################################OpenGL창 초기

    def main(self):
        # setup and run OpenGL
        glutInit()
        glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
        glutInitWindowSize(self.wid, self.hei)
        glutInitWindowPosition(400, 400)
        self.window_id = glutCreateWindow(b"OpenGL Glyphs")
        self._init_gl(self.wid, self.hei)
        glutDisplayFunc(self._draw_scene)
        glutIdleFunc(self._draw_scene)
        glutSpecialFunc(self.keyboard)
        glutMainLoop()
Esempio n. 3
0
class OpenGLGlyphs:
    ##############################################################초기화
    def __init__(self):
        # initialise webcam and start thread
        self.webcam = Webcam()
        self.webcam.start()
        self.find = fp()
        self.find.set_img('book.jpg')

        self.hei, self.wid = self.webcam.get_frame_shape()[:2]
        # initialise cube
        self.d_obj = None
        self.img = None
        # initialise texture
        self.texture_background = None
        self.K = None
        self.mark_kp = None
        self.mark_des = None
        self.set_keypoint()
        self.new_kp = None

        self.mat_kp = None
        self.mat_des = None
        self.H = None

        # self.Rt=None

    ##############################################################카메라 세팅
    def _init_gl(self, Width, Height):
        glClearColor(0.0, 0.0, 0.0, 0.0)
        glClearDepth(1.0)
        glDepthFunc(GL_LESS)
        glEnable(GL_DEPTH_TEST)
        glShadeModel(GL_SMOOTH)
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()

        self.K = my_calibration((Height, Width))
        fx = self.K[0, 0]
        fy = self.K[1, 1]
        fovy = 2 * np.arctan(0.5 * Height / fy) * 180 / np.pi
        aspect = (float)(Width * fy) / (Height * fx)
        # define the near and far clipping planes
        near = 0.1
        far = 100.0
        # set perspective
        gluPerspective(fovy, aspect, near, far)

        glMatrixMode(GL_MODELVIEW)
        glEnable(GL_TEXTURE_2D)
        self.texture_background = glGenTextures(1)

    ##############################################################marker의 kp, des저장
    def set_keypoint(self):

        self.find.start()
        self.mark_kp, self.mark_des = self.find.get_point()

    ##############################################################K값 구하기
    def _draw_scene(self):
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
        glLoadIdentity()

        # get image from webcam
        image = self.webcam.get_current_frame()

        Rt = self._my_cal(image)

        # convert image to OpenGL texture format
        bg_image = cv2.flip(image, 0)
        bg_image = Image.fromarray(bg_image)

        ix = bg_image.size[0]
        iy = bg_image.size[1]
        bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)

        # create background texture
        glBindTexture(GL_TEXTURE_2D, self.texture_background)
        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
        glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, self.wid, self.hei, 0, GL_RGBA,
                     GL_UNSIGNED_BYTE, bg_image)

        # draw background
        glBindTexture(GL_TEXTURE_2D, self.texture_background)
        glPushMatrix()

        # glTranslatef(0.0,0.0,0.0)
        gluLookAt(0.0, 0.0, 12, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0)

        self._draw_background()
        glPopMatrix()
        ################Rt를 구해서 매칭되는 이미지가 있는지 판단

        if Rt is not None:
            self._set_modelview_from_camera(Rt)
            glEnable(GL_LIGHTING)
            glEnable(GL_LIGHT0)
            glEnable(GL_DEPTH_TEST)
            glEnable(GL_NORMALIZE)
            glClear(GL_DEPTH_BUFFER_BIT)
            ObjLoader("jnu.obj").render_scene()

        glutSwapBuffers()

    ##############################################################OpenGL용 Rt변환
    def _set_modelview_from_camera(self, Rt):

        glMatrixMode(GL_MODELVIEW)
        glLoadIdentity()
        Rx = np.array([[0.2, 0, 0], [0, 0, 0.2], [0, 0.2, 0]])

        # set rotation to best approximation
        R = Rt[:, :3]

        # change sign of x-axis
        R[0, :] = -R[0, :]
        # set translation
        t = Rt[:, 3]
        t[0] = -t[0]

        # setup 4*4 model view matrix
        M = np.eye(4)
        M[:3, :3] = np.dot(R, Rx)
        M[:3, 3] = t
        M[3, :3] = t

        # transpose and flatten to get column order
        M = M.T

        m = M.flatten()
        # replace model view with the new matrix
        glLoadMatrixf(m)

    ##############################################################Rt반환
    def _my_cal(self, image):
        find_H = fp()
        find_H.set_cv_img(image)
        find_H.start()
        kp, des = find_H.get_point()

        self.H = self.match_images(self.mark_kp, self.mark_des, kp, des)
        if self.H is not None:
            cam1 = camera.Camera(
                np.hstack((self.K, np.dot(self.K, np.array([[0], [0],
                                                            [-1]])))))
            # Rt1=dot(linalg.inv(self.K),cam1.P)
            cam2 = camera.Camera(np.dot(self.H, cam1.P))

            A = np.dot(np.linalg.inv(self.K), cam2.P[:, :3])
            A = np.array([A[:, 0], A[:, 1], np.cross(A[:, 0], A[:, 1])]).T
            cam2.P[:, :3] = np.dot(self.K, A)
            Rt = np.dot(np.linalg.inv(self.K), cam2.P)

            return Rt
        else:
            return None

    ##############################################################match image
    def match_images(self, kp1, des1, kp2, des2):
        matcher = cv2.BFMatcher()
        match_des = matcher.knnMatch(des1, des2, k=2)
        matches = []
        matA, matB = [], []
        matC = []

        for m in match_des:
            if m[0].distance < 0.8 * m[1].distance:
                matA.append(kp1[m[0].queryIdx])
                matB.append(kp2[m[0].trainIdx])
                matC.append(des1[m[0].queryIdx])

        if len(matA) > 50:
            ptsA = np.float32([m.pt for m in matA])
            ptsB = np.float32([n.pt for n in matB])
            H1 = []
            H1, status = cv2.findHomography(ptsA, ptsB, cv2.RANSAC, 5.0)
            H1 = self.homo_check(H1)
            self.mat_kp = np.array(
                [matB[i] for i in range(status.shape[0]) if status[i] == 1])
            self.mat_des = np.array(
                [matC[i] for i in range(status.shape[0]) if status[i] == 1])

            return H1
        else:
            return None

    ##############################################################homography check
    def homo_check(self, H1):
        if self.H is None:
            return H1
        else:
            if cv2.norm(H1, self.H) > 1.0:
                return H1
            else:
                return self.H

    def _draw_background(self):
        # draw background
        glBegin(GL_QUADS)
        glTexCoord2f(0.0, 1.0)
        glVertex3f(-4.0, -3.0, 0.0)
        glTexCoord2f(1.0, 1.0)
        glVertex3f(4.0, -3.0, 0.0)
        glTexCoord2f(1.0, 0.0)
        glVertex3f(4.0, 3.0, 0.0)
        glTexCoord2f(0.0, 0.0)
        glVertex3f(-4.0, 3.0, 0.0)
        glEnd()
        glDeleteTextures(1)

    def keyboard(self, *args):
        if args[0] == GLUT_KEY_UP:
            glutDestroyWindow(self.window_id)
            self.webcam.finish()
            sys.exit()

    ##############################################################OpenGL창 초기

    def main(self):
        # setup and run OpenGL
        glutInit()
        glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
        glutInitWindowSize(self.wid, self.hei)
        glutInitWindowPosition(200, 200)
        self.window_id = glutCreateWindow(b"OpenGL Glyphs")
        self._init_gl(self.wid, self.hei)
        glutDisplayFunc(self._draw_scene)
        glutIdleFunc(self._draw_scene)
        glutMainLoop()